From 01e0b8b14d1871169ba5ff551c6c23fb3557f770 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 02:10:55 +0200 Subject: [PATCH 01/15] FE-1241: Add isolated brownfield comparison cases Add history-free Brunch and Petrinaut profiles with controller-owned oracles so real repositories can enter the crossed comparison without exposing historical solutions. Co-authored-by: Cursor --- .changeset/brave-rivers-compare.md | 5 + memory/PLAN.md | 33 +- memory/SPEC.md | 15 + package.json | 4 +- src/.pi/extensions/__tests__/registry.test.ts | 46 +- .../extensions/agent-runtime/runtime/index.ts | 35 +- src/.pi/extensions/subagents/session.ts | 56 +- src/app/brunch-tui.ts | 31 +- src/app/pi-extensions.ts | 12 +- src/app/pi-subagents.ts | 13 +- src/dev/TOPOLOGY.md | 14 +- src/dev/end-to-end-comparison.ts | 15 + .../__tests__/case-profile.test.ts | 113 ++++ .../__tests__/execution-adapters.test.ts | 75 ++- .../factorial-browser-oracle.slow.test.ts | 23 +- .../__tests__/solution-isolation.test.ts | 291 +++++++++ .../end-to-end-comparison/brunch-adapter.ts | 6 + .../end-to-end-comparison/claude-adapter.ts | 47 +- .../pinned-source-preparation.ts | 156 +++++ .../end-to-end-comparison/public-packet.ts | 17 +- .../solution-isolation.ts | 585 ++++++++++++++++++ .../end-to-end-comparison/study-contract.ts | 40 +- src/dev/execution-comparison-brunch.ts | 75 ++- src/dev/execution-comparison-operator.ts | 148 ++++- .../__tests__/accessibility-contract.test.ts | 4 +- .../__tests__/brunch-lane.test.ts | 34 +- .../__tests__/case-contract.test.ts | 105 +++- .../execution-comparison-brunch.test.ts | 177 +++++- .../host-landing-oracle.slow.test.ts | 54 ++ .../__tests__/host-landing-oracle.test.ts | 135 ++++ .../__tests__/operator-cli.test.ts | 228 ++++++- .../operator-oracle-dispatch.test.ts | 106 ++++ .../__tests__/oracle-pack.test.ts | 90 +++ ...petrinaut-optimization-oracle.slow.test.ts | 63 ++ .../petrinaut-optimization-oracle.test.ts | 88 +++ .../pinned-source-preparation.test.ts | 189 ++++++ .../accessibility-contract.ts | 6 +- .../execution-comparison/browser-oracle.ts | 21 +- src/dev/execution-comparison/brunch-lane.ts | 146 ++++- src/dev/execution-comparison/case-contract.ts | 263 +++++++- .../host-landing-oracle.ts | 8 + .../host-landing-oracle/fixture.ts | 207 +++++++ .../host-landing-oracle/git-model.ts | 155 +++++ .../host-landing-oracle/runner.ts | 172 +++++ .../host-landing-oracle/types.ts | 64 ++ src/dev/execution-comparison/operator-cli.ts | 96 ++- src/dev/execution-comparison/oracle-pack.ts | 190 +++++- .../petrinaut-optimization-oracle.ts | 9 + .../petrinaut-optimization-oracle/browser.ts | 423 +++++++++++++ .../petrinaut-optimization-oracle/claims.ts | 62 ++ .../fake-optimizer.ts | 107 ++++ .../petrinaut-optimization-oracle/fixture.ts | 263 ++++++++ .../petrinaut-optimization-oracle/runner.ts | 109 ++++ .../petrinaut-optimization-oracle/types.ts | 31 + .../active-branch-reader-inventory.test.ts | 4 + .../missions/brunch-host-landing.md | 5 + .../missions/petrinaut-optimization.md | 5 + .../controller/requirement-registry.json | 64 ++ .../brunch-host-landing/shared-baseline.md | 14 + .../brunch-host-landing/study-contract.json | 52 ++ .../controller/requirement-registry.json | 90 +++ .../petrinaut-optimization/shared-baseline.md | 12 + .../study-contract.json | 52 ++ .../controller/oracle-manifest.json | 32 + .../brunch-host-landing/public-contract.json | 38 ++ .../cases/brunch-host-landing/spec.md | 35 ++ .../controller/oracle-manifest.json | 45 ++ .../public-contract.json | 51 ++ .../cases/petrinaut-optimization/spec.md | 50 ++ 69 files changed, 5899 insertions(+), 110 deletions(-) create mode 100644 .changeset/brave-rivers-compare.md create mode 100644 src/dev/end-to-end-comparison/__tests__/case-profile.test.ts create mode 100644 src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts create mode 100644 src/dev/end-to-end-comparison/pinned-source-preparation.ts create mode 100644 src/dev/end-to-end-comparison/solution-isolation.ts create mode 100644 src/dev/execution-comparison/__tests__/host-landing-oracle.slow.test.ts create mode 100644 src/dev/execution-comparison/__tests__/host-landing-oracle.test.ts create mode 100644 src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts create mode 100644 src/dev/execution-comparison/__tests__/oracle-pack.test.ts create mode 100644 src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts create mode 100644 src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts create mode 100644 src/dev/execution-comparison/__tests__/pinned-source-preparation.test.ts create mode 100644 src/dev/execution-comparison/host-landing-oracle.ts create mode 100644 src/dev/execution-comparison/host-landing-oracle/fixture.ts create mode 100644 src/dev/execution-comparison/host-landing-oracle/git-model.ts create mode 100644 src/dev/execution-comparison/host-landing-oracle/runner.ts create mode 100644 src/dev/execution-comparison/host-landing-oracle/types.ts create mode 100644 src/dev/execution-comparison/petrinaut-optimization-oracle.ts create mode 100644 src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts create mode 100644 src/dev/execution-comparison/petrinaut-optimization-oracle/claims.ts create mode 100644 src/dev/execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts create mode 100644 src/dev/execution-comparison/petrinaut-optimization-oracle/fixture.ts create mode 100644 src/dev/execution-comparison/petrinaut-optimization-oracle/runner.ts create mode 100644 src/dev/execution-comparison/petrinaut-optimization-oracle/types.ts create mode 100644 testing/comparisons/missions/brunch-host-landing.md create mode 100644 testing/comparisons/missions/petrinaut-optimization.md create mode 100644 testing/end-to-end-comparisons/cases/brunch-host-landing/controller/requirement-registry.json create mode 100644 testing/end-to-end-comparisons/cases/brunch-host-landing/shared-baseline.md create mode 100644 testing/end-to-end-comparisons/cases/brunch-host-landing/study-contract.json create mode 100644 testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json create mode 100644 testing/end-to-end-comparisons/cases/petrinaut-optimization/shared-baseline.md create mode 100644 testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json create mode 100644 testing/execution-comparisons/cases/brunch-host-landing/controller/oracle-manifest.json create mode 100644 testing/execution-comparisons/cases/brunch-host-landing/public-contract.json create mode 100644 testing/execution-comparisons/cases/brunch-host-landing/spec.md create mode 100644 testing/execution-comparisons/cases/petrinaut-optimization/controller/oracle-manifest.json create mode 100644 testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json create mode 100644 testing/execution-comparisons/cases/petrinaut-optimization/spec.md diff --git a/.changeset/brave-rivers-compare.md b/.changeset/brave-rivers-compare.md new file mode 100644 index 000000000..bed65c4f2 --- /dev/null +++ b/.changeset/brave-rivers-compare.md @@ -0,0 +1,5 @@ +--- +"@hashintel/brunch": minor +--- + +Add isolated Brunch and Petrinaut brownfield comparison cases. diff --git a/memory/PLAN.md b/memory/PLAN.md index 0d8b86515..052f151cd 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -21,7 +21,7 @@ Brunch-next has delivered the original composition spine: the host, sealed Pi pr **Parallel lanes.** Group 3 agent-layer work is pickup-ready. `agent-control-plane-closure` is one earned coverage frontier over foreground and background prompt ingress: it absorbs the former `graph-assurance-conduct` and `subagent-skill-access` items, reconciles the stale eager-context fossils left by D58-L's load-on-demand cutover, and closes prompt replacement/resource-invocation contradictions before capture experiments amplify them. FE-1187 retains only the concern-grouped review renderer. FE-1208's reshaped `automation-observability-dx` closure landed on the restacked base; its invalid warrant pilots remain historical evidence, while the useful evaluator/report primitives are available to `capture-ledger-tracer`. Two evaluation use cases stay deliberately distinct: **seed-based intra-product testing** (Brunch on its own terms — seeds encode Brunch graph/spec state only Brunch understands) and **mission-driven agent-as-user cross-product comparison** (`agent-as-user-comparison` — competitor CLIs share no Brunch state, so the comparable artifact is the "ready" spec/plan document a mission produces). FE-1210 closed the rigorous technical tracer on 2026-07-17; the separately planned operator workflow (`operator-comparison-workflow`, FE-1215) makes saved missions, visible comparison-harness framing, conversational launch, and a readable free-form report approachable without weakening the retained regression/evaluation machinery. Its first operator-led run falsified the nested fresh-actor topology: the top-level project Pi session must act as the simulated user and drive one direct harness subshell at a time, with stock-Pi text interaction as the portable baseline; the corrected real witness is deferred to `saved-mission-comparison-witness`. The KA stream owns executor/orchestrator/Execute-mode work. FE-1192 and FE-1195 completed the attempt, isolated fan-in, durable parallel authority, and epic integration sequence; `executor-plan-coherence` now uses those settled seams to test whether multi-slice plans converge on one working result. Broader instrumentation remains trigger-gated under Later. -**End-to-end comparison lane.** FE-1239 materializes the handoff that the completed reporting procedure names but does not run: two fresh rigorous elicitation lanes produce exact approved specifications, each specification crosses an immutable content-addressed handoff into both Brunch and Claude execution, and the unchanged Petri-editor oracle joins all four outputs into requirement-level evidence. Shared interface requirements are disclosed before elicitation and remain controlled baseline, while private/reveal requirements stay controller-only. The first tracer supports bounded within-case contrasts only — no winner, reliability claim, or cross-case causal conclusion. +**End-to-end comparison lane.** FE-1239 materializes the handoff that the completed reporting procedure names but does not run: two fresh rigorous elicitation lanes produce exact approved specifications, each specification crosses an immutable content-addressed handoff into both Brunch and Claude execution, and the unchanged Petri-editor oracle joins all four outputs into requirement-level evidence. Shared interface requirements are disclosed before elicitation and remain controlled baseline, while private/reveal requirements stay controller-only. The first tracer supports bounded within-case contrasts only — no winner, reliability claim, or cross-case causal conclusion. The planned `brownfield-comparison-cases` frontier keeps that Petri editor as the sole greenfield case and extends the same exact-handoff discipline to one pinned backend feature in Brunch and one pinned frontend feature in Petrinaut; Clay is not part of the campaign. **Current seams.** Brunch ships on the `1.0.0-alpha.x` line. One-shot `ask` is the only interactive structured-exchange terminal; D125-L's live ask registry provides headless discovery/answering, while the transcript-backed pending projection remains a compatibility surface for live offer tools after the legacy `present_question` pending branch retired. Sweep classification remains fail-closed and compile-time anchored to the exchange-schema terminal names (D117-L), while the larger capture-conditional watermark question remains A40-L. @@ -119,6 +119,7 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand - `executor-slice-admission-parity` ([FE-1240](https://linear.app/hash/issue/FE-1240/prevent-invalid-scoped-slices-from-reaching-execution)) — **complete on `ka/fe-1240-slice-admission-parity`, restacked on `next` after FE-1239 landed:** incomplete scoped-slice worker context is rejected during deterministic plan admission, exact findings enter bounded repair before `slice_execute`, and the execution boundary remains fail-closed. Definition below. - `executor-plan-coherence` ([FE-1250](https://linear.app/hash/issue/FE-1250/build-coherent-execution-plans)) — **implementation complete on `ka/fe-1250-coherent-execution-plans`, based on FE-1240 via `next`:** frontier-verified multi-slice epics now require one ordinary terminal member over every sibling, planner conduct establishes shared foundations, and workers preserve cumulative public contracts for the canonical harness. Fast verification passes; next is the separately authorized unchanged Petri comparison rerun. No new browser gate, durable plan kind, or executor lifecycle phase. Definition below. - `comparison-publication-workflow` ([FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports)) — **active on `ka/fe-1251-comparison-publication`, stacked on FE-1250:** capture immutable controller and release provenance before comparison lanes start, then explicitly publish retained validity-first reports into the canonical Notion database through a guarded idempotent skill. Definition below. +- `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases)) — **active on `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240:** A49-L's versioned fail-closed historical-replay admission path and the frozen Brunch backend host-landing profile/controller oracle are complete. Preserve the minimal Petri-net editor as the only greenfield case, then replay Petrinaut optimization UI PR #9051 in the full `hashintel/hash` repository. A50-L is invalidated and D136-L selects the focused `/optimization` route for the common hard gate. Next: build the revised Petrinaut profile/oracle before any historical provider run. - **Carved from FE-1167 (2026-07-13):** the Execute-mode evidence sub-list — Execute entry beats on thin vs rich seeds (assessment honesty: Ask on thin, Proceed on rich) and the FE-1107/KA residue (close-or-narrow, demo/walkthrough session via `TESTING_PLAN.md`, post-KA plan pass). The former sticky-posture question is no longer KA residue: FE-1187 `remediation-4` owns the persistent Specify elicitation-style audit/SPEC revision, and its Continue lexical audit owns the old `continue` ambiguity. Full context in the archived FE-1167 definition (`docs/archive/PLAN_HISTORY.md`). - `planning-process-model` — **moved to the KA stream 2026-07-13; reshaped by D126-L**: the durable scope handoff is settled, so this item now owns only plan projection and epistemic-horizon questions beyond committed scopes. Definition below. - **[1.x data-model handoff owed by FE-1187](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md):** a concise colleague-facing note must distinguish current canon from directional vocabulary before the consolidated outer checkpoint. Current: `{milestone, frontier, scope}` with executor-derived slices; basis (`explicit | implicit`) orthogonal to settlement (`advisory | settled`); no persisted readiness grade or spec-global elicitation-gap table; active-branch Pi JSONL reads; one CommandExecutor mutation authority with spec-local LSN/change log; no new projected `vv_obligation` (legacy rows remain readable). Directional only: explain stored `thesis` as pitch/concept without renaming it yet, and avoid new coupling to spec-local `term` while its possible workspace lift remains future work. Persisted judgment-shaped reconciliation needs remain current, but are YAGNI-suspect: add no kinds/consumers/orchestration dependency without fresh evidence; re-evaluate derivation/removal when KA work first needs that table. Delivered note: [`Brunch 1.x data-model handoff`](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md); it does not block the deterministic TUI queue. @@ -456,6 +457,29 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Boundary:** no hidden-oracle exposure, inferred browser command, new plan-plane node, candidate command surface, execution-time plan repair, or FE-1241 comparison-framework change. - **Current execution pointer:** none; the two-card scope is consumed. Re-enter only for the unchanged frozen Petri comparison rerun. +### brownfield-comparison-cases + +- **Name:** Add Brunch and Petrinaut brownfield end-to-end comparison cases +- **Linear / branch:** [FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases), child of FE-1211; `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240. +- **Kind:** structural evaluation expansion — parameterized case/substrate/oracle composition plus two repository-backed end-to-end cases; one frontier with several scoped slices, not one frontier per case. +- **Certainty:** proving. +- **Status:** active 2026-07-22; the fail-closed brownfield materialization/solution-isolation tracer is complete and A49-L is retired. Both frozen case/oracle slices are complete: Brunch uses black-box `/brunch:land` plus an independent Git model, and Petrinaut's operator now materializes the full pinned HASH tree from an explicit controller source, freezes the exact handoff, and performs the closed immutable install before each network-denied lane; D136-L's focused builds and standalone `/optimization` fake-optimizer oracle run only after lane termination. A50-L remains invalidated and fully provisioned HASH `/processes/draft` host/iframe integration remains non-gating outer evidence. Historical Petrinaut provider runs remain unbuilt. +- **Objective:** prove that FE-1239's exact elicitation → immutable handoff → crossed Brunch/Claude execution → requirement-ledger path works against real pinned brownfield repositories without weakening controller isolation or turning the Petri tracer into generic campaign machinery. +- **Lights up:** pinned repository + private feature mission + shared public baseline → two codebase-aware elicitation lanes → two exact handoffs applied to the same base tree → four isolated execution cells → repository-specific controller oracle → validity-first requirement ledger, once for Brunch and once for Petrinaut. +- **Stabilizes:** one data-driven case profile across `greenfield` and `brownfield`; immutable repository/base-tree identity beside immutable spec identity; clean same-base materialization for every lane; a closed controller-only immutable dependency install before runtime denial; compile-time oracle dispatch; and diff/evidence collection that excludes harness seed files without widening FE-1230's `ExecutionAttempt`. +- **Selected replay — Brunch backend:** source FE-1201 / [PR #336](https://github.com/hashintel/brunch/pull/336), base `f5a423b19f76cf345d88053456870a126e451618`, merged reference `0092a5498d22603eeb22529e8823b365ff59b505`. Derive only the host-landing backend behavior and its temp-repository safety contract; TUI/RPC/web presentation changes in the historical PR are outside the mission, so the whole reference diff is not a golden patch. +- **Selected replay — Petrinaut frontend:** source FE-1162 UI slice / [PR #9051](https://github.com/hashintel/hash/pull/9051), full `hashintel/hash` base `5c7a2d9db5caa851c38938f4b1bac19005b0e978`, merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0`. Replay only the optimization UI; optimizer backend/API capabilities already present at the base commit are shared public baseline, not implementation work or elicitation gain. +- **A49-L retired (2026-07-21):** the live Claude probes established provider continuity while arbitrary target network and forbidden-root reads failed; the integrated v1 admission path records source commit/tree identity, produces one declared synthetic target commit with no worktree residue, remotes, or later refs, and rejects target/symlink escape, missing/weakened policy, unbranded verifier, or unsupported host. Claude uses strict empty MCP, explicit non-web tools, `dontAsk` plus an empty network allowlist, canonical forbidden-root denial, empty ambient setting sources/plugin settings, and `failIfUnavailable`; Brunch removes web/research/Specify-subagent surfaces, retains planner/worker, rejects lexical and symlink escape from foreground/child file tools, and injects a verifier that denies runtime network plus sealed-root reads/writes. The selected historical patches require no package dependency absent from their parent trees, so controller-side preparation can precede runtime denial. +- **Retired:** A49-L; later historical case launches must consume this admission path and may not fall back to prompt-only conduct. +- **Why now / unlocks:** FE-1239 proved the full chain only on an empty greenfield browser app, and FE-1240 closes the slice-admission defect that stopped both Brunch execution cells. The next load-bearing unknown is whether the same contracts survive repository inspection, pinned pre-existing code, and feature-delta verification. +- **Boundary:** retain `minimal-petri-net-editor` unchanged as the sole `greenfield / whole_application / frontend` case; add exactly the derived FE-1201 host-landing slice as `brunch / brownfield / single_feature / backend` and PR #9051 as `petrinaut / brownfield / single_feature / frontend`. Petrinaut receives the full pinned HASH monorepo, not an extracted library fixture. The framework may parameterize case id, frozen axes, repository substrate, delivery contract, and sealed oracle id, but it must not accept arbitrary manifest shell commands or make controller material or historical solution refs target-visible. +- **Acceptance:** the existing Petri case, hashes, browser-oracle behavior, and promoted witness remain valid; both brownfield cases freeze content-addressed mission/baseline/registry/execution/oracle packets; every elicitation and execution lane starts from a separate clean checkout of the same declared base tree with no later refs/remotes or reachable historical-solution services; exact approved spec and contract bytes cross unchanged; pinned cases require an explicit safe controller source and record the compiled immutable install without tracked-source mutation before runtime network denial; harness seed commits are identified separately from implementation diffs; controller roots remain unreachable; a compile-time registry dispatches each known oracle; known-good fixtures pass and one focused negative fixture per case proves oracle sensitivity; each real case retains exactly two elicitation handoffs and four execution attempts with complete cleanup and requirement evidence or `not_assessable`. +- **Verification:** per `memory/SPEC.md` §Verification Design. Inner — case-profile, base/tree identity, materialization, strict solution-isolation, path-isolation, oracle-dispatch, and handoff/matrix contracts. Middle — unchanged Petri regression; Brunch black-box `/brunch:land` TUI journeys resume a settled session under `PI_OFFLINE=1` and are judged by an independent full-range Git model over disposable repositories; Petrinaut deterministic fake-provider browser, host/iframe bridge, and accessibility journeys; focused final-commit-only/behavioral rivals prove sensitivity. Outer — optional real-optimizer smoke, masked Petrinaut visual review, masked code-quality review, then one fresh staged 2×2 run and promoted bounded report per brownfield case. Historical code/diff/architecture similarity is never an oracle. +- **Cross-cutting obligations:** public baseline controls only addressability needed by the common oracle and is never credited as elicitation gain; existing repository knowledge may be inspected equally by both elicitation lanes; private mission/reveal/oracle files never enter target workspaces; failed and invalid attempts remain retained; Brunch stops at `promotion_prepared` and never lands into either source repository; provider/model/budget/intervention rules remain frozen per study. +- **Explicitly out:** Clay/Pi Campaign Machine; any second greenfield case; whole-repository rewrites; evolving-plan studies; reliability repetitions; automatic scoring or cross-case winner; live production deployment; Cursor/Codex lanes; arbitrary oracle plugins; production `/compare-end-to-end`; landing provider outputs into either source repository; and FE-1230 `ExecutionAttempt` schema widening. +- **Traceability:** D70-L fixture taxonomy; D134-L/I67-L controller/mission isolation; D40-L/D120-L/I62-L execution and no-landing boundaries; FE-1210/FE-1230/FE-1232/FE-1239/FE-1240; [`docs/praxis/comparison-runs.md`](../docs/praxis/comparison-runs.md); Notion `brunch Test Plan` axes. +- **Current execution pointer:** [`memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md`](cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md) is complete, including the operator-level pinned materialization and pre-lane dependency-install correction. Next scope the first Petrinaut historical provider run and separately owned host-integration outer evidence; neither may weaken or replace the frozen standalone mechanical hard gate. + ### comparison-reporting-skills - **Name:** Report comparison evidence @@ -632,7 +656,7 @@ KA stream: lights_up: frozen spec -> Brunch/Claude isolated lanes -> hidden browser/Petri oracles -> adjudication excludes: host landing | product operator command | Cursor/Codex | broad benchmark claims -[hard]-> end-to-end-comparison-tracer (FE-1239) - status: active; staged 2x2 Petri-editor tracer + status: complete 2026-07-21; promoted staged 2x2 Petri-editor tracer reuses: FE-1210 fresh actor | FE-1230 attempt/oracle contracts | FE-1232 reporting grammar lights_up: mission -> exact elicited specs -> four execution cells -> requirement traceability excludes: winner | repetitions | multi-case causality | product command | ExecutionAttempt schema widening @@ -644,6 +668,11 @@ KA stream: lights_up: committed scope -> coherent candidate plan -> ordinary reconciliation slice reuses: canonical harness | isolated integration | epic verification excludes: browser-specific executor gate | durable plan kind | new lifecycle phase + -[hard]-> brownfield-comparison-cases + status: active FE-1241; A49-L and Brunch profile/oracle complete, Petrinaut scope revised to standalone hard gate after A50-L invalidation + lights_up: same-base brownfield elicitation -> exact handoff -> crossed execution -> case oracle + cases: minimal-petri-net-editor = sole greenfield reference | FE-1201-derived brunch backend | PR #9051 petrinaut frontend + excludes: Clay | more greenfields | repetitions | aggregate winner | ExecutionAttempt widening # executor-run-environment (FE-1166) resolved 2026-07-15: policy merged (PR #302), # live remainder folded into executor-plan-synthesis (FE-1197), card consumed. diff --git a/memory/SPEC.md b/memory/SPEC.md index 2bf714c91..224da5602 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -127,6 +127,8 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | A46-L | A closed Zod schema can represent every `SessionPresentationDelta`, including `OpenAsk` questions with questionnaire questions present or exactly absent. **Validated 2026-07-15:** the contract composes the owned question schemas as exact alternatives and round-trips all delta variants without a cast or restated owner type. | medium | validated | D133-L; `src/rpc/__tests__/standalone-web-session-host.contract.test.ts` | | A47-L | Pi's valuable `InteractiveMode` TUI behavior can be preserved while one independent cwd-scoped Brunch session host owns the sole writable sealed Pi runtime, JSONL session manager, graph command authority, and semantic live-event fan-out used by both TUI and React clients. The unknown is the TUI attachment seam: Pi exports `InteractiveMode` over an in-process `AgentSessionRuntime`, not a remote TUI client. Validation must prove a real TUI + browser target without a second writable runtime, raw-Pi browser contract, or permanent second relay. Frontier: `shared-session-host-tracer`; retirement/cutover: `shared-session-host-cutover`. | medium | open | D39-L, D132-L, D133-L; I64-L, I65-L | | A48-L | A read-only semantic preflight can improve orientation-menu availability over deterministic graph-fact heuristics while returning within a ≤3-second interaction budget often enough to justify a model-backed path. Admission is limited to the configured soft recommended evaluator model; other foreground selections, missing evaluator auth, timeout, malformed output, or failure use the deterministic safe subset without restricting Pi-native `/model`. The path must consume no foreground turn, write no transcript/graph/session truth, and cache only in process by `{specId, lsn, operationalMode}`. A Brunch-owned reconciliation-blocker reader participates in gating now with an explicit empty implementation; injected non-empty blockers veto availability so future derived/persisted blocker wiring cannot be forgotten. Validation: a tracked human-approved contrastive catalog plus structured scratch tracer report; first run three uncached feasibility calls, then only if plausible run ten uncached labeled cases, requiring ≥8/10 within 3 seconds, exact flags on every completed response, at least one named deterministic-fallback miss corrected, and zero false-positive moves. If the budget or quality gate fails, retire only the model path, not the deterministic fallback. | low | open | D74-L, D109-L, D123-L; `walkthrough-remediation-2` | + +| A50-L | The real HASH `/processes/draft` host/iframe route can be the Petrinaut comparison's common hard-gate address under focused controller preparation, with unrelated auth/backend behavior cheaply stubbed. **Invalidated 2026-07-21:** the route source accepts a backend-free draft, but the Next application shell fails before rendering without generated Panda and GraphQL artifacts plus built workspace dependencies. An immutable install fetched 3,570 packages (2.79 GiB); frontend dependency preparation fanned into 61 packages including Rust/toolchain work and stopped on a pinned mise-version mismatch. By contrast, focused design-system/Petrinaut builds brought the merged standalone `/optimization` route to an accessible Optimizations view without HASH backend state. | high | invalidated | D136-L; `brownfield-comparison-cases` | ### Active Decisions @@ -304,6 +306,7 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D132-L | Standalone interactive web uses one cwd-scoped combined Brunch host with a target-addressed inventory of sealed, in-process Pi `AgentSession`s (2026-07-14). The same process serves React assets/WebSocket Brunch RPC and owns coordinator/graph authority; it does not construct `InteractiveMode`, expose raw Pi RPC, spawn one Pi child per session, host multiple projects, or promise in-flight survival across host restart. One durable session target has one driver/many observers and cannot be opened as duplicate writable runtimes; write leases wait for real same-session contention. Hosted-session mutations return the complete `LiveSessionHostResult` discriminated `{status}` union as JSON-RPC success payloads, including domain refusals; only malformed boundary input and thrown host failures use JSON-RPC errors. FE-1200 materialized the one-target path and validated simultaneous target isolation (A43-L). Depends on: D5-L, D10-L, D33-L, D39-L, D84-L; req 4, req 31. Supersedes: D10-L/D72-L read-only-sidecar posture and D84-L singleton/TUI-owned target topology. | [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md), [`src/session/TOPOLOGY.md`](../src/session/TOPOLOGY.md), [`src/rpc/TOPOLOGY.md`](../src/rpc/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — target-addressed host and concurrent-session isolation materialized 2026-07-14 | | D133-L | Web and TUI share transport-neutral presentation semantics, not platform components or separately-authored domain decoders (2026-07-14). Validated Brunch `toolResult.details` project to a shared semantic presentation model; LLM-context, TUI, and React adapters render that meaning for their audiences. Web hydrates from a named JSONL-derived product projection and overlays target-addressed live events, then refetches canonical truth at settlement/reconnect. The host emits neither ANSI/TUI strings nor ready-made HTML, and no chat mirror/event store is introduced. FE-1200 materialized the full required persisted family inventory: ordinary text; free-text, choice, choices, and bounded-questionnaire terminal read-back; candidate, review-set, and digest offers/continuations; and receipt-bearing review settlement. Live React controls answer free text and listed single/multi choices; bounded questionnaires remain answerable headlessly through D38-L's schema-tagged string/JSON envelope, without a dedicated React questionnaire form. Depends on: A44-L, D17-L, D19-L, D104-L; req 12, req 17, req 32. | [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — full required-family coverage materialized 2026-07-15 | | D134-L | The approachable `/compare-specs` control topology is one top-level project Pi session acting as the simulated user and driving exactly one comparison-harness interactive subshell at a time. It does not spawn a Pi actor that then opens a nested interactive shell. The top-level agent alone receives the private mission; each harness receives only its approved minimal framing and the user's natural messages. Shared top-level context and lane order are acceptable and disclosed for this exploratory workflow; isolation-sensitive studies continue to use FE-1210's separate rigorous campaign recipe. Ordinary text interaction is the stock-Pi baseline for choices and approvals; a custom structured-question tool may enhance presentation but is never required. Setup checks are bounded to actual selected-harness prerequisites—no throwaway Pi/Claude provider turns or synthetic actor launches on every run. Supersedes: FE-1215's unmaterialized fresh-nested-actor design note. | [`.pi/prompts/compare-specs.md`](../.pi/prompts/compare-specs.md), [`testing/comparisons/missions/README.md`](../testing/comparisons/missions/README.md); PLAN `operator-comparison-workflow` | active — remediation materialized and focused Brunch smoke witnessed 2026-07-17; full comparison witness deferred | +| D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller preparation boundary. Before the network-denied candidate lane, the controller materializes the declared parent tree and exact handoff, then runs only `corepack yarn install --immutable --mode=skip-build`; this closed immutable install is the sole package-registry-network allowance and must leave tracked source clean. After the lane terminates at `promotion_prepared`, the controller runs the closed focused package builds and launches the standalone website `/optimization` route. A deterministic loopback fake optimizer grades capability-present UI, scenario-first configuration, fixed/optimized flat bindings, one saved/custom metric plus direction, request construction, progressive trials/best/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and accessibility. The real HASH `/processes/draft` host/iframe integration is not a common hard gate: its global Next shell requires unrelated GraphQL/codegen and broad 61-package platform/toolchain preparation, so one fully provisioned host/iframe smoke plus masked code/visual review remain explicit non-gating outer evidence and report `not_assessable` when unavailable. Depends on: A50-L, D134-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases`; Petrinaut scope card | active — materialized 2026-07-22 | | D60-L | Agent context splits into pull / projection / render / surface, distinguishes graph-truth from active-context reads, and keeps `workspace.state` separate. Agent context (what the agent reasons over) spans `cwd` (filesystem kickoff heuristic — `.brunch?`, session count/length, README/markdown sizes, file counts), `graph` (overview/list/query), and `node` (variable-hop neighborhood). ... | [`src/graph/TOPOLOGY.md`](../src/graph/TOPOLOGY.md), [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md) | active | | D83-L | Context-render house style: a markdown frame (md-pen) with TOON for uniform data and a fenced ASCII tree for hierarchy, wrapped in `
` tags; agent context clusters into `` / `` / `` scopes. Refines D60-L's RENDER stage. LLM-facing agent-context renders adopt one consistent dialect instead of ad-hoc `[bracket]` + bullet lists: - **Audience scope.** `src/agents/contexts/` owns the LLM agent-context dialect (the `
` scope-clustering plus TOON data blocks and t ... | See archive snapshot for full rationale. | active | | D85-L | Suspended prompt-resource axis model: strategy/lens/method are no longer runtime state. A 2026-06-18 grill consolidation of the `agents/skills/` topology and the D58-L manifest axes, implemented across FE-893, FE-861, and FE-898, produced useful prompt-resource content and path topology. ... | [`src/agents/TOPOLOGY.md`](../src/agents/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md) | retired/suspended | @@ -688,6 +691,8 @@ For agent-as-user evaluation, the primary behavioral claim is **consequential-fa **FE-1230 greenfield execution-comparison assessment (2026-07-20).** Observability is **partial**: build/test output, git trees, browser DOM/accessibility state, console errors, JSON downloads, Brunch run/Petri artifacts, and target-visible interactions are text-native, while ordinary visual hierarchy and drag feel remain human judgments; product-private reasoning and diagnostics are excluded from common evidence. Reproducibility is **partial**: the approved Petri-editor specification, empty-repository base, public automation contract, Opus 4.8 model, budgets, and hidden oracle bytes freeze before the first valid lane, but generated implementation shape and model conduct vary; three Brunch repetitions detect gross instability rather than reliability tails. Controllability is **partial**: the controller can install and run unchanged build, browser, reference-model, round-trip, and negative-space checks against fresh lane outputs, while provider availability and product-native planning remain external. A minimal public accessibility contract gives both lanes stable roles/names without revealing hidden scenarios or expected results. +**Brownfield end-to-end comparison assessment (active 2026-07-21).** Observability is **partial**: pinned base/final trees, diffs, test/build output, controller results, and Brunch temporary-host Git state are text-native; Petrinaut interaction quality and progress presentation remain partly visual. Reproducibility is **partial**: historical parent commits, sanitized missions, exact handoffs, deterministic fake optimization responses, and temp-repository fixtures can freeze the mechanical conditions, while elicitation/execution model conduct still varies and the optional real optimizer remains external. Controllability is **partial with a fail-closed isolation route**: Brunch host landing can run entirely against temporary Git repositories and Petrinaut's hard gates can use a fake provider; A49-L is retired by one versioned admission contract over asymmetric Claude/Brunch controls, history-free source materialization, target-bounded reads, and network-denied verifier commands. For the first Brunch slice specifically, Git outcomes are highly observable and reproducible: the frozen profile and compiled claim-linked oracle now resume a settled session under `PI_OFFLINE=1`, drive `/brunch:land` through the public TUI, and judge disposable repositories with an independent Git model; fresh sessions fail closed because they can auto-kick the configured provider. Petrinaut/profile and all historical provider work remain gated behind later scoped tracers. + **Standalone-web tracer assessment (2026-07-14).** Observability is **partial**: JSONL, target-addressed RPC, semantic event frames, and accessible DOM/text states make structural session behavior observable; stream cadence, visual hierarchy, and error feel remain manual. Reproducibility is **high** for this tracer: paired temporary production boots use the deterministic faux provider and an existing JSONL session, not a live model or a static transcript golden. Controllability is **partial**: the middle loop controls the browser journey and cache/overlay loss, while a workbench walkthrough owns the bounded UX verdict. The normalizer may remove only declared nondeterministic ids/timestamps; it must not mask Brunch binding/runtime/exchange differences. **Combined trajectory/evaluation assessment.** The completed interactive TUI driver raises presentation-path controllability but does not by itself close causal attribution, real-provider reproducibility, or semantic ground truth. The first evaluation frontier should address those gaps with a minimal text-native legibility envelope over existing Pi lifecycle events, provider-payload introspection, Pi JSONL, TUI observations, and graph readback — not with OTel adoption. Its run/scenario/report contract must be product-neutral enough for later Claude Code or Cursor adapters, while Brunch-only trajectory enrichment remains optional diagnostic evidence. Campaigns are automated through a controlled user actor; deterministic checks own structural facts, and a small human-labeled set calibrates and audits semantic judgments. @@ -767,6 +772,10 @@ Dev-loop artifacts route to gitignored `.fixtures/scratch///`, res | Middle | **FE-1230 reference Petri model differential + metamorphic checks** | A tiny controller-owned P/T reference model independently computes enablement and weighted firing; the rendered marking/enablement agrees before and after firing, reset is idempotent, reload returns to the initial marking, export/import preserves structure, and disabled firing plus invalid operations leave state unchanged. | | Middle | **FE-1230 lane/process evidence contract** | Fresh repository identity, pinned model/product versions, budgets, start/end state, final git range, build/test/browser output, retries, interventions, terminal status, and cleanup are retained for every valid, failed, and invalid attempt. Only signals shared across lanes enter comparison packets; unavailable permission/cost/private-tool evidence is `not assessable`. | | Outer | **FE-1230 split execution judgment** | Identity-masked final trees/diffs plus common mechanical results receive criterion-level code/outcome review; a separate unblinded packet judges visible planning/execution conduct. Visual review mechanically rejects only catastrophic unusability (app absent, controls unreachable, interactions impossible); ordinary hierarchy, clarity, and feel remain qualitative and cannot override a mechanical failure. | +| Inner | **Brownfield case, repository-identity, and solution-isolation contracts** | The minimal Petri editor remains the sole greenfield profile; Brunch and Petrinaut profiles name frozen axes, parent commit/tree identity, sanitized mission/baseline/registry hashes, sealed compile-time oracle ids, and exact handoffs. Separate checkouts start from the same tree, contain no later refs/remotes or controller paths, and fail preflight if GitHub/Linear/Notion solution access is possible under the lane policy. | +| Middle | **Brunch black-box `/brunch:land` journey plus independent Git model** | The controller creates disposable host/run/review repositories, resumes a settled candidate session under `PI_OFFLINE=1`, drives only the public TUI command, snapshots Git and run metadata before confirmation, then compares post-apply state with a controller-owned full-range reference model. This proves no pre-confirm mutation, complete multi-slice landing rather than final-commit-only application, mode-aware brownfield integration and greenfield materialization, bookkeeping exclusion, honest failure, and terminal `landed` without importing candidate host-landing modules or permitting a provider turn. | +| Middle | **Petrinaut standalone fake-provider browser and accessibility oracle** | In the full pinned HASH checkout, controller-prepared focused Petrinaut packages launch the public `/optimization` website route. A deterministic loopback fake optimizer proves capability-present Optimizations visibility, scenario-first configuration, fixed/optimized parameter selection, one metric plus direction, request construction, streamed trials/best-so-far/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and reachable accessible controls without requiring the real optimizer service. | +| Outer | **Brownfield live/qualitative evidence** | One fully provisioned HASH `/processes/draft` host/iframe smoke checks capability relay, optimizer proxying, iframe isolation, and adapter reality; one optional real-optimizer Petrinaut smoke checks live response compatibility. Identity-masked visual review judges Petrinaut hierarchy, clarity, and progress legibility; identity-masked code review judges both outputs. These are non-gating findings, are `not_assessable` when unavailable, and cannot override or repair common mechanical failures. | | Middle | **FE-1187 controlled provider conduct gate — paused** | On explicit re-entry, reconcile the extractor/oracle against the landed mixed-settlement contract, then run three fresh normalized-ingest samples. All must use free-text digest feedback, one bounded questionnaire when several questions exist, no combinatorial options, an honestly assigned per-node/per-edge review proposal when review is used, exact atomic settlement preservation, and no post-review mutation that completes or rewrites the approved proposal. Direct advisory mutation without review remains valid. Reports retain provider/model stamps and deterministic conduct markers; the stopped 2026-07-17 run is diagnostic and counts 0/3. | | Inner | **FE-1187 Impact Ledger golden + word-wrap-tolerant render-honesty** | Golden/inline snapshots at narrow/normal/wide widths lock populated-section-only canonical order, absence of empty heading/`None` pairs, per-node/per-edge settlement visibility, elision, `refs:` row shape, and the `obligation` fallback label for the borderless Impact Ledger renderer (D27-L/D131-L). `missingRenderedDetailsLeaves` is extended to reassemble `table`'s word-wrapped physical lines back into logical cells before leaf-presence checking, so a value silently split across wrapped rows cannot pass as "rendered" by accident. | | Middle | **FE-1187 Impact Ledger differential reference extractor** | A deliberately naive reference extractor (flat node/edge/term inventory, no styling or grouping) is compared against the ledger's code/connection inventory over the witnessed fixture plus hand-authored edge fixtures (empty group, single-node group, mixed-settlement group, term-only group, max-refs group). Proves item/status inventory completeness and empty-group omission independent of the "real" renderer's own logic. | @@ -842,6 +851,7 @@ The first required probe is M0: after manual TUI interaction, a checker proves ` - **Operator-led cross-product comparisons (FE-1215; D134-L remediation before later `saved-mission-comparison-witness` evidence).** The PM-facing comparison door is deliberately distinct from rigorous frozen-packet evaluation. One project Pi prompt conversationally creates or revises a rich private **agent-as-user mission** for a simulated user: their objective, context, priorities, preferences, constraints, knowledge, uncertainty, decision latitude, and conversational posture. The invoking top-level Pi agent receives that mission and directly performs the user's side of each interaction while driving exactly one comparison-harness subshell at a time; it must not spawn a Pi actor that opens another interactive shell. Each comparison harness receives only minimal visible framing plus the opening user message and subsequent mission-grounded answers. Harness selection and framing are run setup, not mission content. Ordinary conversational text is the portable baseline for operator choices and approvals; environment-specific structured-question tools are optional presentation only. Setup checks cover actual selected-harness prerequisites without synthetic actor/provider turns on every run. Editable missions live outside `.fixtures/` under `testing/comparisons/missions/`; each run snapshots the private mission, separately identified target-visible setup/interactions, and outputs under `.fixtures/runs/agent-as-user-comparison/` while temporary lane work stays in scratch. The readable operator report may expose the full private mission so elicitation can be compared against what each harness actually learned, but it keeps that baseline separate from target-visible evidence and declares no automatic winner or prescribed rubric. Because the top-level actor context spans sequential harnesses, this approachable workflow discloses order and does not claim the per-lane actor-process isolation required by rigorous frozen-packet studies; frozen reveal policies, matched budgets, blinding, fresh-per-lane actor sessions, structured adjudication, multi-run statistics, and scripted judges remain separate tools for focused improvement/regression claims. - **FE-1230 execution-comparison oracle boundary.** Execution cases are distinct from private elicitation missions: the human-approved specification and a minimal public runtime/accessibility contract are visible to every lane, while exact browser journeys, reference-model states, expected results, claim mapping, and adversarial fixtures remain controller-only and outside every lane cwd. The public contract requires a static production build at `dist/`, `npm run build`, `npm test`, and stable accessible roles/names for the canvas and named controls; it does not prescribe framework, source topology, implementation decomposition, test library, or internal state model. The versioned `petri-editor-browser-v2` suite tests/builds once, then runs every declared journey from a fresh browser context with public-only setup, per-journey runtime evidence, and non-blocking claim-linked verdicts that distinguish harness/setup failure from product assertion failure. Brunch stops at `promotion_prepared` and never lands. The retained first pair remains immutable; replay/promotion waits for its exact artifact paths. Mutants, masked/process judging, repetitions, and generalized campaign machinery are deferred until one valid end-to-end path exists. The tracer pins `anthropic/claude-opus-4-8` in both products, but same model does not imply equivalent hidden prompting or thinking controls. +- **Brownfield historical-replay oracle boundary (`brownfield-comparison-cases`).** Historical implementations bootstrap independent claims, fixtures, and focused wrong rivals; matching their code, module decomposition, diff shape, or architecture is never a correctness gate and the references never enter masked review. Brunch derives a backend-only host-landing mission from FE-1201/PR #336 at parent `f5a423b19f76cf345d88053456870a126e451618`; because the original PR also changed presentation surfaces, its whole merged diff is explicitly not a golden patch. Its common mechanical address is the disclosed `/brunch:land` command: the controller resumes a settled session fixture under `PI_OFFLINE=1`, drives the built candidate through the real TUI, and lets an independent Git model judge pre-confirm and post-apply temporary-repository snapshots. The compiled v1 case/oracle now closes that route over brownfield, greenfield, refusal, stale-acceptance, full-range, and bookkeeping rivals; fresh-session setup fails closed because it can auto-kick the configured provider. Existing `landing.ts`/`GitHostLandPort` tests may bootstrap fixtures and wrong rivals but cannot enter the comparison oracle. Petrinaut replays only optimization UI PR #9051 in the full `hashintel/hash` checkout at parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978`; optimizer backend/API capabilities already present there are disclosed baseline, not implementation work or elicitation gain. D136-L makes the focused public `/optimization` website route the deterministic fake-provider hard gate; the real HASH `/processes/draft` host/iframe integration is a fully provisioned non-gating outer smoke because the global Next shell requires unrelated GraphQL/codegen and broad platform/toolchain preparation. Public baselines expose only mechanical addressability and pre-existing capabilities required by common oracles. Missions omit historical ids, titles, URLs, and solution wording; target trees are archive/materialization products with no later refs or remotes. The retired A49-L recipe is now executable admission: Claude uses fail-closed native sandbox settings, strict empty MCP, an explicit non-web tool set, and domain denial; Brunch comparison launches disable web/research/Specify-subagent access, use target-bounded foreground and child file tools, and sandbox verifier commands with runtime network forbidden. The selected historical patches add no external package dependency unavailable at their parent trees, so focused dependency preparation can finish controller-side before denial. A real optimizer smoke, the full HASH host/iframe smoke, and masked review remain non-gating outer evidence. - **Shared-session-host convergence oracle (planned; `shared-session-host-tracer` → `shared-session-host-cutover`).** FE-1200's standalone proof is necessary but not sufficient for architectural replacement. The proving oracle must launch the production host as the sole owner of one writable sealed Pi runtime, attach a real TUI presentation and React observer/driver to the same durable target, exercise an ordinary turn plus one extension-owned structured ask and one TUI-only product interaction, detach/restart a client without ending the hosted runtime, and reject a duplicate writable open or conflicting driver. Durable settlement must converge through the same JSONL presentation, and browser traffic must remain Brunch semantic RPC/events rather than raw Pi RPC/events. Once A47-L retires, the cutover becomes a closed coverage sweep: every required TUI/web lifecycle, command/UI, exchange, transcript, graph-update, model/auth, and shutdown row has one host-owned path and a closure oracle; only then may `SessionEventRelay`, `brunch.sessionEvent`, `/rpc/driver`, and their sidecar harnesses be deleted. Human outer evidence must confirm that the TUI remains useful rather than becoming a thin degraded shell. - **Standalone-web compound oracle (2026-07-14; coverage completed 2026-07-15, `standalone-web-session-host`).** The initial five complementary oracles prove the host tracer: (1) inner RPC/host negative-space contracts require `(specId, sessionId)` on every lifecycle/driver/ask/event path and reject targetless fallback, duplicate writable opens, second drivers, and mismatched targets; (2) projection shape/malformed-detail tests keep JSONL and raw Pi/detail shapes behind the named semantic presentation; (3) a production-wired standalone-web browser journey, controlled only by a deterministic faux provider, asserts accessible hydration, streamed text, one `ask`, answer, and `agent_settled` states; (4) paired temporary web/TUI runs use a narrow declared normalizer to prove both live→settled→fresh JSONL hydration and equivalent Brunch binding/runtime/exchange semantics—no static JSONL golden; (5) a workbench manual checklist judges only stream cadence, busy/settled truthfulness, ask interaction, reload, and error feel. Browser cache/overlay loss is part of the middle-loop journey. The follow-on production-host concurrency differential retires A43-L with overlapping graph writes, asks, failure/recovery, target-local event sequences, reconnect, and separate JSONL readback; shared graph changes appear only through canonical `worldUpdate` continuity. The completed coverage pass adds projection no-loss/malformed tests for every required persisted ask terminal shape (including questionnaire read-back), React render/answer tests for free text and listed single/multi choices, headless schema-envelope questionnaire answering coverage (with no dedicated React questionnaire form), distinct candidate/review-set/digest production settlement/reconnect witnesses, concurrency/target isolation, and receipt-bearing review settlement. Live-provider conduct and process-restart survival remain explicitly deferred to their named later work; no second truth plane or raw Pi browser contract is introduced. - **Trace → eval → score → regression flywheel.** The first combined trajectory/evaluation proof is a controlled Brunch A/B over the existing consequential-fact claim, not a generic tracing platform or omnibus architecture score. One realistic non-inferable scenario carries a human-authored hidden-fact ledger, forbidden rivals, and a controlled reveal policy. The only intervention is an eval/dev ablation of the warrant-before-commit directive at the real prompt-composition seam; all other run conditions remain fixed. Three real-provider TUI-driven runs per arm produce a joined legibility envelope: stamped run configuration, directive inventory and content hashes, advertised/read/provider-visible directive state, ordered model/tool/exchange/TUI/graph effects, atomic evaluator judgments with evidence and rubric identity, and a replayable report. Completion requires promoted evidence that discriminates the full directive from the ablated rival; the report is an instrument, not the completion claim. This proves bounded evaluator discrimination, not competitor superiority or broad prompt quality. After the tracer, mine one real walkthrough failure into the same corpus. Keep OTel, broad subagent span joining, provider matrices, interaction-quality scoring, and competitor campaigns trigger-gated until a named claim requires them. @@ -876,6 +886,11 @@ The first required probe is M0: after manual TUI interaction, a checker proves ` | FE-1230 incomparable private process signals | Token accounting, permission prompts, hidden reasoning, and private tool traces differ by product or may be unavailable. | Compare only target-visible/common evidence; retain Brunch-only diagnostics outside judgment packets; mark unavailable cost/permission/tool metrics `not assessable` rather than fabricating parity. | | FE-1230 three-run reliability ceiling | Three Brunch repetitions can reveal catastrophic instability but cannot estimate tail reliability or generalize beyond one browser-app case. | State the ceiling in every report; make no broad speed/cost/reliability claim; add cases or repetitions only after the tracer proves the artifact/oracle contract and a named decision requires stronger evidence. | | FE-1230 same-model non-equivalence | Pinning Opus 4.8 does not equalize product-owned system prompts, tool policy, context management, thinking controls, or provider wrappers. | Frame the result as a same-base-model workflow/product comparison, record exact product/provider/model/configuration versions, and keep process interpretation separate from mechanical outcome correctness. | +| Brownfield historical-solution leakage | A target with later Git refs, remotes, GitHub/Linear/Notion access, or exact historical wording could retrieve the reference implementation and invalidate the replay. Package/dependency preparation may still need controlled network access. | The retired A49-L route archive/materializes base trees without Git history, records source commit/tree identity, sanitizes missions, prepares selected dependencies controller-side, disables MCP/web/research surfaces, bounds filesystem tools, and denies target-command network. Admission adversarially rejects extra refs/remotes, path/symlink escape, weakened policy, reachable loopback/GitHub/Linear/Notion, unavailable isolation, or failing declared local checks; reachable solution sources invalidate the lane rather than reducing confidence. | +| Brunch derived-slice reference mismatch | The selected backend-only FE-1201 mission intentionally excludes TUI/RPC/web changes from the historical PR, so no historical whole-diff equality or completeness claim is valid. | Freeze a fresh requirement registry and independent temp-Git model/property oracle for the derived backend contract; use the historical implementation only to bootstrap fixtures and wrong rivals. | +| Brunch public-command oracle ceiling | `/brunch:land` is architecture-neutral and outcome-observable, but fresh sessions auto-kick a provider, settled-session/TUI setup can fail independently of candidate behavior, and a fixed temporary-Git fixture set cannot cover every repository topology. | Resume a controller-supplied settled session under `PI_OFFLINE=1`; separate `setup_failed` from `assertion_failed`; pair the black-box command journey with an independent full-range Git model and focused final-commit-only/bookkeeping rivals; make no reliability claim and add repository topologies only when a named failure or later campaign requires them. | +| Petrinaut fake-provider, host-integration, and visual ceiling | The focused `/optimization` hard gate proves the UI/provider contract but bypasses HASH's authenticated host/iframe bridge; the real `/processes/draft` shell requires unrelated GraphQL/codegen and broad platform/toolchain preparation. A deterministic fake also cannot prove every live optimizer response, while functional browser/accessibility assertions cannot judge ordinary visual hierarchy or progress feel. | Keep the standalone fake-provider oracle as the common mechanical gate. Run one fully provisioned HASH host/iframe smoke, one optional real-backend smoke, and masked human visual/code review as non-gating outer evidence; mark unavailable integration evidence `not_assessable`. Promote recurring live-shape or bridge failures into focused controller fixtures only when they can remain architecture-neutral; never let qualitative review override a mechanical failure. | +| Brownfield one-run portfolio ceiling | One Brunch and one Petrinaut 2×2 matrix broaden codebase/interface coverage but do not estimate reliability or establish an aggregate winner or causal advantage. | Report each case separately, retain `not_assessable`, and make no cross-case score, reliability, speed, or causal claim. Repetitions require a later named decision and separately frozen campaign. | | Full TUI automation | Cost exceeds value before the product state seams are proven, but startup-switcher regressions need a stronger visual signal than store-only checks. | Manual checklist plus artifact/query probe oracle; for FE-744 startup, add pty/ANSI-stripped capture assertions for the pre-Pi decision surface and absence of stale transcript bef … | | LLM elicitation quality and interaction flow | No stable deterministic ground truth for “good interview” early in the POC, and retired M1 scripted exchanges encoded only a thin obsolete exchange model. | Transcript-backed probe runs, human-reviewed probe reports, adversarial probe scenarios, expected structural coverage, … | | Subscription reconnect/resume | POC can prove initial state payload + live update without hardening network recovery yet. | Contract tests for initial state payload and ordered update sequence; **(2026-06-15)** reconnect/resume promoted to a `web-driver-streaming` battery claim — a turn-cut-point prope … | diff --git a/package.json b/package.json index d414d90df..9f684c07e 100644 --- a/package.json +++ b/package.json @@ -44,8 +44,8 @@ "db:generate": "drizzle-kit generate", "db:studio": "drizzle-kit studio", "test": "vitest --run --maxWorkers=4 --exclude='**/*.slow.test.ts'", - "test:full": "vitest --run --maxWorkers=4", - "test:slow": "vitest --run --maxWorkers=4 .slow.test.ts", + "test:full": "npm run test && npm run test:slow", + "test:slow": "vitest --run --maxWorkers=1 .slow.test.ts", "test:watch": "vitest", "test:context-surfaces": "vitest --run src/agents/contexts src/app/__tests__/print-workspace-state.test.ts src/session/__tests__/transcript-markdown.test.ts", "test:context-surfaces:watch": "vitest src/agents/contexts src/app/__tests__/print-workspace-state.test.ts src/session/__tests__/transcript-markdown.test.ts", diff --git a/src/.pi/extensions/__tests__/registry.test.ts b/src/.pi/extensions/__tests__/registry.test.ts index e8c5c0562..4aa001ce9 100644 --- a/src/.pi/extensions/__tests__/registry.test.ts +++ b/src/.pi/extensions/__tests__/registry.test.ts @@ -84,7 +84,12 @@ import { registerBrunchSessionBoundary as sessionLifecycle } from '../session-ho import { hasBrunchDefaultRenderer } from '../shared/define-brunch-tool.js'; import { BRUNCH_TOOL_ACTIVITY_LABELS } from '../shared/tool-activity-labels.js'; import { assertProviderLegalToolSchema, hasToolParametersProvenance } from '../shared/tool-schema.js'; -import { parseSubagentMarkdown, type BrunchSubagentsDeps, type SubagentResult } from '../subagents/index.js'; +import { + BRUNCH_SUBAGENT_TOOL, + parseSubagentMarkdown, + type BrunchSubagentsDeps, + type SubagentResult, +} from '../subagents/index.js'; import { createSubagentToolCatalog } from '../subagents/session.js'; const extensionDefaults = { @@ -276,6 +281,45 @@ describe('Brunch explicit Pi extension registry', () => { expect(sessionStartIndexes[0]).toBeLessThan(sessionStartIndexes[1] ?? -1); }); + it('registers the comparison bundle without web or Specify subagent surfaces', async () => { + const recording = createRecordingExtensionApi(); + const definitions = new Map([ + ['planner', { name: 'planner' }], + ['worker', { name: 'worker' }], + ]) as BrunchSubagentsDeps['definitions']; + + await createBrunchPiExtensions(brunchChromeFixture, recording.onSessionBoundary, { + coordinator: {} as never, + graphMentionSource: { listMentionCandidates: () => [] }, + allowWebTools: false, + foregroundFilesystemRoot: '/tmp/comparison-target', + subagents: { + definitions, + delegatableAgents: [], + maxConcurrency: 1, + agentDir: '/tmp/agent', + createSettingsManager: () => ({}) as never, + resourceLoaderOptions: {} as never, + }, + })(recording.api); + + expect(recording.toolNames).not.toContain('web_fetch'); + expect(recording.toolNames).not.toContain('web_search'); + expect(recording.toolNames).not.toContain(BRUNCH_SUBAGENT_TOOL); + expect(recording.toolNames).toEqual( + expect.arrayContaining([ + 'read', + 'grep', + 'find', + 'ls', + BRUNCH_EXECUTE_ORCHESTRATE_TOOL, + BRUNCH_EXECUTE_AGENT_RESULT_TOOL, + BRUNCH_EXECUTE_TEST_RESULT_TOOL, + ]), + ); + expect([...definitions.keys()]).toEqual(['planner', 'worker']); + }); + it('registers execute_plan_check only with selected graph deps and returns side-effect-free findings', async () => { const registeredTools: Array<{ name: string; diff --git a/src/.pi/extensions/agent-runtime/runtime/index.ts b/src/.pi/extensions/agent-runtime/runtime/index.ts index de9da83ca..3ee8c7e97 100644 --- a/src/.pi/extensions/agent-runtime/runtime/index.ts +++ b/src/.pi/extensions/agent-runtime/runtime/index.ts @@ -6,7 +6,9 @@ * than owning a second authority list. */ +import { realpath } from 'node:fs/promises'; import { homedir } from 'node:os'; +import { resolve, sep } from 'node:path'; import type { ExtensionAPI } from '@earendil-works/pi-coding-agent'; import { @@ -152,7 +154,10 @@ function supportsOperationalModePolicy(pi: ExtensionAPI): boolean { ); } -export function registerBrunchOperationalModePolicy(pi: ExtensionAPI) { +export function registerBrunchOperationalModePolicy( + pi: ExtensionAPI, + options: { readonly filesystemRoot?: string } = {}, +) { if (!supportsOperationalModePolicy(pi)) { return; } @@ -161,7 +166,9 @@ export function registerBrunchOperationalModePolicy(pi: ExtensionAPI) { ...getReadOnlyTools(process.cwd()).read, label: 'read', async execute(toolCallId, params, signal, onUpdate, ctx) { - return getReadOnlyTools(ctx.cwd).read.execute(toolCallId, params, signal, onUpdate); + const root = options.filesystemRoot ?? ctx.cwd; + await assertBoundedReadPath(root, params.path); + return getReadOnlyTools(root).read.execute(toolCallId, params, signal, onUpdate); }, renderCall(args, theme) { const path = shortenPath(args.path || ''); @@ -189,7 +196,9 @@ export function registerBrunchOperationalModePolicy(pi: ExtensionAPI) { ...getReadOnlyTools(process.cwd()).grep, label: 'grep', async execute(toolCallId, params, signal, onUpdate, ctx) { - return getReadOnlyTools(ctx.cwd).grep.execute(toolCallId, params, signal, onUpdate); + const root = options.filesystemRoot ?? ctx.cwd; + await assertBoundedReadPath(root, params.path ?? '.'); + return getReadOnlyTools(root).grep.execute(toolCallId, params, signal, onUpdate); }, renderCall(args, theme) { const path = shortenPath(args.path || '.'); @@ -214,7 +223,9 @@ export function registerBrunchOperationalModePolicy(pi: ExtensionAPI) { ...getReadOnlyTools(process.cwd()).find, label: 'find', async execute(toolCallId, params, signal, onUpdate, ctx) { - return getReadOnlyTools(ctx.cwd).find.execute(toolCallId, params, signal, onUpdate); + const root = options.filesystemRoot ?? ctx.cwd; + await assertBoundedReadPath(root, params.path ?? '.'); + return getReadOnlyTools(root).find.execute(toolCallId, params, signal, onUpdate); }, renderCall(args, theme) { const path = shortenPath(args.path || '.'); @@ -238,7 +249,9 @@ export function registerBrunchOperationalModePolicy(pi: ExtensionAPI) { ...getReadOnlyTools(process.cwd()).ls, label: 'ls', async execute(toolCallId, params, signal, onUpdate, ctx) { - return getReadOnlyTools(ctx.cwd).ls.execute(toolCallId, params, signal, onUpdate); + const root = options.filesystemRoot ?? ctx.cwd; + await assertBoundedReadPath(root, params.path ?? '.'); + return getReadOnlyTools(root).ls.execute(toolCallId, params, signal, onUpdate); }, renderCall(args, theme) { const path = shortenPath(args.path || '.'); @@ -295,3 +308,15 @@ export function registerBrunchOperationalModePolicy(pi: ExtensionAPI) { }; }); } + +async function assertBoundedReadPath(root: string, requestedPath: string): Promise { + const normalizedRoot = resolve(root); + const target = resolve(normalizedRoot, requestedPath); + if (target !== normalizedRoot && !target.startsWith(`${normalizedRoot}${sep}`)) { + throw new Error(`read-only tool path escapes target root: ${requestedPath}`); + } + const [realRoot, realTarget] = await Promise.all([realpath(normalizedRoot), realpath(target)]); + if (realTarget !== realRoot && !realTarget.startsWith(`${realRoot}${sep}`)) { + throw new Error(`read-only tool path escapes target root through symlink: ${requestedPath}`); + } +} diff --git a/src/.pi/extensions/subagents/session.ts b/src/.pi/extensions/subagents/session.ts index bd158c40a..2b7f69441 100644 --- a/src/.pi/extensions/subagents/session.ts +++ b/src/.pi/extensions/subagents/session.ts @@ -20,7 +20,7 @@ * Its last assistant message is returned to the caller as tool-result content. */ -import { mkdir, writeFile } from 'node:fs/promises'; +import { mkdir, realpath, writeFile } from 'node:fs/promises'; import { dirname, isAbsolute, resolve, sep } from 'node:path'; import { @@ -206,13 +206,23 @@ export function createSubagentToolCatalog( injectedWorld?: SubagentInjectedWorld, ): Map { const pool = new Map(); - for (const definition of [ + for (const rawDefinition of [ createReadToolDefinition(cwd), createGrepToolDefinition(cwd), createFindToolDefinition(cwd), createLsToolDefinition(cwd), ]) { - pool.set(definition.name, definition as ToolDefinition); + const definition = rawDefinition as ToolDefinition; + const execute = definition.execute.bind(definition); + const boundedDefinition = { + ...definition, + execute: async (...args: Parameters) => { + const params = args[1] as { readonly path?: string }; + await assertBoundedExistingPath(cwd, params.path ?? '.'); + return await execute(...args); + }, + } as ToolDefinition; + pool.set(boundedDefinition.name, boundedDefinition); } for (const tool of [createWebSearchTool(), createWebFetchTool()]) { pool.set(tool.name, tool as unknown as ToolDefinition); @@ -239,7 +249,10 @@ function createWriteWorktreeFileTool(cwd: string) { parameters: toolParameters(WriteWorktreeFileParams), async execute(_toolCallId, params) { const target = boundedWorktreePath(cwd, params.path); + await assertBoundedExistingAncestor(cwd, target); await mkdir(dirname(target), { recursive: true }); + await assertBoundedExistingPath(cwd, dirname(target)); + await assertBoundedExistingTargetIfPresent(cwd, target); await writeFile(target, params.content, 'utf8'); return { content: [{ type: 'text' as const, text: `wrote ${params.path}` }], @@ -259,6 +272,43 @@ function boundedWorktreePath(cwd: string, rawPath: string): string { return target; } +async function assertBoundedExistingPath(cwd: string, rawPath: string): Promise { + const root = resolve(cwd); + const target = resolve(root, rawPath); + assertContainedPath(root, target); + const [realRoot, realTarget] = await Promise.all([realpath(root), realpath(target)]); + assertContainedPath(realRoot, realTarget); +} + +async function assertBoundedExistingAncestor(cwd: string, target: string): Promise { + let ancestor = dirname(target); + while (true) { + try { + await assertBoundedExistingPath(cwd, ancestor); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + const parent = dirname(ancestor); + if (parent === ancestor) throw error; + ancestor = parent; + } + } +} + +async function assertBoundedExistingTargetIfPresent(cwd: string, target: string): Promise { + try { + await assertBoundedExistingPath(cwd, target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } +} + +function assertContainedPath(root: string, target: string): void { + if (target !== root && !target.startsWith(`${root}${sep}`)) { + throw new Error('subagent filesystem path escapes the worktree'); + } +} + /** * Translate an agent's declared tool allowlist into SDK session options. * Throws on an unknown tool name (a Brunch authoring bug — fail loud). diff --git a/src/app/brunch-tui.ts b/src/app/brunch-tui.ts index 0236a02d1..8ab1356b8 100644 --- a/src/app/brunch-tui.ts +++ b/src/app/brunch-tui.ts @@ -1,4 +1,4 @@ -import { join } from 'node:path'; +import { join, resolve } from 'node:path'; import process from 'node:process'; import { @@ -50,6 +50,7 @@ import { type SpecSessionActivationCoordinator, type SpecSessionActivationDecision, } from '../session/workspace-session-coordinator.js'; +import type { CommandRunner } from './command-runner.js'; import { openUrlBestEffort } from './open-url.js'; import { chromeStateForWorkspace, @@ -67,6 +68,7 @@ import { resolveBrunchStartupTheme, } from './pi-settings.js'; import { loadBrunchSubagents } from './pi-subagents.js'; +import { createTestRunnerPort } from './test-runner-port.js'; export { BRUNCH_SETTINGS_AUDITED_GETTERS, BRUNCH_SETTINGS_POLICY, @@ -114,6 +116,14 @@ export interface BrunchTuiLaunchContext { introspection?: BrunchTuiIntrospectionOptions; /** Product subagent tool registration for Specify mode; defaults on for normal launches. */ allowSubagents?: boolean; + /** + * Dev/eval-only historical-replay boundary. Presence selects the strict, + * non-configurable comparison composition rather than normal product grants. + */ + comparisonIsolation?: { + readonly targetRoot: string; + readonly verifier: CommandRunner; + }; reportAsyncDiagnostic?: (diagnostic: { readonly type: 'warning'; readonly message: string }) => void; /** * Provider-backend substitution seam (faux provider in Tier-2 oracles). @@ -422,6 +432,10 @@ export function createBrunchAgentSessionRuntimeFactory( ): CreateAgentSessionRuntimeFactory { const { coordinator, productUpdates } = context; return async ({ cwd, agentDir: runtimeAgentDir, sessionManager, sessionStartEvent }) => { + const comparisonIsolation = context.comparisonIsolation; + if (comparisonIsolation && resolve(cwd) !== resolve(comparisonIsolation.targetRoot)) { + throw new Error('Brunch comparison runtime cwd must equal its isolated target root'); + } let currentWorkspace = await coordinator.bindCurrentSpecToReplacementSession(sessionManager); const graph = await openWorkspaceGraphRuntime(cwd); const graphDeps = { @@ -518,8 +532,9 @@ export function createBrunchAgentSessionRuntimeFactory( }), ); const agentState = projectBrunchAgentState(sessionManager.getBranch()); - const allowProductSubagents = context.allowSubagents !== false; - const shouldLoadSubagents = allowProductSubagents || agentState.operationalMode === 'execute'; + const allowProductSubagents = comparisonIsolation === undefined && context.allowSubagents !== false; + const shouldLoadSubagents = + comparisonIsolation !== undefined || allowProductSubagents || agentState.operationalMode === 'execute'; const subagents = shouldLoadSubagents ? await loadBrunchSubagents({ cwd, @@ -528,6 +543,7 @@ export function createBrunchAgentSessionRuntimeFactory( allowProductSubagents && agentState.operationalMode === 'specify' ? ['explorer', 'researcher', 'projector', 'reviewer'] : [], + ...(comparisonIsolation ? { includedAgents: ['planner', 'worker'] } : {}), world: { graph: { specId: currentWorkspace.spec.id, @@ -561,6 +577,15 @@ export function createBrunchAgentSessionRuntimeFactory( ...(context.liveExchange ? { liveExchange: context.liveExchange.opener } : {}), ...(context.introspection ? { introspection: context.introspection } : {}), ...(subagents ? { subagents } : {}), + ...(comparisonIsolation + ? { + allowWebTools: false, + foregroundFilesystemRoot: comparisonIsolation.targetRoot, + executionPorts: { + testRunner: createTestRunnerPort({ run: comparisonIsolation.verifier }), + }, + } + : {}), promptContext: () => { const specId = currentWorkspace.spec.id; const selectedSpec = selectedSpecContext(graph, specId); diff --git a/src/app/pi-extensions.ts b/src/app/pi-extensions.ts index 92963e4ac..c6002e605 100644 --- a/src/app/pi-extensions.ts +++ b/src/app/pi-extensions.ts @@ -250,6 +250,10 @@ export interface BrunchPiExtensionsOptions extends BrunchCommandsOptions { introspection?: BrunchPiIntrospectionOptions; continuityDrains?: () => readonly ContinuityDrain[]; executionPorts?: Partial; + /** Dev/eval-only filesystem boundary for foreground read-only tools. */ + foregroundFilesystemRoot?: string; + /** Defaults on; strict comparison launches set false. */ + allowWebTools?: boolean; /** * Optional subagent registry (D44-L/D92-L). When provided with a non-empty * code-owned delegatable set, the product `subagent` tool is registered and @@ -340,9 +344,13 @@ export function createBrunchPiExtensions( }), registerBrunchBranchPolicyHandlers, registerBrunchCompaction, - registerBrunchOperationalModePolicy, + (api) => + registerBrunchOperationalModePolicy( + api, + options.foregroundFilesystemRoot ? { filesystemRoot: options.foregroundFilesystemRoot } : undefined, + ), registerBrunchContext, - registerBrunchWebTools, + ...(options.allowWebTools === false ? [] : [registerBrunchWebTools]), registerBrunchExecuteStatus, ...(options.productUpdates ? [ diff --git a/src/app/pi-subagents.ts b/src/app/pi-subagents.ts index 68b222c9d..29f600aa3 100644 --- a/src/app/pi-subagents.ts +++ b/src/app/pi-subagents.ts @@ -24,6 +24,7 @@ export interface LoadBrunchSubagentsOptions { readonly cwd: string; readonly agentDir: string; readonly delegatableAgents: readonly string[]; + readonly includedAgents?: readonly string[]; readonly world?: LoadBrunchSubagentsWorld; } @@ -45,10 +46,20 @@ export interface LoadBrunchSubagentsWorld { * non-empty code-owned delegatable set. */ export async function loadBrunchSubagents(options: LoadBrunchSubagentsOptions): Promise { - const [definitions, config] = await Promise.all([ + const [loadedDefinitions, config] = await Promise.all([ loadSubagentDefinitions(subagentAgentsDir()), loadSubagentConfig(subagentConfigPath()), ]); + const definitions = + options.includedAgents === undefined + ? loadedDefinitions + : new Map( + options.includedAgents.flatMap((name) => { + const definition = loadedDefinitions.get(name); + if (!definition) throw new Error(`Brunch subagent definition is unavailable: ${name}`); + return [[name, definition] as const]; + }), + ); return { definitions, diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index 823430c95..8fa71f773 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -11,7 +11,7 @@ This directory owns Brunch-only development loops and curation seams. Nothing he - faux/introspection/tier-2 harnesses used by tests and probes - dev-only witnesses such as `generate-fan-out-witness.ts` - controller-owned execution-comparison packets, lane adapters, immutable evidence contracts, and independently executable black-box oracle journeys (`execution-comparison/`) -- controller-owned end-to-end comparison composition (`end-to-end-comparison.ts` and `end-to-end-comparison/`): content-addressed study and exact-byte handoff contracts, isolated Brunch/Claude adapters, a closed four-cell matrix over unchanged `ExecutionAttempt` leaves, requirement traceability, and audience-safe redaction +- controller-owned end-to-end comparison composition (`end-to-end-comparison.ts` and `end-to-end-comparison/`): content-addressed study and exact-byte handoff contracts, isolated Brunch/Claude adapters, a closed four-cell matrix over unchanged `ExecutionAttempt` leaves, requirement traceability, audience-safe redaction, and the A49-L historical-replay admission boundary - the shell-callable execution-comparison operator wrapper (`execution-comparison-operator.ts`), which lists/resolves frozen cases, prepares only Brunch/Claude targets, invokes the existing oracle after lane termination, and validates immutable attempt records for the project-local `/compare-execution` prompt - the shell-callable comparison provenance writer (`comparison-provenance.ts`), which captures one write-once release/controller snapshot after setup approval and before any comparison lane - the standalone component preview harness (`scripts/dev-components.ts` → `src/dev/component-preview.ts`) for previewing `.pi/components` in isolation on a real terminal, with no workspace/session/DB @@ -19,6 +19,18 @@ This directory owns Brunch-only development loops and curation seams. Nothing he It does not own published CLI behavior, public RPC contracts, database imports from outside `graph/`, or the external `pi-interactive-shell`/`zigpty` runtime. The extension is permanent project development tooling declared under root `.pi` for host-capable manual runs; it never enters Brunch's shipped package manifest, sealed `src/.pi` profile, or runtime dependency graph. `docs/praxis/manual-testing.md` owns the measured priority order, install/health check, trust and auto-install implications, takeover/return, artifact bounds, and teardown procedure. +## Historical Replay Isolation + +`end-to-end-comparison/solution-isolation.ts` owns the versioned fail-closed admission seam for pinned brownfield source trees. It records source commit/tree identity, materializes one declared synthetic target commit without worktree residue, remotes, or later refs, requires exactly one strict Claude/Brunch policy pair, rejects target and symlink escapes, and admits declared local checks only through the branded macOS verifier that denies runtime network plus sealed-root reads/writes. Claude uses native sandbox denial with no ambient MCP/settings/plugins or web tools; Brunch uses target-bounded foreground and planner/worker child file tools. Unsupported hosts stop before provider work. Controller-owned preparation may populate declared dependencies in the target only before isolation admission; its recipe and result metadata stay controller-owned, and case-private packets, historical references, fixtures, and oracles never enter the target. + +## Brownfield Comparison Oracles + +`execution-comparison/host-landing-oracle.ts` is the public Brunch controller entry point; its same-named private subtree owns disposable Git fixtures, settled-session public-TUI actuation, the independent Git outcome model, and report types. The oracle resumes only controller-supplied settled sessions under `PI_OFFLINE=1`, invokes `/brunch:land` through the built candidate, and distinguishes setup invalidity from claim failure. + +Petrinaut preparation selected by D136-L is split across the operator and oracle. `execution-comparison/operator-cli.ts` requires an explicit controller-owned source repository, materializes the frozen full HASH tree and exact handoff per lane, then runs the one compiled immutable install (`corepack yarn install --immutable --mode=skip-build`) before the network-denied lane; install failure or tracked-source mutation removes the target. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against the standalone `/optimization` route. The browser sees only the candidate origin and same-origin proxy, while controller fixtures and the private optimizer origin remain outside target-visible roots. The fully provisioned `/processes/draft` host/iframe journey is non-gating outer evidence and does not alter this mechanical hard gate. + +`execution-comparison-operator.ts` dispatches both brownfield oracles and the retained Petri browser oracle through a closed compile-time registry and includes every implementation module in the immutable oracle-pack hash; manifests cannot select paths, commands, or plugins at runtime. + ## Launcher Surface `npm run dev-cli` is the front door for local workbenches; `npm run dev` directly launches the product CLI from TypeScript source. diff --git a/src/dev/end-to-end-comparison.ts b/src/dev/end-to-end-comparison.ts index c60631972..4e583bffa 100644 --- a/src/dev/end-to-end-comparison.ts +++ b/src/dev/end-to-end-comparison.ts @@ -52,3 +52,18 @@ export { type PreparedClaudeExecutionWorkspace, } from './end-to-end-comparison/claude-adapter.js'; export { retainExecutionCell } from './end-to-end-comparison/execution-cell.js'; +export { + admitHistoricalReplay, + assertTargetBoundedPath, + createBrunchSolutionIsolationPolicy, + createClaudeSolutionIsolationPolicy, + createNetworkDeniedCommandRunner, + materializePinnedSourceTree, + SolutionIsolationAdmissionError, + type BrunchSolutionIsolationPolicy, + type ClaudeSolutionIsolationPolicy, + type IsolationAdmissionReason, + type MaterializedPinnedSourceTree, + type NetworkDeniedCommandRunner, + type SolutionIsolationPolicy, +} from './end-to-end-comparison/solution-isolation.js'; diff --git a/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts b/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts new file mode 100644 index 000000000..385043bd9 --- /dev/null +++ b/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts @@ -0,0 +1,113 @@ +import { execFile } from 'node:child_process'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +import { describe, expect, it } from 'vitest'; + +import { loadPublicCasePacket } from '../../execution-comparison/case-contract.js'; +import { loadEndToEndStudyContract } from '../study-contract.js'; + +const repositoryRoot = fileURLToPath(new URL('../../../../', import.meta.url)); +const petriStudy = join( + repositoryRoot, + 'testing/end-to-end-comparisons/cases/minimal-petri-net-editor/study-contract.json', +); +const brunchStudy = join( + repositoryRoot, + 'testing/end-to-end-comparisons/cases/brunch-host-landing/study-contract.json', +); +const petrinautStudy = join( + repositoryRoot, + 'testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json', +); +const brunchExecution = join(repositoryRoot, 'testing/execution-comparisons/cases/brunch-host-landing'); +const petrinautExecution = join(repositoryRoot, 'testing/execution-comparisons/cases/petrinaut-optimization'); +const execFileAsync = promisify(execFile); + +describe('compiled end-to-end comparison case profiles', () => { + it('loads the unchanged Petri case and exactly two pinned brownfield cases', async () => { + const [petri, brunch, petrinaut, brunchPacket, petrinautPacket] = await Promise.all([ + loadEndToEndStudyContract({ repositoryRoot, contractPath: petriStudy }), + loadEndToEndStudyContract({ repositoryRoot, contractPath: brunchStudy }), + loadEndToEndStudyContract({ repositoryRoot, contractPath: petrinautStudy }), + loadPublicCasePacket(brunchExecution), + loadPublicCasePacket(petrinautExecution), + ]); + + expect(petri.contract).toMatchObject({ + id: 'minimal-petri-net-editor-e2e-v1', + caseId: 'minimal-petri-net-editor-v1', + oracle: { id: 'minimal-petri-net-editor-oracles-v2' }, + }); + expect(brunch.contract).toMatchObject({ + id: 'brunch-host-landing-e2e-v1', + caseId: 'brunch-host-landing-v1', + oracle: { id: 'brunch-host-landing-oracles-v1' }, + source: { + parentCommit: 'f5a423b19f76cf345d88053456870a126e451618', + parentTree: 'a5709715a07faef0b96d3e05a7b6f9f8d693dd38', + }, + }); + expect(petrinaut.contract).toMatchObject({ + id: 'petrinaut-optimization-e2e-v1', + caseId: 'petrinaut-optimization-v1', + oracle: { id: 'petrinaut-optimization-oracles-v1' }, + source: { + parentCommit: '5c7a2d9db5caa851c38938f4b1bac19005b0e978', + parentTree: 'a3e08cf75e00cc9016c931f4665341506e03533e', + }, + }); + expect(brunchPacket.contract.case).toMatchObject({ + product: 'brunch', + mode: 'brownfield', + scope: 'single_feature', + surface: 'backend', + repository: { substrate: 'pinned_git' }, + }); + expect(petrinautPacket.contract.case).toMatchObject({ + product: 'petrinaut', + mode: 'brownfield', + scope: 'single_feature', + surface: 'frontend', + repository: { substrate: 'pinned_git' }, + }); + const sources = [brunch.contract.source, petrinaut.contract.source]; + if (sources.some((source) => source === undefined)) { + throw new Error('brownfield profiles must declare source identity'); + } + for (const source of sources) { + const actualTree = ( + await execFileAsync('git', ['rev-parse', `${source!.parentCommit}^{tree}`], { + cwd: + source!.parentCommit === '5c7a2d9db5caa851c38938f4b1bac19005b0e978' + ? fileURLToPath(new URL('../../../../../hash/', import.meta.url)) + : repositoryRoot, + }) + ).stdout.trim(); + expect(actualTree).toBe(source!.parentTree); + } + }); + + it('keeps historical solution locators and controller expectations out of target-visible artifacts', async () => { + const visiblePaths = [ + 'testing/comparisons/missions/brunch-host-landing.md', + 'testing/end-to-end-comparisons/cases/brunch-host-landing/shared-baseline.md', + 'testing/execution-comparisons/cases/brunch-host-landing/spec.md', + 'testing/execution-comparisons/cases/brunch-host-landing/public-contract.json', + 'testing/comparisons/missions/petrinaut-optimization.md', + 'testing/end-to-end-comparisons/cases/petrinaut-optimization/shared-baseline.md', + 'testing/execution-comparisons/cases/petrinaut-optimization/spec.md', + 'testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json', + ]; + const visible = ( + await Promise.all(visiblePaths.map((path) => readFile(join(repositoryRoot, path), 'utf8'))) + ).join('\n'); + + expect(visible).not.toMatch(/FE-1201|PR #336|pull\/336|0092a549|merged reference|historical solution/iu); + expect(visible).not.toMatch( + /FE-1162|PR #9051|pull\/9051|276e17d7|expected (?:tree|event|request)|final-commit-only|bookkeeping-retaining|historical solution/iu, + ); + }); +}); diff --git a/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts b/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts index 4b0474ecb..2374afa89 100644 --- a/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts @@ -1,6 +1,7 @@ +import { realpathSync } from 'node:fs'; import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; @@ -39,10 +40,12 @@ describe('end-to-end execution adapters', () => { const root = await mkdtemp(join(tmpdir(), 'brunch-e2e-adapter-')); roots.push(root); const selected = await handoff(root); + const controllerRoot = join(root, 'controller'); + const repositoryRoot = fileURLToPath(new URL('../../../../', import.meta.url)); const prepared = await prepareBrunchExecutionCell({ cellRoot: join(root, 'cells', 'brunch-spec--brunch'), workspaceDir: join(root, 'targets', 'brunch'), - controllerRoot: join(root, 'controller'), + controllerRoot, specificationPath: selected.specificationPath, publicContractTemplatePath: contractTemplatePath, }); @@ -58,10 +61,15 @@ describe('end-to-end execution adapters', () => { join(root, 'targets', 'brunch'), '--spec-id', '1', + '--solution-isolation', + 'v1', + '--forbidden-root', + controllerRoot, + '--forbidden-root', + repositoryRoot, ], - cwd: fileURLToPath(new URL('../../../../', import.meta.url)), + cwd: repositoryRoot, }); - expect(JSON.stringify(prepared)).not.toContain(join(root, 'controller')); }); it('prepares and finalizes an isolated Claude repository from the same exact packet', async () => { @@ -69,9 +77,10 @@ describe('end-to-end execution adapters', () => { roots.push(root); const selected = await handoff(root); const workspaceDir = join(root, 'targets', 'claude'); + const controllerRoot = join(root, 'controller'); const prepared = await prepareClaudeExecutionWorkspace({ workspaceDir, - controllerRoot: join(root, 'controller'), + controllerRoot, specificationPath: selected.specificationPath, publicContractTemplatePath: contractTemplatePath, }); @@ -88,11 +97,63 @@ describe('end-to-end execution adapters', () => { '--model', 'claude-opus-4-8', '--permission-mode', - 'bypassPermissions', + 'dontAsk', '--no-session-persistence', + '--strict-mcp-config', + '--mcp-config', + '{"mcpServers":{}}', + '--setting-sources', + '', + '--tools', + 'Bash,Edit,Glob,Grep,Read,Write', + '--allowedTools', + 'Bash', + 'Edit', + 'Glob', + 'Grep', + 'Read', + 'Write', + '--disallowedTools', + 'WebFetch', + 'WebSearch', ]), ); - expect(JSON.stringify(prepared.launch)).not.toContain(join(root, 'controller')); + const settingsIndex = prepared.launch.args.indexOf('--settings'); + expect(settingsIndex).toBeGreaterThan(-1); + expect(JSON.parse(prepared.launch.args[settingsIndex + 1]!)).toEqual({ + enabledPlugins: {}, + permissions: { + allow: ['Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write'], + deny: ['WebFetch', 'WebSearch'], + }, + sandbox: { + enabled: true, + failIfUnavailable: true, + autoAllowBashIfSandboxed: true, + allowUnsandboxedCommands: false, + filesystem: { + denyRead: [ + join(realpathSync(root), 'controller'), + realpathSync(resolve(fileURLToPath(new URL('../../../../', import.meta.url)))), + ], + }, + network: { + allowedDomains: [], + deniedDomains: [ + 'github.com', + '*.github.com', + '*.githubusercontent.com', + 'linear.app', + '*.linear.app', + 'notion.so', + '*.notion.so', + 'notion.site', + '*.notion.site', + ], + }, + }, + }); + expect(prepared.launch.args.at(-1)).not.toContain(controllerRoot); await writeFile(join(workspaceDir, 'package.json'), '{"scripts":{"test":"true","build":"true"}}\n'); const finalized = await finalizeClaudeExecutionWorkspace({ workspaceDir }); diff --git a/src/dev/end-to-end-comparison/__tests__/factorial-browser-oracle.slow.test.ts b/src/dev/end-to-end-comparison/__tests__/factorial-browser-oracle.slow.test.ts index 27693eedd..a9c41dd05 100644 --- a/src/dev/end-to-end-comparison/__tests__/factorial-browser-oracle.slow.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/factorial-browser-oracle.slow.test.ts @@ -33,14 +33,21 @@ afterAll(async () => { describe('synthetic end-to-end factorial oracle composition', () => { it('projects one unchanged known-good oracle result across all four synthetic cells', async () => { - const report = await runPetriEditorBrowserOracle({ appDir: fixtureDir, caseDir }); - const reports = MATRIX_CELL_IDS.map((cellId) => ({ cellId, report })); - - expect(reports.map(({ cellId }) => cellId)).toEqual(MATRIX_CELL_IDS); - for (const { report } of reports) { - expect(report.status).toBe('passed'); - expect(report.checks).toHaveLength(5); - expect(report.checks.every((check) => check.status === 'passed')).toBe(true); + const scratchRoot = await mkdtemp(join(tmpdir(), 'brunch-factorial-oracle-')); + try { + const isolatedFixtureDir = join(scratchRoot, 'known-good'); + await cp(fixtureDir, isolatedFixtureDir, { recursive: true }); + const report = await runPetriEditorBrowserOracle({ appDir: isolatedFixtureDir, caseDir }); + const reports = MATRIX_CELL_IDS.map((cellId) => ({ cellId, report })); + + expect(reports.map(({ cellId }) => cellId)).toEqual(MATRIX_CELL_IDS); + for (const { report } of reports) { + expect(report.status).toBe('passed'); + expect(report.checks).toHaveLength(5); + expect(report.checks.every((check) => check.status === 'passed')).toBe(true); + } + } finally { + await rm(scratchRoot, { recursive: true, force: true }); } }, 240_000); }); diff --git a/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts b/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts new file mode 100644 index 000000000..aa1c5facb --- /dev/null +++ b/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts @@ -0,0 +1,291 @@ +import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'; +import { createServer, type Server } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { runCommand } from '../../../app/command-runner.js'; +import { + admitHistoricalReplay, + createBrunchSolutionIsolationPolicy, + createClaudeSolutionIsolationPolicy, + createNetworkDeniedCommandRunner, + materializePinnedSourceTree, + SolutionIsolationAdmissionError, + type NetworkDeniedCommandRunner, + type SolutionIsolationPolicy, +} from '../solution-isolation.js'; + +const roots: string[] = []; +const servers: Server[] = []; + +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + ), + ); + await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); +}); + +async function git(cwd: string, args: readonly string[]): Promise { + const result = await runCommand('git', args, { cwd }); + if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); + return result.stdout.trim(); +} + +async function sourceRepository(root: string): Promise<{ + readonly repositoryDir: string; + readonly sourceCommit: string; + readonly sourceTree: string; +}> { + const repositoryDir = join(root, 'source'); + await mkdir(repositoryDir); + await git(repositoryDir, ['init']); + await writeFile(join(repositoryDir, 'package.json'), '{"scripts":{"verify":"node verify.mjs"}}\n'); + await writeFile(join(repositoryDir, 'verify.mjs'), "process.stdout.write('local-ok')\n"); + await git(repositoryDir, ['add', '.']); + await git(repositoryDir, [ + '-c', + 'user.name=Isolation Test', + '-c', + 'user.email=isolation@invalid.local', + 'commit', + '-m', + 'Pinned source', + ]); + const sourceCommit = await git(repositoryDir, ['rev-parse', 'HEAD']); + const sourceTree = await git(repositoryDir, ['rev-parse', 'HEAD^{tree}']); + return { repositoryDir, sourceCommit, sourceTree }; +} + +async function localProbeUrl(): Promise { + const server = createServer((_request, response) => { + response.writeHead(200, { 'content-type': 'text/plain' }); + response.end('reachable'); + }); + servers.push(server); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('expected TCP server address'); + return `http://127.0.0.1:${address.port}`; +} + +function strictPolicies( + targetDir: string, + forbiddenRoots: readonly string[], +): readonly SolutionIsolationPolicy[] { + return [ + createClaudeSolutionIsolationPolicy(targetDir, forbiddenRoots), + createBrunchSolutionIsolationPolicy(targetDir), + ]; +} + +describe('brownfield historical-solution isolation', () => { + it('materializes a pinned tree as one history-free target with retained source identity', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-solution-isolation-')); + roots.push(root); + const source = await sourceRepository(root); + const targetDir = join(root, 'target'); + await writeFile(join(source.repositoryDir, 'historical-reference.patch'), 'must not enter target\n'); + + const materialized = await materializePinnedSourceTree({ + sourceRepositoryDir: source.repositoryDir, + sourceCommit: source.sourceCommit, + targetDir, + }); + + expect(materialized).toMatchObject({ + targetDir, + sourceCommit: source.sourceCommit, + sourceTree: source.sourceTree, + recipeVersion: 1, + }); + expect(JSON.parse(await readFile(join(targetDir, '.comparison-source.json'), 'utf8'))).toEqual({ + recipeVersion: 1, + sourceCommit: source.sourceCommit, + sourceTree: source.sourceTree, + }); + expect(await git(targetDir, ['rev-list', '--count', 'HEAD'])).toBe('1'); + expect(await git(targetDir, ['remote'])).toBe(''); + expect((await git(targetDir, ['for-each-ref', '--format=%(refname)'])).split('\n')).toEqual([ + 'refs/heads/main', + ]); + expect(await readdir(targetDir)).not.toContain('historical-reference.patch'); + }); + + it('admits both executor policies only when paths and target commands stay sealed', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-solution-isolation-')); + roots.push(root); + const source = await sourceRepository(root); + const targetDir = join(root, 'target'); + const materialized = await materializePinnedSourceTree({ + sourceRepositoryDir: source.repositoryDir, + sourceCommit: source.sourceCommit, + targetDir, + }); + const probeUrl = await localProbeUrl(); + const forbiddenRoots = [ + source.repositoryDir, + join(root, 'controller'), + join(root, 'harness'), + join(root, 'historical-reference'), + ]; + await Promise.all(forbiddenRoots.slice(1).map(async (path) => await mkdir(path))); + + const admission = await admitHistoricalReplay({ + materialized, + policies: strictPolicies(targetDir, forbiddenRoots), + forbiddenRoots, + networkProbeUrls: [probeUrl, 'https://github.com', 'https://linear.app', 'https://www.notion.so'], + verifier: createNetworkDeniedCommandRunner({ forbiddenReadRoots: forbiddenRoots }), + localChecks: [{ command: process.execPath, args: ['verify.mjs'] }], + }); + + expect(admission).toEqual({ + status: 'admitted', + recipeVersion: 1, + sourceCommit: source.sourceCommit, + sourceTree: source.sourceTree, + executors: ['claude_code', 'brunch'], + localChecks: [{ command: process.execPath, args: ['verify.mjs'], exitCode: 0 }], + }); + }); + + it('retains specific reasons for weakened policy, extra Git reachability, path escape, and network', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-solution-isolation-')); + roots.push(root); + const source = await sourceRepository(root); + const targetDir = join(root, 'target'); + const materialized = await materializePinnedSourceTree({ + sourceRepositoryDir: source.repositoryDir, + sourceCommit: source.sourceCommit, + targetDir, + }); + const probeUrl = await localProbeUrl(); + const forbiddenRoots = [source.repositoryDir, join(root, 'controller')]; + await mkdir(forbiddenRoots[1]!); + const baseInput = { + materialized, + forbiddenRoots, + networkProbeUrls: [probeUrl, 'https://github.com', 'https://linear.app', 'https://www.notion.so'], + verifier: createNetworkDeniedCommandRunner({ forbiddenReadRoots: forbiddenRoots }), + localChecks: [{ command: process.execPath, args: ['verify.mjs'] }], + } as const; + + await expect( + admitHistoricalReplay({ + ...baseInput, + policies: [ + { + ...createClaudeSolutionIsolationPolicy(targetDir, forbiddenRoots), + strictMcp: false, + } as unknown as SolutionIsolationPolicy, + createBrunchSolutionIsolationPolicy(targetDir), + ], + }), + ).rejects.toMatchObject({ + reasons: [expect.objectContaining({ code: 'policy_weakened', executor: 'claude_code' })], + }); + + await expect( + admitHistoricalReplay({ + ...baseInput, + policies: [createBrunchSolutionIsolationPolicy(targetDir)], + }), + ).rejects.toMatchObject({ + reasons: [expect.objectContaining({ code: 'policy_weakened' })], + }); + + await writeFile(join(targetDir, 'historical-reference.patch'), 'must not be target-visible\n'); + await expect( + admitHistoricalReplay({ + ...baseInput, + policies: strictPolicies(targetDir, forbiddenRoots), + }), + ).rejects.toMatchObject({ + reasons: [expect.objectContaining({ code: 'git_worktree_changes_present' })], + }); + await rm(join(targetDir, 'historical-reference.patch')); + + await git(targetDir, ['remote', 'add', 'origin', 'https://github.com/hashintel/brunch.git']); + await git(targetDir, ['branch', 'later-solution']); + await expect( + admitHistoricalReplay({ + ...baseInput, + policies: strictPolicies(targetDir, forbiddenRoots), + }), + ).rejects.toMatchObject({ + reasons: expect.arrayContaining([ + expect.objectContaining({ code: 'git_remote_present' }), + expect.objectContaining({ code: 'git_ref_present' }), + ]), + }); + await git(targetDir, ['remote', 'remove', 'origin']); + await git(targetDir, ['branch', '-D', 'later-solution']); + + await expect( + admitHistoricalReplay({ + ...baseInput, + forbiddenRoots: [join(targetDir, 'nested-controller')], + policies: strictPolicies(targetDir, [join(targetDir, 'nested-controller')]), + }), + ).rejects.toMatchObject({ + reasons: expect.arrayContaining([expect.objectContaining({ code: 'path_boundary_weakened' })]), + }); + + const controllerDir = join(root, 'controller'); + await writeFile(join(controllerDir, 'reference.txt'), 'historical solution\n'); + await symlink(join(controllerDir, 'reference.txt'), join(targetDir, 'escaped-reference.txt')); + await expect( + admitHistoricalReplay({ + ...baseInput, + policies: strictPolicies(targetDir, forbiddenRoots), + }), + ).rejects.toMatchObject({ + reasons: expect.arrayContaining([ + expect.objectContaining({ + code: 'path_boundary_weakened', + detail: 'target symlink escapes isolation root: escaped-reference.txt', + }), + ]), + }); + await rm(join(targetDir, 'escaped-reference.txt')); + + const networkCapableVerifier = { + recipeVersion: 1 as const, + platform: 'darwin' as const, + forbiddenReadRoots: forbiddenRoots, + run: runCommand, + } as unknown as NetworkDeniedCommandRunner; + await expect( + admitHistoricalReplay({ + ...baseInput, + verifier: networkCapableVerifier, + policies: strictPolicies(targetDir, forbiddenRoots), + }), + ).rejects.toBeInstanceOf(SolutionIsolationAdmissionError); + await expect( + admitHistoricalReplay({ + ...baseInput, + verifier: networkCapableVerifier, + policies: strictPolicies(targetDir, forbiddenRoots), + }), + ).rejects.toMatchObject({ + reasons: [expect.objectContaining({ code: 'policy_weakened' })], + }); + }); + + it('fails closed when the host cannot provide the versioned isolation recipe', () => { + expect(() => createNetworkDeniedCommandRunner({ platform: 'linux' })).toThrow( + 'solution isolation recipe v1 is unsupported on linux', + ); + }); +}); diff --git a/src/dev/end-to-end-comparison/brunch-adapter.ts b/src/dev/end-to-end-comparison/brunch-adapter.ts index eae129578..b84c88a83 100644 --- a/src/dev/end-to-end-comparison/brunch-adapter.ts +++ b/src/dev/end-to-end-comparison/brunch-adapter.ts @@ -42,6 +42,9 @@ export async function prepareBrunchExecutionCell(input: { caseDir: materialized.packetDir, specificationMode: 'opaque', }); + const forbiddenRoots = [input.controllerRoot, repositoryRoot()].filter( + (root) => !containedPath(root, input.workspaceDir) && !containedPath(input.workspaceDir, root), + ); return { prepared, launch: { @@ -53,6 +56,9 @@ export async function prepareBrunchExecutionCell(input: { input.workspaceDir, '--spec-id', String(prepared.specId), + '--solution-isolation', + 'v1', + ...forbiddenRoots.flatMap((root) => ['--forbidden-root', root]), ], cwd: repositoryRoot(), }, diff --git a/src/dev/end-to-end-comparison/claude-adapter.ts b/src/dev/end-to-end-comparison/claude-adapter.ts index 4b091064e..086f4fe6d 100644 --- a/src/dev/end-to-end-comparison/claude-adapter.ts +++ b/src/dev/end-to-end-comparison/claude-adapter.ts @@ -1,9 +1,11 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { runCommand, type CommandResult, type CommandRunner } from '../../app/command-runner.js'; import type { ExecutionLaunch } from './brunch-adapter.js'; import { materializeExactExecutionPacket } from './public-packet.js'; +import { createClaudeSolutionIsolationPolicy } from './solution-isolation.js'; import { assertControllerIsolation } from './study-contract.js'; import { containedPath } from './validation.js'; @@ -68,6 +70,12 @@ export async function prepareClaudeExecutionWorkspace( 'Freeze comparison input', ]); const baseSha = (await gitChecked(runner, input.workspaceDir, ['rev-parse', 'HEAD'])).stdout.trim(); + const isolationPolicy = createClaudeSolutionIsolationPolicy( + input.workspaceDir, + [input.controllerRoot, repositoryRoot()].filter( + (root) => !containedPath(root, input.workspaceDir) && !containedPath(input.workspaceDir, root), + ), + ); return { workspaceDir: input.workspaceDir, baseSha, @@ -83,10 +91,43 @@ export async function prepareClaudeExecutionWorkspace( '--effort', 'max', '--permission-mode', - 'bypassPermissions', + isolationPolicy.permissionMode, '--no-session-persistence', '--disable-slash-commands', '--no-chrome', + '--strict-mcp-config', + '--mcp-config', + '{"mcpServers":{}}', + '--setting-sources', + '', + '--tools', + isolationPolicy.allowedTools.join(','), + '--allowedTools', + ...isolationPolicy.allowedTools, + '--disallowedTools', + 'WebFetch', + 'WebSearch', + '--settings', + JSON.stringify({ + enabledPlugins: {}, + permissions: { + allow: isolationPolicy.allowedTools, + deny: ['WebFetch', 'WebSearch'], + }, + sandbox: { + enabled: isolationPolicy.nativeSandbox.enabled, + failIfUnavailable: isolationPolicy.nativeSandbox.failIfUnavailable, + autoAllowBashIfSandboxed: true, + allowUnsandboxedCommands: false, + filesystem: { + denyRead: isolationPolicy.nativeSandbox.deniedReadRoots, + }, + network: { + allowedDomains: isolationPolicy.nativeSandbox.allowedDomains, + deniedDomains: isolationPolicy.nativeSandbox.deniedDomains, + }, + }, + }), implementationPrompt(), ], cwd: input.workspaceDir, @@ -94,6 +135,10 @@ export async function prepareClaudeExecutionWorkspace( }; } +function repositoryRoot(): string { + return fileURLToPath(new URL('../../../', import.meta.url)); +} + export async function runClaudeExecutionWorkspace( input: { readonly prepared: PreparedClaudeExecutionWorkspace; diff --git a/src/dev/end-to-end-comparison/pinned-source-preparation.ts b/src/dev/end-to-end-comparison/pinned-source-preparation.ts new file mode 100644 index 000000000..f49791e00 --- /dev/null +++ b/src/dev/end-to-end-comparison/pinned-source-preparation.ts @@ -0,0 +1,156 @@ +import { copyFile, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { runCommand, type CommandRunner } from '../../app/command-runner.js'; +import { materializeExactExecutionPacket } from './public-packet.js'; +import { materializePinnedSourceTree } from './solution-isolation.js'; +import { assertControllerIsolation } from './study-contract.js'; + +const COMPARISON_GIT_IDENTITY = [ + '-c', + 'user.name=Brunch Comparison', + '-c', + 'user.email=brunch-comparison@invalid.local', +] as const; + +export const PINNED_DEPENDENCY_PREPARATION = { + command: 'corepack', + args: ['yarn', 'install', '--immutable', '--mode=skip-build'], +} as const; + +export interface PreparedPinnedExecutionWorkspace { + readonly lane: 'brunch' | 'claude_code'; + readonly targetDir: string; + readonly sourceCommit: string; + readonly sourceTree: string; + readonly materializedCommit: string; + readonly baseSha: string; + readonly dependencyPreparation: { + readonly command: typeof PINNED_DEPENDENCY_PREPARATION.command; + readonly args: typeof PINNED_DEPENDENCY_PREPARATION.args; + readonly status: 'passed'; + readonly exitCode: 0; + }; +} + +export async function preparePinnedExecutionWorkspace( + input: { + readonly lane: PreparedPinnedExecutionWorkspace['lane']; + readonly sourceRepositoryDir: string; + readonly sourceCommit: string; + readonly expectedSourceTree: string; + readonly targetDir: string; + readonly controllerRoot: string; + readonly specificationPath: string; + readonly publicContractTemplatePath: string; + readonly dependencyInstallRunner?: CommandRunner; + }, + runner: CommandRunner = runCommand, +): Promise { + assertControllerIsolation({ + controllerRoot: input.controllerRoot, + targetRoots: [input.targetDir], + }); + assertControllerIsolation({ + controllerRoot: input.sourceRepositoryDir, + targetRoots: [input.targetDir], + }); + const materialized = await materializePinnedSourceTree({ + sourceRepositoryDir: input.sourceRepositoryDir, + sourceCommit: input.sourceCommit, + targetDir: input.targetDir, + runner, + }); + if (materialized.sourceTree !== input.expectedSourceTree) { + await rm(input.targetDir, { recursive: true, force: true }); + throw new Error('pinned source tree does not match the frozen execution contract'); + } + + const packetRoot = await mkdtemp(join(tmpdir(), 'brunch-pinned-execution-packet-')); + try { + const packet = await materializeExactExecutionPacket({ + specificationPath: input.specificationPath, + publicContractTemplatePath: input.publicContractTemplatePath, + packetDir: join(packetRoot, 'packet'), + }); + for (const file of packet.packet.files) { + await copyFile(join(packet.packetDir, file.path), join(input.targetDir, file.path)); + } + await gitChecked(runner, input.targetDir, ['add', '--', 'public-contract.json', 'spec.md']); + await gitChecked(runner, input.targetDir, [ + ...COMPARISON_GIT_IDENTITY, + 'commit', + '-m', + 'Freeze exact comparison handoff', + ]); + const baseSha = (await gitChecked(runner, input.targetDir, ['rev-parse', 'HEAD'])).stdout.trim(); + const dependencyInstallRunner = input.dependencyInstallRunner ?? runCommand; + const installed = await dependencyInstallRunner( + PINNED_DEPENDENCY_PREPARATION.command, + PINNED_DEPENDENCY_PREPARATION.args, + { + cwd: input.targetDir, + timeoutMs: 30 * 60_000, + maxOutputBytes: 256 * 1024, + }, + ); + if (installed.exitCode !== 0) { + throw new Error( + `${PINNED_DEPENDENCY_PREPARATION.command} ${PINNED_DEPENDENCY_PREPARATION.args.join( + ' ', + )} controller dependency preparation failed (${installed.exitCode}): ${ + installed.stderr || installed.stdout + }`, + ); + } + const trackedStatus = ( + await gitChecked(runner, input.targetDir, ['status', '--porcelain', '--untracked-files=no']) + ).stdout.trim(); + if (trackedStatus.length > 0) { + throw new Error(`controller dependency preparation modified tracked source: ${trackedStatus}`); + } + const sourceIdentity = JSON.parse( + await readFile(join(input.targetDir, '.comparison-source.json'), 'utf8'), + ) as { + sourceCommit?: string; + sourceTree?: string; + }; + if ( + sourceIdentity.sourceCommit !== materialized.sourceCommit || + sourceIdentity.sourceTree !== materialized.sourceTree + ) { + throw new Error('materialized source identity drifted while freezing the execution handoff'); + } + return { + lane: input.lane, + targetDir: input.targetDir, + sourceCommit: materialized.sourceCommit, + sourceTree: materialized.sourceTree, + materializedCommit: materialized.syntheticCommit, + baseSha, + dependencyPreparation: { + ...PINNED_DEPENDENCY_PREPARATION, + status: 'passed', + exitCode: 0, + }, + }; + } catch (error) { + await rm(input.targetDir, { recursive: true, force: true }); + throw error; + } finally { + await rm(packetRoot, { recursive: true, force: true }); + } +} + +async function gitChecked( + runner: CommandRunner, + cwd: string, + args: readonly string[], +): ReturnType { + const result = await runner('git', args, { cwd }); + if (result.exitCode !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${result.stderr || result.stdout}`); + } + return result; +} diff --git a/src/dev/end-to-end-comparison/public-packet.ts b/src/dev/end-to-end-comparison/public-packet.ts index e68ae141e..d59f1e738 100644 --- a/src/dev/end-to-end-comparison/public-packet.ts +++ b/src/dev/end-to-end-comparison/public-packet.ts @@ -17,16 +17,20 @@ export async function materializeExactExecutionPacket(input: { readonly packet: PublicCasePacket; }> { const specification = await readFile(input.specificationPath); - const template = parsePublicCaseContract( - parseJson(await readFile(input.publicContractTemplatePath, 'utf8')), - ); + const templateBytes = await readFile(input.publicContractTemplatePath); + const template = parsePublicCaseContract(parseJson(templateBytes.toString('utf8'))); + const specificationSha256 = createHash('sha256').update(specification).digest('hex'); const contract = parsePublicCaseContract({ ...template, case: { ...template.case, - specificationSha256: createHash('sha256').update(specification).digest('hex'), + specificationSha256, }, }); + const contractBytes = + template.case.specificationSha256 === specificationSha256 + ? templateBytes + : Buffer.from(`${JSON.stringify(contract, null, 2)}\n`); await mkdir(dirname(input.packetDir), { recursive: true }); try { @@ -41,10 +45,7 @@ export async function materializeExactExecutionPacket(input: { await writeFile(join(input.packetDir, contract.case.specification), specification, { flag: 'wx', }); - await writeFile(join(input.packetDir, 'public-contract.json'), `${JSON.stringify(contract, null, 2)}\n`, { - encoding: 'utf8', - flag: 'wx', - }); + await writeFile(join(input.packetDir, 'public-contract.json'), contractBytes, { flag: 'wx' }); const packet = await loadPublicCasePacket(input.packetDir); return { packetDir: input.packetDir, packet }; } catch (error) { diff --git a/src/dev/end-to-end-comparison/solution-isolation.ts b/src/dev/end-to-end-comparison/solution-isolation.ts new file mode 100644 index 000000000..80f8e0386 --- /dev/null +++ b/src/dev/end-to-end-comparison/solution-isolation.ts @@ -0,0 +1,585 @@ +import { realpathSync } from 'node:fs'; +import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; + +import { runCommand, type CommandResult, type CommandRunner } from '../../app/command-runner.js'; + +const RECIPE_VERSION = 1 as const; +const SOURCE_IDENTITY_FILE = '.comparison-source.json'; +const COMPARISON_GIT_IDENTITY = [ + '-c', + 'user.name=Brunch Comparison', + '-c', + 'user.email=brunch-comparison@invalid.local', +] as const; +const NETWORK_DENIED_RUNNER = Symbol('network-denied-runner'); +const REQUIRED_SOLUTION_PROBE_URLS = [ + 'https://github.com', + 'https://linear.app', + 'https://www.notion.so', +] as const; +const DENIED_SOLUTION_SOURCE_DOMAINS = [ + 'github.com', + '*.github.com', + '*.githubusercontent.com', + 'linear.app', + '*.linear.app', + 'notion.so', + '*.notion.so', + 'notion.site', + '*.notion.site', +] as const; + +export interface MaterializedPinnedSourceTree { + readonly recipeVersion: typeof RECIPE_VERSION; + readonly targetDir: string; + readonly sourceCommit: string; + readonly sourceTree: string; + readonly syntheticCommit: string; +} + +export interface ClaudeSolutionIsolationPolicy { + readonly executor: 'claude_code'; + readonly recipeVersion: typeof RECIPE_VERSION; + readonly targetRoot: string; + readonly strictMcp: true; + readonly mcpServers: readonly []; + readonly webTools: false; + readonly ambientSettings: false; + readonly ambientPlugins: false; + readonly permissionMode: 'dontAsk'; + readonly nativeSandbox: { + readonly enabled: true; + readonly failIfUnavailable: true; + readonly allowedDomains: readonly []; + readonly deniedDomains: typeof DENIED_SOLUTION_SOURCE_DOMAINS; + readonly deniedReadRoots: readonly string[]; + }; + readonly allowedTools: readonly ['Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write']; +} + +export interface BrunchSolutionIsolationPolicy { + readonly executor: 'brunch'; + readonly recipeVersion: typeof RECIPE_VERSION; + readonly targetRoot: string; + readonly foregroundWebTools: false; + readonly specifySubagents: false; + readonly foregroundFileRoot: string; + readonly executionSubagents: readonly ['planner', 'worker']; + readonly networkDeniedVerifier: true; +} + +export type SolutionIsolationPolicy = ClaudeSolutionIsolationPolicy | BrunchSolutionIsolationPolicy; + +export interface NetworkDeniedCommandRunner { + readonly recipeVersion: typeof RECIPE_VERSION; + readonly platform: 'darwin'; + readonly forbiddenReadRoots: readonly string[]; + readonly [NETWORK_DENIED_RUNNER]: true; + readonly run: CommandRunner; +} + +export interface IsolationAdmissionReason { + readonly code: + | 'identity_mismatch' + | 'git_history_present' + | 'git_ref_present' + | 'git_remote_present' + | 'git_worktree_changes_present' + | 'local_check_failed' + | 'network_probe_reachable' + | 'path_boundary_weakened' + | 'policy_weakened'; + readonly detail: string; + readonly executor?: SolutionIsolationPolicy['executor']; +} + +export class SolutionIsolationAdmissionError extends Error { + readonly reasons: readonly IsolationAdmissionReason[]; + + constructor(reasons: readonly IsolationAdmissionReason[]) { + super(`historical replay isolation admission failed: ${reasons.map(({ code }) => code).join(', ')}`); + this.name = 'SolutionIsolationAdmissionError'; + this.reasons = reasons; + } +} + +export async function materializePinnedSourceTree(input: { + readonly sourceRepositoryDir: string; + readonly sourceCommit: string; + readonly targetDir: string; + readonly runner?: CommandRunner; +}): Promise { + const runner = input.runner ?? runCommand; + const sourceCommit = ( + await gitChecked(runner, input.sourceRepositoryDir, [ + 'rev-parse', + '--verify', + `${input.sourceCommit}^{commit}`, + ]) + ).stdout.trim(); + const sourceTree = ( + await gitChecked(runner, input.sourceRepositoryDir, ['rev-parse', '--verify', `${sourceCommit}^{tree}`]) + ).stdout.trim(); + const archiveDir = await mkdtemp(join(tmpdir(), 'brunch-pinned-source-')); + const archivePath = join(archiveDir, 'source.tar'); + try { + await mkdir(dirname(input.targetDir), { recursive: true }); + await mkdir(input.targetDir); + await commandChecked(runner, input.sourceRepositoryDir, 'git', [ + 'archive', + '--format=tar', + `--output=${archivePath}`, + sourceCommit, + ]); + await commandChecked(runner, input.targetDir, 'tar', ['-xf', archivePath]); + await writeFile( + join(input.targetDir, SOURCE_IDENTITY_FILE), + `${JSON.stringify({ recipeVersion: RECIPE_VERSION, sourceCommit, sourceTree }, null, 2)}\n`, + { encoding: 'utf8', flag: 'wx' }, + ); + await gitChecked(runner, input.targetDir, ['init', '--initial-branch=main']); + await gitChecked(runner, input.targetDir, ['add', '--all']); + await gitChecked(runner, input.targetDir, [ + ...COMPARISON_GIT_IDENTITY, + 'commit', + '-m', + 'Materialize pinned comparison source', + ]); + const syntheticCommit = (await gitChecked(runner, input.targetDir, ['rev-parse', 'HEAD'])).stdout.trim(); + return { + recipeVersion: RECIPE_VERSION, + targetDir: input.targetDir, + sourceCommit, + sourceTree, + syntheticCommit, + }; + } catch (error) { + await rm(input.targetDir, { recursive: true, force: true }); + throw error; + } finally { + await rm(archiveDir, { recursive: true, force: true }); + } +} + +export function createClaudeSolutionIsolationPolicy( + targetRoot: string, + forbiddenRoots: readonly string[] = [], +): ClaudeSolutionIsolationPolicy { + return { + executor: 'claude_code', + recipeVersion: RECIPE_VERSION, + targetRoot: resolve(targetRoot), + strictMcp: true, + mcpServers: [], + webTools: false, + ambientSettings: false, + ambientPlugins: false, + permissionMode: 'dontAsk', + nativeSandbox: { + enabled: true, + failIfUnavailable: true, + allowedDomains: [], + deniedDomains: DENIED_SOLUTION_SOURCE_DOMAINS, + deniedReadRoots: [...new Set(forbiddenRoots.map(canonicalPolicyPath))], + }, + allowedTools: ['Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write'], + }; +} + +export function createBrunchSolutionIsolationPolicy(targetRoot: string): BrunchSolutionIsolationPolicy { + const root = resolve(targetRoot); + return { + executor: 'brunch', + recipeVersion: RECIPE_VERSION, + targetRoot: root, + foregroundWebTools: false, + specifySubagents: false, + foregroundFileRoot: root, + executionSubagents: ['planner', 'worker'], + networkDeniedVerifier: true, + }; +} + +export function createNetworkDeniedCommandRunner( + options: { + readonly platform?: NodeJS.Platform; + readonly run?: CommandRunner; + readonly forbiddenReadRoots?: readonly string[]; + } = {}, +): NetworkDeniedCommandRunner { + // ceiling: recipe v1 supports macOS sandbox-exec only; add an explicit equivalent host recipe before admitting another platform. + const platform = options.platform ?? process.platform; + if (platform !== 'darwin') { + throw new Error(`solution isolation recipe v1 is unsupported on ${platform}`); + } + const run = options.run ?? runCommand; + const forbiddenReadRoots = [...new Set((options.forbiddenReadRoots ?? []).map(canonicalPolicyPath))]; + const sandboxProfile = denyTargetCommandAccessProfile(forbiddenReadRoots); + return { + recipeVersion: RECIPE_VERSION, + platform, + forbiddenReadRoots, + [NETWORK_DENIED_RUNNER]: true, + run: async (command, args, commandOptions) => + await run('sandbox-exec', ['-p', sandboxProfile, command, ...args], commandOptions), + }; +} + +export function assertTargetBoundedPath(policy: SolutionIsolationPolicy, requestedPath: string): string { + const targetRoot = resolve(policy.targetRoot); + const candidate = resolve(targetRoot, requestedPath); + if (!containedPath(targetRoot, candidate)) { + throw new Error(`${policy.executor} path escapes target root: ${requestedPath}`); + } + return candidate; +} + +export async function admitHistoricalReplay(input: { + readonly materialized: MaterializedPinnedSourceTree; + readonly policies: readonly SolutionIsolationPolicy[]; + readonly forbiddenRoots: readonly string[]; + readonly networkProbeUrls: readonly string[]; + readonly verifier: NetworkDeniedCommandRunner; + readonly localChecks: readonly { readonly command: string; readonly args: readonly string[] }[]; + readonly runner?: CommandRunner; +}): Promise<{ + readonly status: 'admitted'; + readonly recipeVersion: typeof RECIPE_VERSION; + readonly sourceCommit: string; + readonly sourceTree: string; + readonly executors: readonly SolutionIsolationPolicy['executor'][]; + readonly localChecks: readonly { + readonly command: string; + readonly args: readonly string[]; + readonly exitCode: number; + }[]; +}> { + const runner = input.runner ?? runCommand; + const reasons: IsolationAdmissionReason[] = []; + inspectVerifier(input.verifier, input.forbiddenRoots, reasons); + inspectPolicies(input.policies, input.materialized.targetDir, input.forbiddenRoots, reasons); + inspectForbiddenRoots(input.policies, input.forbiddenRoots, reasons); + await inspectMaterializedRepository(input.materialized, runner, reasons); + await inspectTargetSymlinks(input.materialized.targetDir, reasons); + if (reasons.length > 0) throw new SolutionIsolationAdmissionError(reasons); + + for (const forbiddenRoot of input.forbiddenRoots) { + const result = await input.verifier.run('/bin/ls', ['-la', forbiddenRoot], { + cwd: input.materialized.targetDir, + timeoutMs: 15_000, + maxOutputBytes: 16 * 1024, + }); + if (result.exitCode === 0) { + throw new SolutionIsolationAdmissionError([ + { + code: 'path_boundary_weakened', + detail: `target verifier can read forbidden root: ${forbiddenRoot}`, + }, + ]); + } + } + + const curlReady = await input.verifier.run('/usr/bin/curl', ['--version'], { + cwd: input.materialized.targetDir, + timeoutMs: 15_000, + maxOutputBytes: 16 * 1024, + }); + if (curlReady.exitCode !== 0) { + throw new SolutionIsolationAdmissionError([ + { code: 'policy_weakened', detail: 'network-denied verifier cannot execute /usr/bin/curl' }, + ]); + } + + for (const url of new Set([...input.networkProbeUrls, ...REQUIRED_SOLUTION_PROBE_URLS])) { + const result = await input.verifier.run('/usr/bin/curl', ['--fail', '--silent', '--show-error', url], { + cwd: input.materialized.targetDir, + timeoutMs: 15_000, + maxOutputBytes: 16 * 1024, + }); + if (result.exitCode === 0) { + throw new SolutionIsolationAdmissionError([{ code: 'network_probe_reachable', detail: url }]); + } + } + + const localChecks = []; + for (const check of input.localChecks) { + const result = await input.verifier.run(check.command, check.args, { + cwd: input.materialized.targetDir, + timeoutMs: 10 * 60_000, + maxOutputBytes: 128 * 1024, + }); + if (result.exitCode !== 0) { + throw new SolutionIsolationAdmissionError([ + { + code: 'local_check_failed', + detail: `${[check.command, ...check.args].join(' ')} exited ${result.exitCode}`, + }, + ]); + } + localChecks.push({ ...check, exitCode: result.exitCode }); + } + + return { + status: 'admitted', + recipeVersion: RECIPE_VERSION, + sourceCommit: input.materialized.sourceCommit, + sourceTree: input.materialized.sourceTree, + executors: input.policies.map(({ executor }) => executor), + localChecks, + }; +} + +function inspectVerifier( + verifier: NetworkDeniedCommandRunner, + forbiddenRoots: readonly string[], + reasons: IsolationAdmissionReason[], +): void { + if ( + forbiddenRoots.length === 0 || + verifier.recipeVersion !== RECIPE_VERSION || + verifier.platform !== 'darwin' || + verifier[NETWORK_DENIED_RUNNER] !== true || + !forbiddenRoots.every((forbiddenRoot) => + verifier.forbiddenReadRoots.some((deniedRoot) => + containedPath(deniedRoot, canonicalPolicyPath(forbiddenRoot)), + ), + ) + ) { + reasons.push({ + code: 'policy_weakened', + detail: 'target verifier does not match isolation recipe v1', + }); + } +} + +function inspectPolicies( + policies: readonly SolutionIsolationPolicy[], + targetDir: string, + forbiddenRoots: readonly string[], + reasons: IsolationAdmissionReason[], +): void { + if (!sameStrings(policies.map(({ executor }) => executor).sort(), ['brunch', 'claude_code'])) { + reasons.push({ + code: 'policy_weakened', + detail: 'isolation recipe v1 requires exactly one Brunch and one Claude Code policy', + }); + } + for (const policy of policies) { + const deniesForbiddenRoots = + policy.executor !== 'claude_code' || + forbiddenRoots.every((forbiddenRoot) => + policy.nativeSandbox.deniedReadRoots.some((deniedRoot) => + containedPath(deniedRoot, canonicalPolicyPath(forbiddenRoot)), + ), + ); + if ( + resolve(policy.targetRoot) !== resolve(targetDir) || + !isStrictPolicy(policy) || + !deniesForbiddenRoots + ) { + reasons.push({ + code: 'policy_weakened', + detail: `${policy.executor} policy does not match isolation recipe v1`, + executor: policy.executor, + }); + } + } +} + +function isStrictPolicy(policy: SolutionIsolationPolicy): boolean { + if (policy.recipeVersion !== RECIPE_VERSION) return false; + if (policy.executor === 'claude_code') { + return ( + policy.strictMcp === true && + policy.mcpServers.length === 0 && + policy.webTools === false && + policy.ambientSettings === false && + policy.ambientPlugins === false && + policy.permissionMode === 'dontAsk' && + policy.nativeSandbox.enabled === true && + policy.nativeSandbox.failIfUnavailable === true && + policy.nativeSandbox.allowedDomains.length === 0 && + sameStrings(policy.nativeSandbox.deniedDomains, DENIED_SOLUTION_SOURCE_DOMAINS) && + policy.nativeSandbox.deniedReadRoots.every( + (deniedRoot) => !containedPath(deniedRoot, canonicalPolicyPath(policy.targetRoot)), + ) && + sameStrings(policy.allowedTools, ['Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write']) + ); + } + return ( + policy.foregroundWebTools === false && + policy.specifySubagents === false && + resolve(policy.foregroundFileRoot) === resolve(policy.targetRoot) && + sameStrings(policy.executionSubagents, ['planner', 'worker']) && + policy.networkDeniedVerifier === true + ); +} + +function inspectForbiddenRoots( + policies: readonly SolutionIsolationPolicy[], + forbiddenRoots: readonly string[], + reasons: IsolationAdmissionReason[], +): void { + for (const forbiddenRoot of forbiddenRoots) { + if ( + policies.some( + (policy) => + containedPath(policy.targetRoot, forbiddenRoot) || containedPath(forbiddenRoot, policy.targetRoot), + ) + ) { + reasons.push({ + code: 'path_boundary_weakened', + detail: `forbidden root is inside target boundary: ${forbiddenRoot}`, + }); + return; + } + for (const policy of policies) { + try { + assertTargetBoundedPath(policy, forbiddenRoot); + reasons.push({ + code: 'path_boundary_weakened', + detail: `${policy.executor} can reach forbidden root: ${forbiddenRoot}`, + executor: policy.executor, + }); + return; + } catch { + // Expected: every controller, harness, parent, and reference root is outside the target. + } + } + } +} + +async function inspectMaterializedRepository( + materialized: MaterializedPinnedSourceTree, + runner: CommandRunner, + reasons: IsolationAdmissionReason[], +): Promise { + const identity = JSON.parse( + await readFile(join(materialized.targetDir, SOURCE_IDENTITY_FILE), 'utf8'), + ) as Partial; + if ( + identity.recipeVersion !== RECIPE_VERSION || + identity.sourceCommit !== materialized.sourceCommit || + identity.sourceTree !== materialized.sourceTree + ) { + reasons.push({ + code: 'identity_mismatch', + detail: 'materialized source identity does not match declaration', + }); + } + const commitCount = ( + await gitChecked(runner, materialized.targetDir, ['rev-list', '--count', 'HEAD']) + ).stdout.trim(); + if (commitCount !== '1') { + reasons.push({ code: 'git_history_present', detail: `target has ${commitCount} reachable commits` }); + } + const head = (await gitChecked(runner, materialized.targetDir, ['rev-parse', 'HEAD'])).stdout.trim(); + if (head !== materialized.syntheticCommit) { + reasons.push({ + code: 'identity_mismatch', + detail: 'materialized target commit does not match declaration', + }); + } + const remotes = (await gitChecked(runner, materialized.targetDir, ['remote'])).stdout.trim(); + if (remotes.length > 0) { + reasons.push({ code: 'git_remote_present', detail: remotes }); + } + const refs = ( + await gitChecked(runner, materialized.targetDir, ['for-each-ref', '--format=%(refname)']) + ).stdout + .trim() + .split('\n') + .filter(Boolean); + const extraRefs = refs.filter((ref) => ref !== 'refs/heads/main'); + if (extraRefs.length > 0) { + reasons.push({ code: 'git_ref_present', detail: extraRefs.join(', ') }); + } + const worktreeChanges = ( + await gitChecked(runner, materialized.targetDir, ['status', '--porcelain']) + ).stdout.trim(); + if (worktreeChanges.length > 0) { + reasons.push({ code: 'git_worktree_changes_present', detail: worktreeChanges }); + } +} + +async function inspectTargetSymlinks(targetDir: string, reasons: IsolationAdmissionReason[]): Promise { + const root = await realpath(targetDir); + const inspectDirectory = async (directory: string): Promise => { + for (const entry of await readdir(directory, { withFileTypes: true })) { + if (entry.name === '.git' && directory === root) continue; + const path = join(directory, entry.name); + if (entry.isDirectory()) { + await inspectDirectory(path); + continue; + } + if (!entry.isSymbolicLink()) continue; + try { + const destination = await realpath(path); + if (!containedPath(root, destination)) { + reasons.push({ + code: 'path_boundary_weakened', + detail: `target symlink escapes isolation root: ${relative(root, path)}`, + }); + } + } catch { + reasons.push({ + code: 'path_boundary_weakened', + detail: `target symlink cannot be resolved safely: ${relative(root, path)}`, + }); + } + } + }; + await inspectDirectory(root); +} + +function containedPath(root: string, candidate: string): boolean { + const relation = relative(resolve(root), resolve(candidate)); + return relation === '' || (!relation.startsWith(`..${sep}`) && relation !== '..' && !isAbsolute(relation)); +} + +function sameStrings(actual: readonly string[], expected: readonly string[]): boolean { + return actual.length === expected.length && actual.every((value, index) => value === expected[index]); +} + +function canonicalPolicyPath(path: string): string { + const resolved = resolve(path); + try { + return realpathSync(resolved); + } catch { + const parent = dirname(resolved); + if (parent === resolved) return resolved; + return join(canonicalPolicyPath(parent), resolved.slice(parent.length + 1)); + } +} + +function denyTargetCommandAccessProfile(forbiddenReadRoots: readonly string[]): string { + const rules = ['(version 1)', '(allow default)', '(deny network*)']; + for (const root of forbiddenReadRoots) { + const literal = root.replaceAll('\\', '\\\\').replaceAll('"', '\\"'); + rules.push(`(deny file-read* file-write* (subpath "${literal}"))`); + } + return rules.join('\n'); +} + +async function gitChecked( + runner: CommandRunner, + cwd: string, + args: readonly string[], +): Promise { + return await commandChecked(runner, cwd, 'git', args); +} + +async function commandChecked( + runner: CommandRunner, + cwd: string, + command: string, + args: readonly string[], +): Promise { + const result = await runner(command, args, { cwd }); + if (result.exitCode !== 0) { + throw new Error(`${command} ${args.join(' ')} failed: ${result.stderr || result.stdout}`); + } + return result; +} diff --git a/src/dev/end-to-end-comparison/study-contract.ts b/src/dev/end-to-end-comparison/study-contract.ts index e61719f9e..9ecedff0d 100644 --- a/src/dev/end-to-end-comparison/study-contract.ts +++ b/src/dev/end-to-end-comparison/study-contract.ts @@ -38,6 +38,10 @@ export interface EndToEndStudyContract { readonly manifestPath: string; readonly manifestSha256: string; }; + readonly source?: { + readonly parentCommit: string; + readonly parentTree: string; + }; readonly budgets: { readonly elicitation: { readonly qualifyingQuestions: number; @@ -66,16 +70,38 @@ export function parseEndToEndStudyContract(value: unknown): EndToEndStudyContrac const requirementRegistry = child(value, 'requirementRegistry'); const executionContractTemplate = child(value, 'executionContractTemplate'); const oracle = child(value, 'oracle'); + const source = value['source'] === undefined ? undefined : child(value, 'source'); const budgets = child(value, 'budgets'); const elicitationBudget = child(budgets, 'elicitation'); const executionBudget = child(budgets, 'execution'); const actorRecipes = child(value, 'actorRecipes'); const executionRecipes = child(actorRecipes, 'execution'); + const knownCase = + (value['id'] === 'minimal-petri-net-editor-e2e-v1' && + value['caseId'] === 'minimal-petri-net-editor-v1' && + oracle['id'] === 'minimal-petri-net-editor-oracles-v2' && + source === undefined) || + (value['id'] === 'petrinaut-optimization-e2e-v1' && + value['caseId'] === 'petrinaut-optimization-v1' && + oracle['id'] === 'petrinaut-optimization-oracles-v1' && + source !== undefined && + source['parentCommit'] === '5c7a2d9db5caa851c38938f4b1bac19005b0e978' && + source['parentTree'] === 'a3e08cf75e00cc9016c931f4665341506e03533e' && + gitObjectId(source['parentCommit']) && + gitObjectId(source['parentTree']) && + exactKeys(source, ['parentCommit', 'parentTree'])) || + (value['id'] === 'brunch-host-landing-e2e-v1' && + value['caseId'] === 'brunch-host-landing-v1' && + oracle['id'] === 'brunch-host-landing-oracles-v1' && + source !== undefined && + gitObjectId(source['parentCommit']) && + gitObjectId(source['parentTree']) && + exactKeys(source, ['parentCommit', 'parentTree'])); + if ( value['schemaVersion'] !== 1 || - !safeId(value['id']) || - !safeId(value['caseId']) || + !knownCase || !addressedPath(mission) || !addressedPath(baseline) || !addressedPath(requirementRegistry) || @@ -163,6 +189,16 @@ function addressedPath(value: Record): boolean { return safeRelativePath(value['path']) && sha256(value['sha256']); } +function gitObjectId(value: unknown): value is string { + return typeof value === 'string' && /^[a-f0-9]{40}$/u.test(value); +} + +function exactKeys(value: Record, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + return actual.length === wanted.length && actual.every((key, index) => key === wanted[index]); +} + function child(value: Record, key: string): Record { const selected = value[key]; if (!record(selected)) invalid(); diff --git a/src/dev/execution-comparison-brunch.ts b/src/dev/execution-comparison-brunch.ts index 9de34cef5..1d69b48c2 100644 --- a/src/dev/execution-comparison-brunch.ts +++ b/src/dev/execution-comparison-brunch.ts @@ -13,34 +13,72 @@ import { runWithScopedBrunchOfflineDefault, type BrunchTuiLaunchContext, } from '../app/brunch-tui.js'; +import { openWorkspaceCommandExecutor } from '../graph/index.js'; +import { createNetworkDeniedCommandRunner } from './end-to-end-comparison/solution-isolation.js'; export async function runPinnedBrunchExecutionTui(input: { readonly workspaceDir: string; readonly specId: number; + readonly forbiddenRoots: readonly string[]; readonly provider: 'anthropic'; readonly model: 'claude-opus-4-8'; }): Promise { const model = getBuiltinModel(input.provider, input.model); + const preflight = await resolvePinnedBrunchPreflight({ + workspaceDir: input.workspaceDir, + specId: input.specId, + }); await runBrunchTui({ cwd: input.workspaceDir, openWeb: false, webSidecarRunner: async () => null, - runWorkspaceDialogPreflight: async () => ({ - action: 'newSession', - specId: input.specId, - establish: { origin: 'greenfield' }, - }), + runWorkspaceDialogPreflight: async () => preflight, launchInteractive: async (context) => { - await launchPinnedInteractive(context, model); + await launchPinnedInteractive(context, model, input.forbiddenRoots); }, }); } -async function launchPinnedInteractive(context: BrunchTuiLaunchContext, model: Model): Promise { +export async function resolvePinnedBrunchPreflight(input: { + readonly workspaceDir: string; + readonly specId: number; +}): Promise< + | { readonly action: 'newSession'; readonly specId: number } + | { + readonly action: 'newSession'; + readonly specId: number; + readonly establish: { readonly origin: 'greenfield' }; + } +> { + const executor = await openWorkspaceCommandExecutor(input.workspaceDir); + const spec = executor.getSpec(input.specId); + if (spec === undefined) throw new Error(`prepared Brunch specification ${input.specId} is missing`); + return spec.origin === null + ? { + action: 'newSession', + specId: input.specId, + establish: { origin: 'greenfield' }, + } + : { action: 'newSession', specId: input.specId }; +} + +async function launchPinnedInteractive( + context: BrunchTuiLaunchContext, + model: Model, + forbiddenRoots: readonly string[], +): Promise { const agentDir = getAgentDir(); + const verifier = createNetworkDeniedCommandRunner({ + forbiddenReadRoots: forbiddenRoots, + }); const createRuntime = createBrunchAgentSessionRuntimeFactory({ ...context, + allowSubagents: false, + comparisonIsolation: { + targetRoot: context.workspace.cwd, + verifier: verifier.run, + }, agentServices: { model }, }); const runtime = await createAgentSessionRuntime(createRuntime, { @@ -58,6 +96,8 @@ async function launchPinnedInteractive(context: BrunchTuiLaunchContext, model: M export function parseExecutionComparisonArgs(args: readonly string[]): { readonly workspaceDir: string; readonly specId: number; + readonly forbiddenRoots: readonly string[]; + readonly solutionIsolation: 'v1'; } { const { values } = parseNodeArgs({ args: [...args], @@ -66,20 +106,33 @@ export function parseExecutionComparisonArgs(args: readonly string[]): { options: { workspace: { type: 'string' }, 'spec-id': { type: 'string' }, + 'solution-isolation': { type: 'string' }, + 'forbidden-root': { type: 'string', multiple: true }, }, }); const workspaceDir = values.workspace; const specId = Number(values['spec-id']); - if (!workspaceDir || !Number.isSafeInteger(specId) || specId <= 0) { - throw new Error('Usage: execution-comparison-brunch --workspace --spec-id '); + const forbiddenRoots = values['forbidden-root'] ?? []; + if ( + !workspaceDir || + forbiddenRoots.length === 0 || + !Number.isSafeInteger(specId) || + specId <= 0 || + values['solution-isolation'] !== 'v1' + ) { + throw new Error( + 'Usage: execution-comparison-brunch --workspace --spec-id --solution-isolation v1 --forbidden-root [--forbidden-root ...]', + ); } - return { workspaceDir, specId }; + return { workspaceDir, specId, forbiddenRoots, solutionIsolation: 'v1' }; } async function main(): Promise { const args = parseExecutionComparisonArgs(process.argv.slice(2)); await runPinnedBrunchExecutionTui({ - ...args, + workspaceDir: args.workspaceDir, + specId: args.specId, + forbiddenRoots: args.forbiddenRoots, provider: 'anthropic', model: 'claude-opus-4-8', }); diff --git a/src/dev/execution-comparison-operator.ts b/src/dev/execution-comparison-operator.ts index 45e7c31d9..cb033b7ca 100644 --- a/src/dev/execution-comparison-operator.ts +++ b/src/dev/execution-comparison-operator.ts @@ -7,25 +7,132 @@ import { parseExecutionAttempt, writeExecutionAttemptImmutable, } from './execution-comparison/artifact-contract.js'; -import { runPetriEditorBrowserOracle } from './execution-comparison/browser-oracle.js'; +import { + runPetriEditorBrowserOracle, + type BrowserOracleReport, +} from './execution-comparison/browser-oracle.js'; +import { + runBrunchHostLandingOracle, + type HostLandingOracleReport, +} from './execution-comparison/host-landing-oracle.js'; import { listExecutionCases, prepareExecutionTarget, resolveExecutionCase, } from './execution-comparison/operator-cli.js'; -import { loadControllerOraclePack } from './execution-comparison/oracle-pack.js'; +import { + loadControllerOracleManifest, + loadControllerOraclePack, +} from './execution-comparison/oracle-pack.js'; +import { + runPetrinautOptimizationOracle, + type PetrinautOptimizationOracleReport, +} from './execution-comparison/petrinaut-optimization-oracle.js'; + +type CompiledOracleId = + | 'minimal-petri-net-editor-oracles-v2' + | 'brunch-host-landing-oracles-v1' + | 'petrinaut-optimization-oracles-v1'; + +interface CompiledOracle { + readonly implementationFiles: readonly string[]; + readonly run: (input: { + readonly appDir: string; + readonly caseDir: string; + }) => Promise; +} + +const COMPILED_ORACLES: Readonly> = { + 'minimal-petri-net-editor-oracles-v2': { + implementationFiles: [ + fileURLToPath(new URL('./execution-comparison/browser-oracle.ts', import.meta.url)), + fileURLToPath(new URL('./execution-comparison/browser-oracle/journey-runner.ts', import.meta.url)), + fileURLToPath(new URL('./execution-comparison/petri-reference.ts', import.meta.url)), + ], + run: runPetriEditorBrowserOracle, + }, + 'brunch-host-landing-oracles-v1': { + implementationFiles: [ + fileURLToPath(new URL('./execution-comparison/host-landing-oracle.ts', import.meta.url)), + fileURLToPath(new URL('./execution-comparison/host-landing-oracle/types.ts', import.meta.url)), + fileURLToPath(new URL('./execution-comparison/host-landing-oracle/git-model.ts', import.meta.url)), + fileURLToPath(new URL('./execution-comparison/host-landing-oracle/fixture.ts', import.meta.url)), + fileURLToPath(new URL('./execution-comparison/host-landing-oracle/runner.ts', import.meta.url)), + fileURLToPath(new URL('./tui-driver.ts', import.meta.url)), + fileURLToPath(new URL('./tui-driver/session.ts', import.meta.url)), + fileURLToPath(new URL('./tui-driver/screen.ts', import.meta.url)), + ], + run: async ({ appDir }) => await runBrunchHostLandingOracle({ candidateRoot: appDir }), + }, + 'petrinaut-optimization-oracles-v1': { + implementationFiles: [ + fileURLToPath(new URL('./execution-comparison/petrinaut-optimization-oracle.ts', import.meta.url)), + fileURLToPath( + new URL('./execution-comparison/petrinaut-optimization-oracle/types.ts', import.meta.url), + ), + fileURLToPath( + new URL('./execution-comparison/petrinaut-optimization-oracle/runner.ts', import.meta.url), + ), + fileURLToPath( + new URL('./execution-comparison/petrinaut-optimization-oracle/browser.ts', import.meta.url), + ), + fileURLToPath( + new URL('./execution-comparison/petrinaut-optimization-oracle/claims.ts', import.meta.url), + ), + fileURLToPath( + new URL('./execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts', import.meta.url), + ), + ], + run: async ({ appDir, caseDir }) => + await runPetrinautOptimizationOracle({ candidateRoot: appDir, caseDir }), + }, +}; + +export function resolveCompiledExecutionOracle(id: string): CompiledOracle { + if ( + id !== 'minimal-petri-net-editor-oracles-v2' && + id !== 'brunch-host-landing-oracles-v1' && + id !== 'petrinaut-optimization-oracles-v1' + ) { + throw new Error(`unknown compiled execution oracle id: ${id}`); + } + return COMPILED_ORACLES[id]; +} + +export async function retainCompiledOracleReport(input: { + readonly out: string; + readonly oracleId: CompiledOracleId; + readonly oraclePackSha256: string; + readonly report: BrowserOracleReport | HostLandingOracleReport | PetrinautOptimizationOracleReport; +}): Promise<{ + readonly out: string; + readonly status: 'passed' | 'failed' | 'assertion_failed' | 'setup_failed'; + readonly oraclePackSha256: string; + readonly oracleId: CompiledOracleId; +}> { + const out = resolve(input.out); + await mkdir(dirname(out), { recursive: true }); + await writeFile(out, `${JSON.stringify(input.report, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' }); + return { + out, + status: input.report.status, + oraclePackSha256: input.oraclePackSha256, + oracleId: input.oracleId, + }; +} export const EXECUTION_COMPARISON_SHARED_FRAMING = [ 'Implement the frozen specification and public contract supplied in this isolated target.', 'Treat their bytes as immutable input: do not normalize, repair, or replace either file.', 'Work only in the target repository and do not inspect controller paths or seek hidden comparison material.', - 'Deliver the static browser application in dist/ and run npm test followed by npm run build.', - 'Stop after those commands and report the visible result; do not add a backend or runtime network dependency.', + 'Follow the case-specific delivery, acceptance, network, and terminal rules exactly.', + 'Stop at the contract-declared execution terminal and report only target-visible results.', ].join('\n'); const DEFAULT_CASES_ROOT = fileURLToPath( new URL('../../testing/execution-comparisons/cases/', import.meta.url), ); +const DEFAULT_CONTROLLER_ROOT = fileURLToPath(new URL('../../', import.meta.url)); export async function runExecutionComparisonOperatorCli(args: readonly string[]): Promise { const [command, ...rest] = args; @@ -60,16 +167,19 @@ export async function runExecutionComparisonOperatorCli(args: readonly string[]) return; } case 'prepare': { - assertOnlyOptions(options, ['case', 'lane', 'target']); + assertOnlyOptions(options, ['case', 'lane', 'target', 'source-repository']); const lane = required(options, 'lane'); if (lane !== 'brunch' && lane !== 'claude_code') { throw new Error('--lane must be brunch or claude_code'); } + const sourceRepository = options.get('source-repository'); const prepared = await prepareExecutionTarget({ lane, caseReference: required(options, 'case'), casesRoot, targetDir: resolve(required(options, 'target')), + controllerRoot: DEFAULT_CONTROLLER_ROOT, + ...(sourceRepository === undefined ? {} : { sourceRepositoryDir: resolve(sourceRepository) }), }); process.stdout.write(`${JSON.stringify(prepared, null, 2)}\n`); return; @@ -77,29 +187,23 @@ export async function runExecutionComparisonOperatorCli(args: readonly string[]) case 'oracle': { assertOnlyOptions(options, ['case', 'app', 'out']); const selected = await resolveExecutionCase(required(options, 'case'), casesRoot); + const manifest = await loadControllerOracleManifest(selected.caseDir); + const oracle = resolveCompiledExecutionOracle(manifest.id); const oraclePack = await loadControllerOraclePack({ caseDir: selected.caseDir, - implementationFiles: [ - fileURLToPath(new URL('./execution-comparison/browser-oracle.ts', import.meta.url)), - fileURLToPath(new URL('./execution-comparison/browser-oracle/journey-runner.ts', import.meta.url)), - fileURLToPath(new URL('./execution-comparison/petri-reference.ts', import.meta.url)), - ], + implementationFiles: oracle.implementationFiles, }); - const report = await runPetriEditorBrowserOracle({ + const report = await oracle.run({ appDir: resolve(required(options, 'app')), caseDir: selected.caseDir, }); - const out = resolve(required(options, 'out')); - await mkdir(dirname(out), { recursive: true }); - await writeFile(out, `${JSON.stringify(report, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' }); - process.stdout.write( - `${JSON.stringify({ - out, - status: report.status, - oraclePackSha256: oraclePack.packSha256, - browserSuiteVersion: oraclePack.manifest.browserSuiteVersion, - })}\n`, - ); + const retained = await retainCompiledOracleReport({ + out: required(options, 'out'), + oracleId: oraclePack.manifest.id, + oraclePackSha256: oraclePack.packSha256, + report, + }); + process.stdout.write(`${JSON.stringify(retained)}\n`); return; } case 'retain-attempt': { diff --git a/src/dev/execution-comparison/__tests__/accessibility-contract.test.ts b/src/dev/execution-comparison/__tests__/accessibility-contract.test.ts index fe8594669..2d63ea2f0 100644 --- a/src/dev/execution-comparison/__tests__/accessibility-contract.test.ts +++ b/src/dev/execution-comparison/__tests__/accessibility-contract.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest'; import { assertAccessibilityContract } from '../accessibility-contract.js'; -import type { ExecutionCasePublicContract } from '../case-contract.js'; +import type { BrowserExecutionCasePublicContract } from '../case-contract.js'; const accessibility = { application: { role: 'application', name: 'Petri net editor' }, @@ -31,7 +31,7 @@ const accessibility = { { role: 'spinbutton', name: 'Arc weight' }, ], feedbackRoles: ['status', 'alert'], -} satisfies ExecutionCasePublicContract['accessibility']; +} satisfies BrowserExecutionCasePublicContract['accessibility']; function buttons(): string { return accessibility.controls.map(({ name }) => ``).join(''); diff --git a/src/dev/execution-comparison/__tests__/brunch-lane.test.ts b/src/dev/execution-comparison/__tests__/brunch-lane.test.ts index d551eecc0..5b828f3eb 100644 --- a/src/dev/execution-comparison/__tests__/brunch-lane.test.ts +++ b/src/dev/execution-comparison/__tests__/brunch-lane.test.ts @@ -15,18 +15,23 @@ import { queryGraph } from '../../../graph/queries.js'; import { seedFixture } from '../../../graph/seed-fixtures.js'; import { buildBrunchExecutionSeed, + buildOpaqueBrownfieldExecutionSeed, buildOpaqueBrunchExecutionSeed, prepareBrunchExecutionWorkspace, } from '../brunch-lane.js'; -import { loadPublicCasePacket } from '../case-contract.js'; +import { isBrowserExecutionCaseContract, loadPublicCasePacket } from '../case-contract.js'; const caseDir = fileURLToPath( new URL('../../../../testing/execution-comparisons/cases/minimal-petri-net-editor/', import.meta.url), ); +const petrinautCaseDir = fileURLToPath( + new URL('../../../../testing/execution-comparisons/cases/petrinaut-optimization/', import.meta.url), +); describe('Brunch execution comparison lane adapter', () => { it('projects the frozen public packet into one complete greenfield execution scope', async () => { const packet = await loadPublicCasePacket(caseDir); + if (!isBrowserExecutionCaseContract(packet.contract)) throw new Error('expected browser case'); const specification = await readFile(`${caseDir}/spec.md`, 'utf8'); const fixture = buildBrunchExecutionSeed({ specification, contract: packet.contract }); expect(fixture.nodes.find((node) => node.source === 'approved-spec [D1]')).toMatchObject({ @@ -98,6 +103,7 @@ describe('Brunch execution comparison lane adapter', () => { it('preserves an arbitrary target-authored specification as one exact settled requirement', async () => { const packet = await loadPublicCasePacket(caseDir); + if (!isBrowserExecutionCaseContract(packet.contract)) throw new Error('expected browser case'); const specification = '# Target-authored specification\n\nSpacing stays exact. \n'; const fixture = buildOpaqueBrunchExecutionSeed({ specification, @@ -130,6 +136,32 @@ describe('Brunch execution comparison lane adapter', () => { expect(() => assertExecuteProjectionPlanReady(projection)).not.toThrow(); }); + it('projects an opaque brownfield specification without Petri-specific execution wording', async () => { + const packet = await loadPublicCasePacket(petrinautCaseDir); + const specification = await readFile(join(petrinautCaseDir, 'spec.md'), 'utf8'); + const fixture = buildOpaqueBrownfieldExecutionSeed({ + specification, + contract: packet.contract, + }); + expect(fixture.nodes.find((node) => node.source === 'e2e-handoff [exact-spec]')).toMatchObject({ + kind: 'requirement', + body: specification, + }); + expect(JSON.stringify(fixture)).not.toMatch(/Petri-net|static browser|npm install/iu); + + const db = createDb(':memory:'); + const seeded = seedFixture(new CommandExecutor(db), fixture); + const graph = queryGraph(db, seeded.specId); + const projection = projectExecuteGraph({ + specId: seeded.specId, + graphLsn: graph.lsn, + mode: 'brownfield', + nodes: graph.nodes, + edges: graph.edges, + }); + expect(() => assertExecuteProjectionPlanReady(projection)).not.toThrow(); + }); + it('prepares a fresh Brunch workspace with only the content-addressed public packet', async () => { const workspaceDir = await mkdtemp(join(tmpdir(), 'brunch-execution-lane-')); const prepared = await prepareBrunchExecutionWorkspace({ workspaceDir, caseDir }); diff --git a/src/dev/execution-comparison/__tests__/case-contract.test.ts b/src/dev/execution-comparison/__tests__/case-contract.test.ts index 7cd2e78ba..73e977563 100644 --- a/src/dev/execution-comparison/__tests__/case-contract.test.ts +++ b/src/dev/execution-comparison/__tests__/case-contract.test.ts @@ -5,14 +5,114 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { loadPublicCasePacket } from '../case-contract.js'; -import { loadControllerOraclePack } from '../oracle-pack.js'; +import { loadPublicCasePacket, parsePublicCaseContract } from '../case-contract.js'; +import { isPetriControllerOracleManifest, loadControllerOraclePack } from '../oracle-pack.js'; const caseDir = fileURLToPath( new URL('../../../../testing/execution-comparisons/cases/minimal-petri-net-editor/', import.meta.url), ); describe('execution comparison public case contract', () => { + it('accepts only the frozen greenfield and two exact brownfield profile variants', () => { + const brunchContract = { + schemaVersion: 1, + case: { + id: 'brunch-host-landing-v1', + specification: 'spec.md', + specificationSha256: 'a'.repeat(64), + provider: 'anthropic', + model: 'claude-opus-4-8', + product: 'brunch', + mode: 'brownfield', + scope: 'single_feature', + surface: 'backend', + repository: { + substrate: 'pinned_git', + parentCommit: '1'.repeat(40), + parentTree: '2'.repeat(40), + }, + }, + budgets: { + elapsedMinutes: 90, + mechanicalInterventions: 2, + substantiveHumanInterventions: 0, + }, + delivery: { + runtimeNetwork: 'forbidden', + dependencyInstallNetwork: 'forbidden', + }, + acceptance: { + publicCommand: '/brunch:land', + executionTerminal: 'promotion_prepared', + }, + rules: ['Work only in the target repository.'], + }; + + expect(parsePublicCaseContract(brunchContract)).toEqual(brunchContract); + for (const mutation of [ + { accessibility: { application: { role: 'application', name: 'Browser-only leak' } } }, + { case: { ...brunchContract.case, mode: 'greenfield' } }, + { case: { ...brunchContract.case, repository: { substrate: 'empty_dir' } } }, + { delivery: { ...brunchContract.delivery, test: { command: 'sh', args: ['oracle.sh'] } } }, + ]) { + expect(() => parsePublicCaseContract({ ...brunchContract, ...mutation })).toThrow( + 'invalid fixed public execution contract', + ); + } + + const petrinautContract = { + ...brunchContract, + case: { + ...brunchContract.case, + id: 'petrinaut-optimization-v1', + product: 'petrinaut', + surface: 'frontend', + repository: { + substrate: 'pinned_git', + parentCommit: '5c7a2d9db5caa851c38938f4b1bac19005b0e978', + parentTree: 'a3e08cf75e00cc9016c931f4665341506e03533e', + }, + }, + delivery: { + runtimeNetwork: 'forbidden', + dependencyInstallNetwork: 'controller_only', + }, + acceptance: { + publicRoute: '/optimization', + sameOriginApi: '/api/petrinaut-opt/optimize/all', + executionTerminal: 'promotion_prepared', + }, + accessibility: { + view: { role: 'heading', name: 'Optimizations' }, + tab: { role: 'tab', name: 'Optimizations' }, + create: { role: 'button', name: 'Create optimization' }, + scenario: { role: 'combobox', name: 'Scenario' }, + metric: { role: 'combobox', name: 'Objective metric' }, + direction: { role: 'combobox', name: 'Objective direction' }, + run: { role: 'button', name: 'Run optimization' }, + cancel: { role: 'button', name: 'Cancel optimization' }, + status: { role: 'status', name: 'Optimization status' }, + results: { role: 'region', name: 'Optimization results' }, + }, + }; + expect(parsePublicCaseContract(petrinautContract)).toEqual(petrinautContract); + for (const mutation of [ + { case: { ...petrinautContract.case, product: 'brunch' } }, + { case: { ...petrinautContract.case, surface: 'backend' } }, + { + case: { + ...petrinautContract.case, + repository: { ...petrinautContract.case.repository, parentTree: '2'.repeat(40) }, + }, + }, + { acceptance: { ...petrinautContract.acceptance, publicRoute: '/processes/draft' } }, + ]) { + expect(() => parsePublicCaseContract({ ...petrinautContract, ...mutation })).toThrow( + 'invalid fixed public execution contract', + ); + } + }); + it('freezes only the approved specification and public contract', async () => { const packet = await loadPublicCasePacket(caseDir); @@ -83,6 +183,7 @@ describe('execution comparison public case contract', () => { browserSuiteVersion: 'petri-editor-browser-v2', referenceModelVersion: 'weighted-pt-v1', }); + if (!isPetriControllerOracleManifest(pack.manifest)) throw new Error('expected Petri manifest'); expect(pack.manifest.journeys.map((journey) => journey.id)).toEqual([ 'mount', 'node-lifecycle', diff --git a/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts b/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts index a344c5611..fb9b8c17c 100644 --- a/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts +++ b/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts @@ -1,24 +1,193 @@ -import { describe, expect, it } from 'vitest'; +import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { createSubagentToolCatalog } from '../../../.pi/extensions/subagents/session.js'; +import { registerBrunchOperationalModePolicy } from '../../../app/pi-extensions.js'; +import { loadBrunchSubagents } from '../../../app/pi-subagents.js'; import { parseExecutionComparisonArgs } from '../../execution-comparison-brunch.js'; +const roots: string[] = []; + +afterEach(async () => { + await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); +}); + describe('execution comparison Brunch CLI arguments', () => { it('parses a complete workspace and positive specification id', () => { - expect(parseExecutionComparisonArgs(['--workspace', '/tmp/petri-editor', '--spec-id', '17'])).toEqual({ + expect( + parseExecutionComparisonArgs([ + '--workspace', + '/tmp/petri-editor', + '--spec-id', + '17', + '--solution-isolation', + 'v1', + '--forbidden-root', + '/tmp/controller', + ]), + ).toEqual({ workspaceDir: '/tmp/petri-editor', specId: 17, + forbiddenRoots: ['/tmp/controller'], + solutionIsolation: 'v1', }); }); it('rejects another option where the workspace value is required', () => { - expect(() => parseExecutionComparisonArgs(['--workspace', '--spec-id', '17'])).toThrow(); + expect(() => + parseExecutionComparisonArgs([ + '--workspace', + '--spec-id', + '17', + '--solution-isolation', + 'v1', + '--forbidden-root', + '/tmp/controller', + ]), + ).toThrow(); }); it('rejects missing, non-integer, and unknown options', () => { expect(() => parseExecutionComparisonArgs(['--workspace', '/tmp/petri-editor'])).toThrow('Usage:'); expect(() => - parseExecutionComparisonArgs(['--workspace', '/tmp/petri-editor', '--spec-id', '1.5']), + parseExecutionComparisonArgs([ + '--workspace', + '/tmp/petri-editor', + '--spec-id', + '1.5', + '--solution-isolation', + 'v1', + '--forbidden-root', + '/tmp/controller', + ]), + ).toThrow('Usage:'); + expect(() => + parseExecutionComparisonArgs(['--workspace', '/tmp/petri-editor', '--spec-id', '1']), + ).toThrow('Usage:'); + expect(() => + parseExecutionComparisonArgs([ + '--workspace', + '/tmp/petri-editor', + '--spec-id', + '1', + '--solution-isolation', + 'prompt-only', + '--forbidden-root', + '/tmp/controller', + ]), ).toThrow('Usage:'); expect(() => parseExecutionComparisonArgs(['--unknown', 'value'])).toThrow(); }); + + it('bounds every foreground filesystem tool to the comparison target, including symlinks', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-comparison-tools-')); + roots.push(root); + const targetRoot = join(root, 'target'); + const outsideRoot = join(root, 'controller'); + await mkdir(targetRoot); + await mkdir(outsideRoot); + await writeFile(join(targetRoot, 'inside.txt'), 'inside\n'); + await writeFile(join(outsideRoot, 'secret.txt'), 'historical solution\n'); + await symlink(join(outsideRoot, 'secret.txt'), join(targetRoot, 'escaped-link.txt')); + await symlink(outsideRoot, join(targetRoot, 'escaped-directory')); + + const tools: Array<{ + readonly name: string; + readonly execute: ( + toolCallId: string, + params: Record, + signal: AbortSignal | undefined, + onUpdate: undefined, + ctx: { cwd: string }, + ) => Promise; + }> = []; + registerBrunchOperationalModePolicy( + { + registerTool: (tool: unknown) => tools.push(tool as (typeof tools)[number]), + getAllTools: () => tools, + setActiveTools: () => {}, + on: () => {}, + } as never, + { filesystemRoot: targetRoot }, + ); + + const parameters = new Map>([ + ['read', { path: join(targetRoot, 'inside.txt') }], + ['grep', { pattern: 'inside', path: targetRoot }], + ['find', { pattern: '*.txt', path: targetRoot }], + ['ls', { path: targetRoot }], + ]); + expect(tools.map(({ name }) => name)).toEqual(['read', 'grep', 'find', 'ls']); + for (const tool of tools) { + await expect( + tool.execute('allowed', parameters.get(tool.name)!, undefined, undefined, { + cwd: targetRoot, + }), + ).resolves.toBeDefined(); + await expect( + tool.execute( + 'parent-escape', + { ...parameters.get(tool.name), path: outsideRoot }, + undefined, + undefined, + { cwd: targetRoot }, + ), + ).rejects.toThrow('escapes target root'); + } + await expect( + tools[0]!.execute( + 'symlink-escape', + { path: join(targetRoot, 'escaped-link.txt') }, + undefined, + undefined, + { cwd: targetRoot }, + ), + ).rejects.toThrow('escapes target root through symlink'); + + const childRead = createSubagentToolCatalog(targetRoot).get('read')!; + await expect( + childRead.execute('child-allowed', { path: 'inside.txt' }, undefined, undefined, { + cwd: targetRoot, + } as never), + ).resolves.toBeDefined(); + await expect( + childRead.execute('child-parent-escape', { path: outsideRoot }, undefined, undefined, { + cwd: targetRoot, + } as never), + ).rejects.toThrow(); + await expect( + childRead.execute('child-symlink-escape', { path: 'escaped-link.txt' }, undefined, undefined, { + cwd: targetRoot, + } as never), + ).rejects.toThrow(); + const childWrite = createSubagentToolCatalog(targetRoot).get('write_worktree_file')!; + await expect( + childWrite.execute( + 'child-write-symlink-escape', + { path: 'escaped-directory/leak.txt', content: 'must stay sealed\n' }, + undefined, + undefined, + { cwd: targetRoot } as never, + ), + ).rejects.toThrow('escapes the worktree'); + }); + + it('loads only the sealed planner and worker for comparison execution', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-comparison-subagents-')); + roots.push(root); + const subagents = await loadBrunchSubagents({ + cwd: root, + agentDir: join(root, 'agent'), + delegatableAgents: [], + includedAgents: ['planner', 'worker'], + }); + + expect([...subagents.definitions.keys()]).toEqual(['planner', 'worker']); + expect(subagents.definitions.get('planner')?.tools).toEqual(['read']); + expect(subagents.definitions.get('worker')?.tools).toEqual(['read', 'write_worktree_file']); + expect(subagents.definitions.has('researcher')).toBe(false); + }); }); diff --git a/src/dev/execution-comparison/__tests__/host-landing-oracle.slow.test.ts b/src/dev/execution-comparison/__tests__/host-landing-oracle.slow.test.ts new file mode 100644 index 000000000..1c727da0a --- /dev/null +++ b/src/dev/execution-comparison/__tests__/host-landing-oracle.slow.test.ts @@ -0,0 +1,54 @@ +import { execFile } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +import { beforeAll, describe, expect, it } from 'vitest'; + +import { runBrunchHostLandingOracle } from '../host-landing-oracle.js'; + +const execFileAsync = promisify(execFile); +const candidateRoot = fileURLToPath(new URL('../../../../', import.meta.url)); + +describe('built Brunch host-landing oracle through the public TUI', () => { + beforeAll(async () => { + await execFileAsync('npm', ['run', 'build'], { + cwd: candidateRoot, + timeout: 240_000, + maxBuffer: 2 * 1024 * 1024, + }); + }, 250_000); + + it('rejects a fresh session before any provider-capable candidate launch', async () => { + const report = await runBrunchHostLandingOracle({ + candidateRoot, + sessionMode: 'fresh', + }); + + expect(report.status).toBe('setup_failed'); + expect(report.setupFailure).toContain('not settled'); + }); + + it.each([ + ['brownfield_success', 'passed'], + ['greenfield_success', 'passed'], + ['decline', 'passed'], + ['dirty_host', 'passed'], + ['conflict', 'passed'], + ['stale_acceptance', 'passed'], + ['final_commit_only', 'assertion_failed'], + ['bookkeeping_retained', 'assertion_failed'], + ] as const)( + 'judges %s as %s through /brunch:land', + async (scenario, expectedStatus) => { + const report = await runBrunchHostLandingOracle({ candidateRoot, scenario }); + + expect( + report.status, + [report.setupFailure, ...report.terminalEvidence].filter(Boolean).join('\n'), + ).toBe(expectedStatus); + expect(report.terminalEvidence.join('\n')).toContain('/brunch:land'); + expect(report.checks.find(({ id }) => id === 'public-tui-preflight')?.status).toBe('passed'); + }, + 90_000, + ); +}); diff --git a/src/dev/execution-comparison/__tests__/host-landing-oracle.test.ts b/src/dev/execution-comparison/__tests__/host-landing-oracle.test.ts new file mode 100644 index 000000000..bd83610f5 --- /dev/null +++ b/src/dev/execution-comparison/__tests__/host-landing-oracle.test.ts @@ -0,0 +1,135 @@ +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +import { describe, expect, it } from 'vitest'; + +import { + evaluateHostLandingGitOutcome, + snapshotGitState, + type HostLandingScenario, +} from '../host-landing-oracle.js'; + +const execFileAsync = promisify(execFile); +const oracleSources = [ + '../host-landing-oracle.ts', + '../host-landing-oracle/types.ts', + '../host-landing-oracle/git-model.ts', + '../host-landing-oracle/fixture.ts', + '../host-landing-oracle/runner.ts', +].map((path) => fileURLToPath(new URL(path, import.meta.url))); +const IDENTITY = ['-c', 'user.name=oracle', '-c', 'user.email=oracle@invalid.local'] as const; + +describe('controller-owned host-landing oracle boundary', () => { + it('depends only on the public candidate launch, TUI driver, Git, and controller outputs', async () => { + const source = (await Promise.all(oracleSources.map((path) => readFile(path, 'utf8')))).join('\n'); + const imports = source + .split('\n') + .filter((line) => line.startsWith('import ') || line.startsWith('} from ')); + + expect(imports.join('\n')).not.toMatch( + /landing\.js|git-host-land-port|execute-land|executor\/|historical|FE-1201/iu, + ); + expect(source).toContain("join(input.candidateRoot, 'bin', 'brunch.js')"); + expect(source).toContain("'/usr/bin/env'"); + expect(source).toContain("'PI_OFFLINE=1'"); + }); + + it.each(['final_commit_only', 'bookkeeping_retained'] as const)( + 'rejects the focused %s rival against the independent expected tree', + async (scenario) => { + const fixture = await rivalFixture(scenario); + const report = await evaluateHostLandingGitOutcome({ + scenario, + hostDir: fixture.hostDir, + metadataPath: fixture.metadataPath, + canonicalExpectedTree: fixture.expectedTree, + before: fixture.snapshot, + preConfirm: fixture.snapshot, + terminalEvidence: ['3 commits across the complete range'], + providerActivity: false, + }); + + expect(report.status).toBe('assertion_failed'); + expect(report.checks.find(({ id }) => id === 'brownfield-full-range')?.status).toBe('failed'); + }, + ); + + it.each(['decline', 'dirty_host', 'conflict', 'stale_acceptance'] as const)( + 'accepts byte-identical %s refusal evidence without a landed status', + async (scenario) => { + const fixture = await refusalFixture(); + const report = await evaluateHostLandingGitOutcome({ + scenario, + hostDir: fixture.hostDir, + metadataPath: fixture.metadataPath, + canonicalExpectedTree: fixture.snapshot.tree, + before: fixture.snapshot, + preConfirm: fixture.snapshot, + terminalEvidence: ['complete range; Nothing changed'], + providerActivity: false, + }); + + expect(report.status).toBe('passed'); + expect(report.checks.find(({ id }) => id === 'refusal-safety')?.status).toBe('passed'); + }, + ); +}); + +async function rivalFixture( + scenario: Extract, +) { + const root = await mkdtemp(join(tmpdir(), `brunch-host-oracle-${scenario}-`)); + const model = join(root, 'model'); + await init(model); + await commit(model, 'src/a.ts', 'a\n', 'a'); + await commit(model, 'src/b.ts', 'b\n', 'b'); + await commit(model, 'src/c.ts', 'c\n', 'c'); + const expectedTree = await git(model, ['rev-parse', 'HEAD^{tree}']); + + const hostDir = join(root, 'host'); + await init(hostDir); + if (scenario === 'final_commit_only') { + await commit(hostDir, 'src/c.ts', 'c\n', 'tip only'); + } else { + await commit(hostDir, 'src/a.ts', 'a\n', 'a'); + await commit(hostDir, 'src/b.ts', 'b\n', 'b'); + await commit(hostDir, 'src/c.ts', 'c\n', 'c'); + await commit(hostDir, '.brunch/leak.json', '{}\n', 'bookkeeping'); + } + const metadataPath = join(hostDir, '.brunch/controller-run.json'); + await mkdir(dirname(metadataPath), { recursive: true }); + await writeFile(metadataPath, '{"status":"landed"}\n'); + const snapshot = await snapshotGitState(hostDir, metadataPath); + return { hostDir, metadataPath, expectedTree, snapshot }; +} + +async function refusalFixture() { + const root = await mkdtemp(join(tmpdir(), 'brunch-host-oracle-refusal-')); + const hostDir = join(root, 'host'); + await init(hostDir); + const metadataPath = join(hostDir, '.brunch/controller-run.json'); + await mkdir(dirname(metadataPath), { recursive: true }); + await writeFile(metadataPath, '{"status":"promotion_prepared"}\n'); + return { hostDir, metadataPath, snapshot: await snapshotGitState(hostDir, metadataPath) }; +} + +async function init(cwd: string): Promise { + await mkdir(cwd); + await execFileAsync('git', ['init', '-q', '-b', 'main'], { cwd }); + await execFileAsync('git', [...IDENTITY, 'commit', '--allow-empty', '-q', '-m', 'base'], { cwd }); +} + +async function commit(cwd: string, path: string, content: string, message: string): Promise { + await mkdir(dirname(join(cwd, path)), { recursive: true }); + await writeFile(join(cwd, path), content); + await execFileAsync('git', ['add', '--', path], { cwd }); + await execFileAsync('git', [...IDENTITY, 'commit', '-q', '-m', message], { cwd }); +} + +async function git(cwd: string, args: readonly string[]): Promise { + return (await execFileAsync('git', [...args], { cwd })).stdout.trim(); +} diff --git a/src/dev/execution-comparison/__tests__/operator-cli.test.ts b/src/dev/execution-comparison/__tests__/operator-cli.test.ts index a0c4b478c..2c4bb6936 100644 --- a/src/dev/execution-comparison/__tests__/operator-cli.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-cli.test.ts @@ -5,24 +5,50 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; +import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; +import { + assertExecuteProjectionPlanReady, + projectExecuteGraph, +} from '../../../executor/execute-projection.js'; +import { openWorkspaceDb } from '../../../graph/index.js'; +import { queryGraph } from '../../../graph/queries.js'; +import { + parseExecutionComparisonArgs, + resolvePinnedBrunchPreflight, +} from '../../execution-comparison-brunch.js'; +import { runExecutionComparisonOperatorCli } from '../../execution-comparison-operator.js'; import { listExecutionCases, prepareExecutionTarget, resolveExecutionCase } from '../operator-cli.js'; const casesRoot = fileURLToPath(new URL('../../../../testing/execution-comparisons/cases/', import.meta.url)); const frozenCase = join(casesRoot, 'minimal-petri-net-editor'); +const controllerRoot = fileURLToPath(new URL('../../../../', import.meta.url)); +const hashSourceRepository = fileURLToPath(new URL('../../../../../hash/', import.meta.url)); describe('execution comparison operator case selection', () => { it('lists and resolves eligible case ids only inside the cases root', async () => { await expect(listExecutionCases(casesRoot)).resolves.toEqual([ + { + caseId: 'brunch-host-landing-v1', + directoryId: 'brunch-host-landing', + }, { caseId: 'minimal-petri-net-editor-v1', directoryId: 'minimal-petri-net-editor', }, + { + caseId: 'petrinaut-optimization-v1', + directoryId: 'petrinaut-optimization', + }, ]); await expect(resolveExecutionCase('minimal-petri-net-editor', casesRoot)).resolves.toMatchObject({ caseId: 'minimal-petri-net-editor-v1', directoryId: 'minimal-petri-net-editor', caseDir: frozenCase, }); + await expect(resolveExecutionCase('petrinaut-optimization-v1', casesRoot)).resolves.toMatchObject({ + caseId: 'petrinaut-optimization-v1', + directoryId: 'petrinaut-optimization', + }); }); it.each(['/tmp/minimal-petri-net-editor', '../minimal-petri-net-editor', 'controller', 'x/controller/y'])( @@ -84,7 +110,7 @@ describe('execution comparison target preparation', () => { expect(paths.some((path) => path.toLowerCase().includes('controller'))).toBe(false); expect(prepared.packet.files.map((file) => file.path)).toEqual(['public-contract.json', 'spec.md']); - if (prepared.lane === 'claude_code') { + if (prepared.preparation === 'empty_git') { expect(await readFile(join(targetDir, 'spec.md'), 'utf8')).toBe( await readFile(join(frozenCase, 'spec.md'), 'utf8'), ); @@ -92,7 +118,7 @@ describe('execution comparison target preparation', () => { await readFile(join(frozenCase, 'public-contract.json'), 'utf8'), ); expect(prepared.baseSha).toMatch(/^[a-f0-9]{40}$/u); - } else { + } else if (prepared.preparation === 'legacy_brunch') { expect(prepared.specId).toBe(1); } } finally { @@ -100,4 +126,202 @@ describe('execution comparison target preparation', () => { } }, ); + + it('requires the explicit source repository only for pinned cases', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-pinned-source-option-')); + try { + await expect( + runExecutionComparisonOperatorCli([ + 'prepare', + '--case', + 'petrinaut-optimization-v1', + '--lane', + 'claude_code', + '--target', + join(root, 'petrinaut'), + ]), + ).rejects.toThrow('requires --source-repository'); + await expect( + runExecutionComparisonOperatorCli([ + 'prepare', + '--case', + 'minimal-petri-net-editor-v1', + '--lane', + 'claude_code', + '--target', + join(root, 'petri'), + '--source-repository', + hashSourceRepository, + ]), + ).rejects.toThrow('valid only for pinned execution cases'); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('rejects pinned targets overlapping controller or source roots before materialization', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-pinned-source-boundary-')); + const sourceRoot = join(root, 'source'); + const sourceLink = join(root, 'source-link'); + await mkdir(sourceRoot); + await symlink(sourceRoot, sourceLink); + try { + await expect( + prepareExecutionTarget({ + lane: 'claude_code', + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir: join(root, 'relative-source-target'), + controllerRoot, + sourceRepositoryDir: 'relative-source', + dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + }), + ).rejects.toThrow('must be an absolute path'); + await expect( + prepareExecutionTarget({ + lane: 'claude_code', + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir: join(controllerRoot, '.unsafe-petrinaut-target'), + controllerRoot, + sourceRepositoryDir: sourceRoot, + dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + }), + ).rejects.toThrow('controller and target roots must be disjoint'); + await expect( + prepareExecutionTarget({ + lane: 'claude_code', + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir: join(root, 'linked-source-target'), + controllerRoot, + sourceRepositoryDir: sourceLink, + dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + }), + ).rejects.toThrow('real directory, not a symlink'); + await expect( + prepareExecutionTarget({ + lane: 'claude_code', + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir: join(sourceRoot, 'target'), + controllerRoot, + sourceRepositoryDir: sourceRoot, + dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + }), + ).rejects.toThrow('controller and target roots must be disjoint'); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it.each(['brunch', 'claude_code'] as const)( + 'materializes and controller-installs the pinned Petrinaut source for %s', + async (lane) => { + const root = await mkdtemp(join(tmpdir(), `brunch-petrinaut-operator-${lane}-`)); + const targetDir = join(root, 'target'); + const installCalls: { command: string; args: readonly string[]; cwd: string }[] = []; + const installRunner: CommandRunner = async (command, args, options) => { + installCalls.push({ command, args, cwd: options.cwd }); + return { exitCode: 0, stdout: '', stderr: '' }; + }; + try { + const prepared = await prepareExecutionTarget({ + lane, + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir, + controllerRoot, + sourceRepositoryDir: hashSourceRepository, + dependencyInstallRunner: installRunner, + }); + + expect(prepared).toMatchObject({ + lane, + caseId: 'petrinaut-optimization-v1', + sourceCommit: '5c7a2d9db5caa851c38938f4b1bac19005b0e978', + sourceTree: 'a3e08cf75e00cc9016c931f4665341506e03533e', + dependencyPreparation: { + command: 'corepack', + args: ['yarn', 'install', '--immutable', '--mode=skip-build'], + status: 'passed', + exitCode: 0, + }, + }); + expect(installCalls).toEqual([ + { + command: 'corepack', + args: ['yarn', 'install', '--immutable', '--mode=skip-build'], + cwd: targetDir, + }, + ]); + expect(await readFile(join(targetDir, 'spec.md'))).toEqual( + await readFile(join(casesRoot, 'petrinaut-optimization', 'spec.md')), + ); + expect(await readFile(join(targetDir, 'public-contract.json'))).toEqual( + await readFile(join(casesRoot, 'petrinaut-optimization', 'public-contract.json')), + ); + expect(await git(targetDir, ['rev-list', '--count', 'HEAD'])).toBe('2'); + expect(await git(targetDir, ['remote'])).toBe(''); + expect(await git(targetDir, ['for-each-ref', '--format=%(refname)'])).toBe('refs/heads/main'); + expect(await git(targetDir, ['status', '--porcelain', '--untracked-files=no'])).toBe(''); + if (lane === 'brunch') { + expect(prepared).toMatchObject({ lane: 'brunch', specId: expect.any(Number) }); + if (!('specId' in prepared)) throw new Error('pinned Brunch preparation omitted specId'); + expect(prepared.specId).toBeGreaterThan(0); + await expect( + resolvePinnedBrunchPreflight({ + workspaceDir: prepared.targetDir, + specId: prepared.specId, + }), + ).resolves.toEqual({ + action: 'newSession', + specId: prepared.specId, + }); + expect( + parseExecutionComparisonArgs([ + '--workspace', + prepared.targetDir, + '--spec-id', + String(prepared.specId), + '--solution-isolation', + 'v1', + '--forbidden-root', + controllerRoot, + ]), + ).toEqual({ + workspaceDir: prepared.targetDir, + specId: prepared.specId, + forbiddenRoots: [controllerRoot], + solutionIsolation: 'v1', + }); + const db = await openWorkspaceDb(targetDir); + const graph = queryGraph(db, prepared.specId); + expect(graph.nodes.find(({ source }) => source === 'e2e-handoff [exact-spec]')).toMatchObject({ + kind: 'requirement', + body: await readFile(join(targetDir, 'spec.md'), 'utf8'), + }); + const projection = projectExecuteGraph({ + specId: prepared.specId, + graphLsn: graph.lsn, + mode: 'brownfield', + nodes: graph.nodes, + edges: graph.edges, + }); + expect(() => assertExecuteProjectionPlanReady(projection)).not.toThrow(); + } else { + expect('specId' in prepared).toBe(false); + } + } finally { + await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); + } + }, + 120_000, + ); }); + +async function git(cwd: string, args: readonly string[]): Promise { + const result = await runCommand('git', args, { cwd }); + if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); + return result.stdout.trim(); +} diff --git a/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts b/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts new file mode 100644 index 000000000..ebb1fe330 --- /dev/null +++ b/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts @@ -0,0 +1,106 @@ +import { mkdtemp, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import { + EXECUTION_COMPARISON_SHARED_FRAMING, + resolveCompiledExecutionOracle, + retainCompiledOracleReport, +} from '../../execution-comparison-operator.js'; + +describe('execution comparison compiled oracle dispatch', () => { + it('keeps shared framing neutral across browser and backend delivery contracts', () => { + expect(EXECUTION_COMPARISON_SHARED_FRAMING).toContain('case-specific delivery'); + expect(EXECUTION_COMPARISON_SHARED_FRAMING).not.toContain('static browser'); + expect(EXECUTION_COMPARISON_SHARED_FRAMING).not.toContain('do not add a backend'); + }); + + it('selects only the three compiled implementations and rejects unknown ids before launch', () => { + const petri = resolveCompiledExecutionOracle('minimal-petri-net-editor-oracles-v2'); + const brunch = resolveCompiledExecutionOracle('brunch-host-landing-oracles-v1'); + const petrinaut = resolveCompiledExecutionOracle('petrinaut-optimization-oracles-v1'); + + expect(petri.implementationFiles).toEqual( + expect.arrayContaining([expect.stringContaining('browser-oracle.ts')]), + ); + expect(brunch.implementationFiles).toEqual( + expect.arrayContaining([expect.stringContaining('host-landing-oracle.ts')]), + ); + expect(petrinaut.implementationFiles).toEqual( + expect.arrayContaining([ + expect.stringContaining('petrinaut-optimization-oracle.ts'), + expect.stringContaining('petrinaut-optimization-oracle/browser.ts'), + expect.stringContaining('petrinaut-optimization-oracle/claims.ts'), + expect.stringContaining('petrinaut-optimization-oracle/fake-optimizer.ts'), + ]), + ); + expect(() => resolveCompiledExecutionOracle('runtime-plugin')).toThrow( + 'unknown compiled execution oracle id', + ); + }); + + it('retains a claim-linked report beside its immutable oracle-pack hash', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-oracle-dispatch-')); + const out = join(root, 'report.json'); + const report = { + schemaVersion: 1 as const, + caseId: 'brunch-host-landing-v1' as const, + oracleId: 'brunch-host-landing-oracles-v1' as const, + status: 'assertion_failed' as const, + scenario: 'final_commit_only' as const, + checks: [ + { + id: 'brownfield-full-range' as const, + claims: ['REQ2'], + status: 'failed' as const, + evidence: ['missing slice content'], + }, + ], + terminalEvidence: [], + gitEvidence: { + before: snapshot(), + preConfirm: snapshot(), + after: snapshot(), + expectedTree: 'a'.repeat(40), + actualTree: 'b'.repeat(40), + changedPaths: ['src/c.ts'], + }, + }; + const oraclePackSha256 = `sha256:${'c'.repeat(64)}`; + + await expect( + retainCompiledOracleReport({ + out, + oracleId: 'brunch-host-landing-oracles-v1', + oraclePackSha256, + report, + }), + ).resolves.toEqual({ + out, + status: 'assertion_failed', + oraclePackSha256, + oracleId: 'brunch-host-landing-oracles-v1', + }); + expect(JSON.parse(await readFile(out, 'utf8'))).toEqual(report); + await expect( + retainCompiledOracleReport({ + out, + oracleId: 'brunch-host-landing-oracles-v1', + oraclePackSha256, + report, + }), + ).rejects.toMatchObject({ code: 'EEXIST' }); + }); +}); + +function snapshot() { + return { + head: '1'.repeat(40), + tree: '2'.repeat(40), + status: '', + runMetadataSha256: '3'.repeat(64), + runMetadataBytes: '{"status":"promotion_prepared"}\n', + }; +} diff --git a/src/dev/execution-comparison/__tests__/oracle-pack.test.ts b/src/dev/execution-comparison/__tests__/oracle-pack.test.ts new file mode 100644 index 000000000..774814e1b --- /dev/null +++ b/src/dev/execution-comparison/__tests__/oracle-pack.test.ts @@ -0,0 +1,90 @@ +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +import { + assertOracleClaimCoverage, + loadControllerOracleManifest, + parseControllerOracleManifest, +} from '../oracle-pack.js'; + +const casesRoot = fileURLToPath(new URL('../../../../testing/execution-comparisons/cases/', import.meta.url)); +const requirementsPath = fileURLToPath( + new URL( + '../../../../testing/end-to-end-comparisons/cases/brunch-host-landing/controller/requirement-registry.json', + import.meta.url, + ), +); +const petrinautRequirementsPath = fileURLToPath( + new URL( + '../../../../testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json', + import.meta.url, + ), +); + +describe('compiled controller oracle manifests', () => { + it('accepts exactly three compiled variants with complete brownfield claim coverage', async () => { + const [petri, brunch, petrinaut, registry, petrinautRegistry] = await Promise.all([ + loadControllerOracleManifest(join(casesRoot, 'minimal-petri-net-editor')), + loadControllerOracleManifest(join(casesRoot, 'brunch-host-landing')), + loadControllerOracleManifest(join(casesRoot, 'petrinaut-optimization')), + readFile(requirementsPath, 'utf8').then( + (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, + ), + readFile(petrinautRequirementsPath, 'utf8').then( + (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, + ), + ]); + + expect(petri.id).toBe('minimal-petri-net-editor-oracles-v2'); + expect(brunch.id).toBe('brunch-host-landing-oracles-v1'); + expect(petrinaut.id).toBe('petrinaut-optimization-oracles-v1'); + expect(() => + assertOracleClaimCoverage( + brunch, + registry.rows.map(({ id }) => id), + ), + ).not.toThrow(); + expect(() => + assertOracleClaimCoverage( + petrinaut, + petrinautRegistry.rows.map(({ id }) => id), + ), + ).not.toThrow(); + expect(JSON.stringify(brunch)).not.toMatch(/manifestPath|command|plugin|implementationPath/u); + expect(JSON.stringify(petrinaut)).not.toMatch(/manifestPath|command|plugin|implementationPath/u); + expect(() => + parseControllerOracleManifest({ + ...petrinaut, + validityRules: [ + 'The candidate execution lane terminates at promotion_prepared before controller dependency preparation.', + 'The browser opens /optimization after preparation.', + ], + }), + ).toThrow('invalid fixed controller oracle manifest'); + }); + + it('rejects unknown ids and runtime implementation selectors', () => { + expect(() => + parseControllerOracleManifest({ + schemaVersion: 1, + id: 'unknown-oracle', + }), + ).toThrow('invalid fixed controller oracle manifest'); + expect(() => + parseControllerOracleManifest({ + schemaVersion: 1, + id: 'brunch-host-landing-oracles-v1', + publicCaseId: 'brunch-host-landing-v1', + runnerVersion: 'brunch-host-landing-v1', + referenceModelVersion: 'git-full-range-v1', + checks: [{ id: 'x', claims: ['REQ1'] }], + validityRules: ['promotion_prepared then controller landed'], + replacementRule: 'retain', + plugin: './oracle.js', + }), + ).toThrow('invalid fixed controller oracle manifest'); + }); +}); diff --git a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts new file mode 100644 index 000000000..00aa564d2 --- /dev/null +++ b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts @@ -0,0 +1,63 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { afterAll, describe, expect, it } from 'vitest'; + +import { + PETRINAUT_FOCUSED_PREPARATION, + runPetrinautOptimizationOracle, +} from '../petrinaut-optimization-oracle.js'; +import { createKnownGoodPetrinautCandidate } from '../petrinaut-optimization-oracle/fixture.js'; + +const caseDir = fileURLToPath( + new URL('../../../../testing/execution-comparisons/cases/petrinaut-optimization/', import.meta.url), +); +const roots: string[] = []; + +afterAll(async () => { + await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); +}); + +describe('standalone Petrinaut optimization browser oracle', () => { + it('proves the focused route, request, stream, failure, cancellation, origin, and accessibility leaves', async () => { + const candidateRoot = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-known-good-')); + roots.push(candidateRoot); + await createKnownGoodPetrinautCandidate(candidateRoot); + + const report = await runPetrinautOptimizationOracle({ candidateRoot, caseDir }); + + expect(report.status, report.setupFailure ?? JSON.stringify(report, null, 2)).toBe('passed'); + expect(report.preparation.map(({ id }) => id)).toEqual(PETRINAUT_FOCUSED_PREPARATION.map(({ id }) => id)); + expect(report.preparation.every(({ status }) => status === 'passed')).toBe(true); + expect(report.checks.map(({ id, status }) => ({ id, status }))).toEqual([ + { id: 'route-and-accessibility', status: 'passed' }, + { id: 'scenario-configuration', status: 'passed' }, + { id: 'request-contract', status: 'passed' }, + { id: 'progress-and-completion', status: 'passed' }, + { id: 'service-error', status: 'passed' }, + { id: 'cancel-and-abort', status: 'passed' }, + { id: 'private-origin-secrecy', status: 'passed' }, + ]); + expect(report.checks.find(({ id }) => id === 'request-contract')?.evidence).toEqual( + expect.arrayContaining([ + 'captured flat fixed/optimized bindings', + 'captured saved and custom objectives with direction', + ]), + ); + expect(report.checks.find(({ id }) => id === 'progress-and-completion')?.evidence).toEqual( + expect.arrayContaining(['progressive trial rendered', 'best-so-far rendered', 'completion rendered']), + ); + expect(report.checks.find(({ id }) => id === 'cancel-and-abort')?.evidence).toEqual( + expect.arrayContaining(['host request aborted', 'cancelled state rendered']), + ); + expect(report.checks.find(({ id }) => id === 'private-origin-secrecy')?.evidence).toEqual( + expect.arrayContaining([ + 'browser traffic remained same-origin', + 'DOM omitted private optimizer origin', + ]), + ); + expect(report.consoleErrors).toEqual([]); + }, 120_000); +}); diff --git a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts new file mode 100644 index 000000000..88f99bbbc --- /dev/null +++ b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts @@ -0,0 +1,88 @@ +import { readFile, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +import { assessPetrinautFocusedObservation } from '../petrinaut-optimization-oracle/claims.js'; + +const oracleRoot = fileURLToPath(new URL('../petrinaut-optimization-oracle/', import.meta.url)); +const oracleEntry = fileURLToPath(new URL('../petrinaut-optimization-oracle.ts', import.meta.url)); + +describe('controller-owned Petrinaut optimization oracle boundary', () => { + it('contains no historical, candidate-internal, or runtime-selected implementation dependency', async () => { + const sources = await sourceFiles(oracleRoot); + const source = ( + await Promise.all([ + readFile(oracleEntry, 'utf8'), + ...sources.map(async (path) => await readFile(path, 'utf8')), + ]) + ).join('\n'); + const imports = source + .split('\n') + .filter((line) => line.startsWith('import ') || line.startsWith('} from ')) + .join('\n'); + + expect(imports).not.toMatch( + /apps\/petrinaut|libs\/@hashintel\/petrinaut|create-optimization|optimizations-view|provider\.js/iu, + ); + expect(source).not.toMatch(/FE-1162|9051|276e17d7|historical solution|merged reference/iu); + expect(source).not.toMatch(/implementationPath|oraclePath|pluginPath|manifest\.(?:command|path)/u); + }); + + it.each([ + [ + 'missing-route', + { + check: 'route-and-accessibility', + pathname: '/processes/draft', + expectedPathname: '/optimization', + controlsReachable: true, + } as const, + 'focused route missing', + ], + [ + 'final-only', + { + check: 'progress-and-completion', + progressiveTrialCount: 0, + bestSoFarVisible: false, + completionVisible: true, + } as const, + 'progressive trials missing', + ], + [ + 'direct-private-origin', + { + check: 'private-origin-secrecy', + candidateOrigin: 'http://candidate.test', + browserRequestUrls: ['http://candidate.test/optimization', 'http://optimizer.private/optimize/all'], + domText: 'Optimizations', + privateOrigin: 'http://optimizer.private', + } as const, + 'browser contacted private origin', + ], + [ + 'UI-only-cancel', + { + check: 'cancel-and-abort', + cancelControlVisible: true, + cancelledVisible: true, + hostRequestAborted: false, + } as const, + 'host request was not aborted', + ], + ])('%s rival fails its focused claim', (_name, observation, expectedFailure) => { + expect(assessPetrinautFocusedObservation(observation)).toContain(expectedFailure); + }); +}); + +async function sourceFiles(root: string): Promise { + const files: string[] = []; + for (const entry of await readdir(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory()) files.push(...(await sourceFiles(path))); + else if (entry.isFile() && entry.name.endsWith('.ts')) files.push(path); + } + return files; +} diff --git a/src/dev/execution-comparison/__tests__/pinned-source-preparation.test.ts b/src/dev/execution-comparison/__tests__/pinned-source-preparation.test.ts new file mode 100644 index 000000000..3af888ee0 --- /dev/null +++ b/src/dev/execution-comparison/__tests__/pinned-source-preparation.test.ts @@ -0,0 +1,189 @@ +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { afterAll, describe, expect, it } from 'vitest'; + +import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; +import { + PINNED_DEPENDENCY_PREPARATION, + preparePinnedExecutionWorkspace, +} from '../../end-to-end-comparison/pinned-source-preparation.js'; + +const HASH_PARENT_COMMIT = '5c7a2d9db5caa851c38938f4b1bac19005b0e978'; +const HASH_PARENT_TREE = 'a3e08cf75e00cc9016c931f4665341506e03533e'; +const sourceRepositoryDir = fileURLToPath(new URL('../../../../../hash/', import.meta.url)); +const contractTemplatePath = fileURLToPath( + new URL( + '../../../../testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json', + import.meta.url, + ), +); +const roots: string[] = []; + +afterAll(async () => { + await Promise.all( + roots + .splice(0) + .map(async (root) => await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 })), + ); +}, 60_000); + +describe('pinned HASH source preparation', () => { + it('gives Brunch and Claude separate history-free materializations of the same full parent tree', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-pinned-source-')); + roots.push(root); + const controllerRoot = join(root, 'controller'); + const handoffRoot = join(root, 'handoff'); + await Promise.all([mkdir(controllerRoot), mkdir(handoffRoot)]); + const specification = Buffer.from('# Exact approved specification\n\nSpacing survives. \n'); + const specificationPath = join(handoffRoot, 'spec.md'); + await writeFile(specificationPath, specification); + const installs: { command: string; args: readonly string[]; cwd: string }[] = []; + const dependencyInstallRunner: CommandRunner = async (command, args, options) => { + installs.push({ command, args, cwd: options.cwd }); + return { exitCode: 0, stdout: '', stderr: '' }; + }; + + const prepared = await Promise.all( + (['brunch', 'claude_code'] as const).map( + async (lane) => + await preparePinnedExecutionWorkspace({ + lane, + sourceRepositoryDir, + sourceCommit: HASH_PARENT_COMMIT, + expectedSourceTree: HASH_PARENT_TREE, + targetDir: join(root, lane), + controllerRoot, + specificationPath, + publicContractTemplatePath: contractTemplatePath, + dependencyInstallRunner, + }), + ), + ); + + for (const target of prepared) { + expect(target).toMatchObject({ + sourceCommit: HASH_PARENT_COMMIT, + sourceTree: HASH_PARENT_TREE, + lane: expect.stringMatching(/^(?:brunch|claude_code)$/u), + }); + expect(await git(target.targetDir, ['rev-list', '--count', 'HEAD'])).toBe('2'); + expect(await git(target.targetDir, ['rev-parse', `${target.materializedCommit}^{tree}`])).not.toBe( + target.sourceTree, + ); + expect(await git(target.targetDir, ['remote'])).toBe(''); + expect(await git(target.targetDir, ['for-each-ref', '--format=%(refname)'])).toBe('refs/heads/main'); + expect(await git(target.targetDir, ['status', '--porcelain'])).toBe(''); + expect( + (await git(target.targetDir, ['diff', '--name-only', target.materializedCommit, target.baseSha])) + .split('\n') + .filter(Boolean) + .sort(), + ).toEqual(['public-contract.json', 'spec.md']); + expect(await readFile(join(target.targetDir, 'spec.md'))).toEqual(specification); + expect(JSON.parse(await readFile(join(target.targetDir, '.comparison-source.json'), 'utf8'))).toEqual({ + recipeVersion: 1, + sourceCommit: HASH_PARENT_COMMIT, + sourceTree: HASH_PARENT_TREE, + }); + expect(target.baseSha).not.toBe(target.materializedCommit); + expect(target.dependencyPreparation).toEqual({ + ...PINNED_DEPENDENCY_PREPARATION, + status: 'passed', + exitCode: 0, + }); + } + expect(prepared.map(({ targetDir }) => targetDir)).toEqual([ + join(root, 'brunch'), + join(root, 'claude_code'), + ]); + expect(installs.sort((left, right) => left.cwd.localeCompare(right.cwd))).toEqual( + prepared + .map(({ targetDir }) => ({ + command: 'corepack', + args: ['yarn', 'install', '--immutable', '--mode=skip-build'], + cwd: targetDir, + })) + .sort((left, right) => left.cwd.localeCompare(right.cwd)), + ); + }, 120_000); + + it.each([ + [ + 'install failure', + async (): Promise => async () => ({ + exitCode: 23, + stdout: '', + stderr: 'immutable install failed', + }), + 'controller dependency preparation failed (23)', + ], + [ + 'tracked mutation', + async (): Promise => async (_command, _args, options) => { + await writeFile(join(options.cwd, 'package.json'), '{"mutated":true}\n'); + return { exitCode: 0, stdout: '', stderr: '' }; + }, + 'modified tracked source', + ], + ])('fails closed and removes the target after %s', async (_name, createRunner, message) => { + const root = await mkdtemp(join(tmpdir(), 'brunch-pinned-install-failure-')); + roots.push(root); + const source = await createTinyPinnedSource(root); + const controllerRoot = join(root, 'controller'); + const handoffRoot = join(root, 'handoff'); + const targetDir = join(root, 'target'); + await Promise.all([mkdir(controllerRoot), mkdir(handoffRoot)]); + const specificationPath = join(handoffRoot, 'spec.md'); + await writeFile(specificationPath, '# Exact approved specification\n'); + + await expect( + preparePinnedExecutionWorkspace({ + lane: 'claude_code', + sourceRepositoryDir: source.repositoryDir, + sourceCommit: source.commit, + expectedSourceTree: source.tree, + targetDir, + controllerRoot, + specificationPath, + publicContractTemplatePath: contractTemplatePath, + dependencyInstallRunner: await createRunner(), + }), + ).rejects.toThrow(message); + await expect(readFile(join(targetDir, 'package.json'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); +}); + +async function git(cwd: string, args: readonly string[]): Promise { + const result = await runCommand('git', args, { cwd }); + if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); + return result.stdout.trim(); +} + +async function createTinyPinnedSource(root: string): Promise<{ + readonly repositoryDir: string; + readonly commit: string; + readonly tree: string; +}> { + const repositoryDir = join(root, 'source'); + await mkdir(repositoryDir); + await writeFile(join(repositoryDir, 'package.json'), '{"private":true}\n'); + await git(repositoryDir, ['init', '--initial-branch=main']); + await git(repositoryDir, ['add', '--all']); + await git(repositoryDir, [ + '-c', + 'user.name=Comparison Test', + '-c', + 'user.email=comparison@example.invalid', + 'commit', + '-m', + 'source', + ]); + return { + repositoryDir, + commit: await git(repositoryDir, ['rev-parse', 'HEAD']), + tree: await git(repositoryDir, ['rev-parse', 'HEAD^{tree}']), + }; +} diff --git a/src/dev/execution-comparison/accessibility-contract.ts b/src/dev/execution-comparison/accessibility-contract.ts index 6f561652b..909cf6d11 100644 --- a/src/dev/execution-comparison/accessibility-contract.ts +++ b/src/dev/execution-comparison/accessibility-contract.ts @@ -3,7 +3,7 @@ import { queryAllByRole } from '@testing-library/dom'; import { compileAccessibleNamePattern, type AccessibleNameContract, - type ExecutionCasePublicContract, + type BrowserExecutionCasePublicContract, } from './case-contract.js'; export interface AccessibilityContractAssertion { @@ -15,9 +15,9 @@ export interface AccessibilityContractAssertion { export function assertAccessibilityContract( root: HTMLElement, - contract: ExecutionCasePublicContract['accessibility'], + contract: BrowserExecutionCasePublicContract['accessibility'], requirements: { - readonly dynamic?: readonly (keyof ExecutionCasePublicContract['accessibility']['dynamic'])[]; + readonly dynamic?: readonly (keyof BrowserExecutionCasePublicContract['accessibility']['dynamic'])[]; readonly inspectorFields?: readonly string[]; } = {}, ): AccessibilityContractAssertion { diff --git a/src/dev/execution-comparison/browser-oracle.ts b/src/dev/execution-comparison/browser-oracle.ts index c907000a5..5cde3b709 100644 --- a/src/dev/execution-comparison/browser-oracle.ts +++ b/src/dev/execution-comparison/browser-oracle.ts @@ -6,8 +6,16 @@ import { chromium, type Browser, type BrowserContext, type Locator, type Page } import { runCommand } from '../../app/command-runner.js'; import { runIndependentJourneys, type IndependentJourney } from './browser-oracle/journey-runner.js'; -import { loadPublicCasePacket, type ExecutionCasePublicContract } from './case-contract.js'; -import { loadControllerOracleManifest, type ControllerOracleManifest } from './oracle-pack.js'; +import { + isBrowserExecutionCaseContract, + loadPublicCasePacket, + type BrowserExecutionCasePublicContract, +} from './case-contract.js'; +import { + isPetriControllerOracleManifest, + loadControllerOracleManifest, + type PetriControllerOracleManifest, +} from './oracle-pack.js'; type AriaRole = Parameters[0]; @@ -40,6 +48,9 @@ export async function runPetriEditorBrowserOracle(input: { }): Promise { const packet = await loadPublicCasePacket(input.caseDir); const manifest = await loadControllerOracleManifest(input.caseDir); + if (!isBrowserExecutionCaseContract(packet.contract) || !isPetriControllerOracleManifest(manifest)) { + throw new Error('Petri browser oracle received a non-browser case'); + } const commands: BrowserOracleReport['commands'][number][] = []; for (const [id, command] of [ ['test', packet.contract.delivery.test], @@ -156,8 +167,8 @@ async function openJourneyEnvironment(browser: Browser, url: string): Promise[] { const definitions = new Map, 'id' | 'claims'>>([ @@ -397,7 +408,7 @@ async function assertRoundTripAndClear(page: Page): Promise { async function assertBaseAccessibility( page: Page, - contract: ExecutionCasePublicContract['accessibility'], + contract: BrowserExecutionCasePublicContract['accessibility'], ): Promise { await requireExactlyOne(page, contract.application.role, contract.application.name); await requireExactlyOne(page, contract.canvas.role, contract.canvas.name); diff --git a/src/dev/execution-comparison/brunch-lane.ts b/src/dev/execution-comparison/brunch-lane.ts index fe8c3bee3..8a62c4db5 100644 --- a/src/dev/execution-comparison/brunch-lane.ts +++ b/src/dev/execution-comparison/brunch-lane.ts @@ -9,7 +9,9 @@ import { type SeedFixtureNode, } from '../../graph/seed-fixtures.js'; import { + isBrowserExecutionCaseContract, loadPublicCasePacket, + type BrowserExecutionCasePublicContract, type ExecutionCasePublicContract, type PublicCasePacket, } from './case-contract.js'; @@ -27,6 +29,12 @@ export interface PreparedBrunchExecutionWorkspace { readonly packet: PublicCasePacket; } +export interface SeededBrownfieldBrunchWorkspace { + readonly workspaceDir: string; + readonly specId: number; + readonly packet: PublicCasePacket; +} + export async function prepareBrunchExecutionWorkspace(input: { readonly workspaceDir: string; readonly caseDir: string; @@ -39,6 +47,9 @@ export async function prepareBrunchExecutionWorkspace(input: { } const packet = await loadPublicCasePacket(input.caseDir); + if (!isBrowserExecutionCaseContract(packet.contract)) { + throw new Error('legacy Brunch execution seed supports only the Petri browser case'); + } const specification = await readFile(join(input.caseDir, packet.contract.case.specification), 'utf8'); const executor = await openWorkspaceCommandExecutor(input.workspaceDir); const seeded = seedFixture( @@ -72,9 +83,39 @@ export async function prepareBrunchExecutionWorkspace(input: { return { workspaceDir: input.workspaceDir, specId: seeded.specId, publicDir, packet }; } +export async function seedBrownfieldBrunchExecutionWorkspace(input: { + readonly workspaceDir: string; +}): Promise { + const packet = await loadPublicCasePacket(input.workspaceDir); + if (isBrowserExecutionCaseContract(packet.contract) || packet.contract.case.mode !== 'brownfield') { + throw new Error('brownfield Brunch execution seed requires a brownfield public contract'); + } + const specification = await readFile(join(input.workspaceDir, packet.contract.case.specification), 'utf8'); + const executor = await openWorkspaceCommandExecutor(input.workspaceDir); + const seeded = seedFixture( + executor, + buildOpaqueBrownfieldExecutionSeed({ + specification, + contract: packet.contract, + }), + ); + const established = executor.establishSpecPosture({ + specId: seeded.specId, + origin: 'brownfield', + }); + if (established.status !== 'success') { + throw new Error(`failed to establish brownfield Brunch execution posture: ${established.status}`); + } + return { + workspaceDir: input.workspaceDir, + specId: seeded.specId, + packet, + }; +} + export function buildBrunchExecutionSeed(input: { readonly specification: string; - readonly contract: ExecutionCasePublicContract; + readonly contract: BrowserExecutionCasePublicContract; }): SeedFixture { const sections = parseApprovedSpecification(input.specification); const nodes: SeedFixtureNode[] = sections.map((section, index) => sectionNode(section, index + 1)); @@ -86,7 +127,7 @@ export function buildBrunchExecutionSeed(input: { export function buildOpaqueBrunchExecutionSeed(input: { readonly specification: string; - readonly contract: ExecutionCasePublicContract; + readonly contract: BrowserExecutionCasePublicContract; }): SeedFixture { return buildExecutionSeed({ nodes: [ @@ -115,9 +156,106 @@ export function buildOpaqueBrunchExecutionSeed(input: { }); } +export function buildOpaqueBrownfieldExecutionSeed(input: { + readonly specification: string; + readonly contract: ExecutionCasePublicContract; +}): SeedFixture { + if (isBrowserExecutionCaseContract(input.contract) || input.contract.case.mode !== 'brownfield') { + throw new Error('opaque brownfield seed requires a brownfield public contract'); + } + return { + spec: { + slug: input.contract.case.id, + name: `Approved ${input.contract.case.product} brownfield execution`, + kind: 'feature', + }, + nodes: [ + { + local_id: 1, + plane: 'intent', + kind: 'requirement', + title: 'Approved target-authored specification', + body: input.specification, + basis: 'explicit', + settlement: 'settled', + source: 'e2e-handoff [exact-spec]', + }, + { + local_id: 2, + plane: 'plan', + kind: 'frontier', + title: 'Deliver the approved brownfield change', + body: 'Implement the approved feature in the existing target repository.', + basis: 'explicit', + settlement: 'settled', + source: 'execution-adapter [frontier]', + }, + { + local_id: 3, + plane: 'plan', + kind: 'scope', + title: 'Implement and verify the approved brownfield change', + body: 'Execute the frozen specification as one coherent change while preserving the surrounding repository.', + basis: 'explicit', + settlement: 'settled', + source: 'execution-adapter [scope]', + }, + { + local_id: 4, + plane: 'design', + kind: 'module', + title: 'Brownfield feature implementation', + body: 'The implementation location and internal design remain the executor’s responsibility.', + basis: 'explicit', + settlement: 'settled', + source: 'execution-adapter [module]', + }, + { + local_id: 5, + plane: 'intent', + kind: 'criterion', + title: 'Approved brownfield behavior is satisfied', + body: 'The completed change satisfies the frozen approved specification in the existing repository.', + basis: 'explicit', + settlement: 'settled', + source: 'execution-adapter [criterion]', + }, + { + local_id: 6, + plane: 'oracle', + kind: 'check', + title: 'Repository-local verification passes', + body: 'Use the prepared repository’s own focused build and test surfaces to verify the approved change.', + basis: 'explicit', + settlement: 'settled', + source: 'execution-adapter [check]', + }, + { + local_id: 7, + plane: 'oracle', + kind: 'vv_method', + title: 'Prepared-target verification', + body: 'Run only repository-local verification available inside the prepared target under the execution isolation policy.', + basis: 'explicit', + settlement: 'settled', + source: 'execution-adapter [vv_method]', + }, + ], + edges: [ + edge('composition', 2, 3), + edge('realization', 1, 3), + edge('dependency', 5, 3), + edge('composition', 3, 4), + edge('dependency', 6, 3), + edge('witness', 6, 5, 'for'), + edge('realization', 7, 6), + ], + }; +} + function buildExecutionSeed(input: { readonly nodes: SeedFixtureNode[]; - readonly contract: ExecutionCasePublicContract; + readonly contract: BrowserExecutionCasePublicContract; }): SeedFixture { const nodes = [...input.nodes]; const next = nodes.length + 1; @@ -294,7 +432,7 @@ function kindForCode(code: string): { throw new Error(`unsupported approved specification code: ${code}`); } -function renderAccessibilityContract(contract: ExecutionCasePublicContract): string { +function renderAccessibilityContract(contract: BrowserExecutionCasePublicContract): string { const controls = contract.accessibility.controls .map((control) => `${control.role} "${control.name}"`) .join(', '); diff --git a/src/dev/execution-comparison/case-contract.ts b/src/dev/execution-comparison/case-contract.ts index d2074ca7b..f01cb0c23 100644 --- a/src/dev/execution-comparison/case-contract.ts +++ b/src/dev/execution-comparison/case-contract.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; import { readFile } from 'node:fs/promises'; import { basename, join } from 'node:path'; -export interface ExecutionCasePublicContract { +export interface BrowserExecutionCasePublicContract { readonly schemaVersion: 1; readonly case: { readonly id: 'minimal-petri-net-editor-v1'; @@ -39,6 +39,107 @@ export interface ExecutionCasePublicContract { readonly rules: readonly string[]; } +export interface BrunchHostLandingExecutionCasePublicContract { + readonly schemaVersion: 1; + readonly case: { + readonly id: 'brunch-host-landing-v1'; + readonly specification: 'spec.md'; + readonly specificationSha256: string; + readonly provider: 'anthropic'; + readonly model: 'claude-opus-4-8'; + readonly product: 'brunch'; + readonly mode: 'brownfield'; + readonly scope: 'single_feature'; + readonly surface: 'backend'; + readonly repository: { + readonly substrate: 'pinned_git'; + readonly parentCommit: string; + readonly parentTree: string; + }; + }; + readonly budgets: { + readonly elapsedMinutes: 90; + readonly mechanicalInterventions: 2; + readonly substantiveHumanInterventions: 0; + }; + readonly delivery: { + readonly runtimeNetwork: 'forbidden'; + readonly dependencyInstallNetwork: 'forbidden'; + }; + readonly acceptance: { + readonly publicCommand: '/brunch:land'; + readonly executionTerminal: 'promotion_prepared'; + }; + readonly accessibility: Readonly< + Record< + | 'view' + | 'tab' + | 'create' + | 'scenario' + | 'metric' + | 'direction' + | 'run' + | 'cancel' + | 'status' + | 'results', + AccessibleNameContract + > + >; + readonly rules: readonly string[]; +} + +export interface PetrinautOptimizationExecutionCasePublicContract { + readonly schemaVersion: 1; + readonly case: { + readonly id: 'petrinaut-optimization-v1'; + readonly specification: 'spec.md'; + readonly specificationSha256: string; + readonly provider: 'anthropic'; + readonly model: 'claude-opus-4-8'; + readonly product: 'petrinaut'; + readonly mode: 'brownfield'; + readonly scope: 'single_feature'; + readonly surface: 'frontend'; + readonly repository: { + readonly substrate: 'pinned_git'; + readonly parentCommit: '5c7a2d9db5caa851c38938f4b1bac19005b0e978'; + readonly parentTree: 'a3e08cf75e00cc9016c931f4665341506e03533e'; + }; + }; + readonly budgets: { + readonly elapsedMinutes: 90; + readonly mechanicalInterventions: 2; + readonly substantiveHumanInterventions: 0; + }; + readonly delivery: { + readonly runtimeNetwork: 'forbidden'; + readonly dependencyInstallNetwork: 'controller_only'; + }; + readonly acceptance: { + readonly publicRoute: '/optimization'; + readonly sameOriginApi: '/api/petrinaut-opt/optimize/all'; + readonly executionTerminal: 'promotion_prepared'; + }; + readonly rules: readonly string[]; +} + +export type ExecutionCasePublicContract = + | BrowserExecutionCasePublicContract + | BrunchHostLandingExecutionCasePublicContract + | PetrinautOptimizationExecutionCasePublicContract; + +export function isBrowserExecutionCaseContract( + value: ExecutionCasePublicContract, +): value is BrowserExecutionCasePublicContract { + return value.case.id === 'minimal-petri-net-editor-v1'; +} + +export function isPetrinautOptimizationExecutionCaseContract( + value: ExecutionCasePublicContract, +): value is PetrinautOptimizationExecutionCasePublicContract { + return value.case.id === 'petrinaut-optimization-v1'; +} + export interface CommandContract { readonly command: string; readonly args: readonly string[]; @@ -102,6 +203,16 @@ export async function loadPublicCasePacket(caseDir: string): Promise): BrowserExecutionCasePublicContract { const caseValue = requiredRecord(value, 'case'); const repository = requiredRecord(caseValue, 'repository'); const budgets = requiredRecord(value, 'budgets'); @@ -160,7 +271,145 @@ export function parsePublicCaseContract(value: unknown): ExecutionCasePublicCont invalid(); } - return value as unknown as ExecutionCasePublicContract; + return value as unknown as BrowserExecutionCasePublicContract; +} + +function parseBrunchHostLandingContract( + value: Record, +): BrunchHostLandingExecutionCasePublicContract { + const caseValue = requiredRecord(value, 'case'); + const repository = requiredRecord(caseValue, 'repository'); + const budgets = requiredRecord(value, 'budgets'); + const delivery = requiredRecord(value, 'delivery'); + const acceptance = requiredRecord(value, 'acceptance'); + if ( + !exactKeys(value, ['schemaVersion', 'case', 'budgets', 'delivery', 'acceptance', 'rules']) || + !exactKeys(caseValue, [ + 'id', + 'specification', + 'specificationSha256', + 'provider', + 'model', + 'product', + 'mode', + 'scope', + 'surface', + 'repository', + ]) || + !exactKeys(repository, ['substrate', 'parentCommit', 'parentTree']) || + !exactKeys(budgets, ['elapsedMinutes', 'mechanicalInterventions', 'substantiveHumanInterventions']) || + !exactKeys(delivery, ['runtimeNetwork', 'dependencyInstallNetwork']) || + !exactKeys(acceptance, ['publicCommand', 'executionTerminal']) || + value['schemaVersion'] !== 1 || + caseValue['id'] !== 'brunch-host-landing-v1' || + caseValue['specification'] !== 'spec.md' || + !sha256HexValue(caseValue['specificationSha256']) || + caseValue['provider'] !== 'anthropic' || + caseValue['model'] !== 'claude-opus-4-8' || + caseValue['product'] !== 'brunch' || + caseValue['mode'] !== 'brownfield' || + caseValue['scope'] !== 'single_feature' || + caseValue['surface'] !== 'backend' || + repository['substrate'] !== 'pinned_git' || + !gitObjectId(repository['parentCommit']) || + !gitObjectId(repository['parentTree']) || + budgets['elapsedMinutes'] !== 90 || + budgets['mechanicalInterventions'] !== 2 || + budgets['substantiveHumanInterventions'] !== 0 || + delivery['runtimeNetwork'] !== 'forbidden' || + delivery['dependencyInstallNetwork'] !== 'forbidden' || + acceptance['publicCommand'] !== '/brunch:land' || + acceptance['executionTerminal'] !== 'promotion_prepared' || + !nonemptyStrings(value['rules']) + ) { + invalid(); + } + return value as unknown as BrunchHostLandingExecutionCasePublicContract; +} + +function parsePetrinautOptimizationContract( + value: Record, +): PetrinautOptimizationExecutionCasePublicContract { + const caseValue = requiredRecord(value, 'case'); + const repository = requiredRecord(caseValue, 'repository'); + const budgets = requiredRecord(value, 'budgets'); + const delivery = requiredRecord(value, 'delivery'); + const acceptance = requiredRecord(value, 'acceptance'); + const accessibility = requiredRecord(value, 'accessibility'); + if ( + !exactKeys(value, [ + 'schemaVersion', + 'case', + 'budgets', + 'delivery', + 'acceptance', + 'accessibility', + 'rules', + ]) || + !exactKeys(caseValue, [ + 'id', + 'specification', + 'specificationSha256', + 'provider', + 'model', + 'product', + 'mode', + 'scope', + 'surface', + 'repository', + ]) || + !exactKeys(repository, ['substrate', 'parentCommit', 'parentTree']) || + !exactKeys(budgets, ['elapsedMinutes', 'mechanicalInterventions', 'substantiveHumanInterventions']) || + !exactKeys(delivery, ['runtimeNetwork', 'dependencyInstallNetwork']) || + !exactKeys(acceptance, ['publicRoute', 'sameOriginApi', 'executionTerminal']) || + !exactKeys(accessibility, [ + 'view', + 'tab', + 'create', + 'scenario', + 'metric', + 'direction', + 'run', + 'cancel', + 'status', + 'results', + ]) || + value['schemaVersion'] !== 1 || + caseValue['id'] !== 'petrinaut-optimization-v1' || + caseValue['specification'] !== 'spec.md' || + !sha256HexValue(caseValue['specificationSha256']) || + caseValue['provider'] !== 'anthropic' || + caseValue['model'] !== 'claude-opus-4-8' || + caseValue['product'] !== 'petrinaut' || + caseValue['mode'] !== 'brownfield' || + caseValue['scope'] !== 'single_feature' || + caseValue['surface'] !== 'frontend' || + repository['substrate'] !== 'pinned_git' || + repository['parentCommit'] !== '5c7a2d9db5caa851c38938f4b1bac19005b0e978' || + repository['parentTree'] !== 'a3e08cf75e00cc9016c931f4665341506e03533e' || + budgets['elapsedMinutes'] !== 90 || + budgets['mechanicalInterventions'] !== 2 || + budgets['substantiveHumanInterventions'] !== 0 || + delivery['runtimeNetwork'] !== 'forbidden' || + delivery['dependencyInstallNetwork'] !== 'controller_only' || + acceptance['publicRoute'] !== '/optimization' || + acceptance['sameOriginApi'] !== '/api/petrinaut-opt/optimize/all' || + acceptance['executionTerminal'] !== 'promotion_prepared' || + !accessibleName(accessibility['view'], 'heading', 'Optimizations') || + !accessibleName(accessibility['tab'], 'tab', 'Optimizations') || + !accessibleName(accessibility['create'], 'button', 'Create optimization') || + !accessibleName(accessibility['scenario'], 'combobox', 'Scenario') || + !accessibleName(accessibility['metric'], 'combobox', 'Objective metric') || + !accessibleName(accessibility['direction'], 'combobox', 'Objective direction') || + !accessibleName(accessibility['run'], 'button', 'Run optimization') || + !accessibleName(accessibility['cancel'], 'button', 'Cancel optimization') || + !accessibleName(accessibility['status'], 'status', 'Optimization status') || + !accessibleName(accessibility['results'], 'region', 'Optimization results') || + !nonemptyStrings(value['rules']) + ) { + invalid(); + } + return value as unknown as PetrinautOptimizationExecutionCasePublicContract; } function parseJson(raw: string): unknown { @@ -247,6 +496,16 @@ function sha256HexValue(value: unknown): value is string { return typeof value === 'string' && /^[a-f0-9]{64}$/u.test(value); } +function gitObjectId(value: unknown): value is string { + return typeof value === 'string' && /^[a-f0-9]{40}$/u.test(value); +} + +function exactKeys(value: Record, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(); + const wanted = [...expected].sort(); + return actual.length === wanted.length && actual.every((key, index) => key === wanted[index]); +} + function sha256Hex(value: string): string { return createHash('sha256').update(value).digest('hex'); } diff --git a/src/dev/execution-comparison/host-landing-oracle.ts b/src/dev/execution-comparison/host-landing-oracle.ts new file mode 100644 index 000000000..fce5bd17b --- /dev/null +++ b/src/dev/execution-comparison/host-landing-oracle.ts @@ -0,0 +1,8 @@ +export { evaluateHostLandingGitOutcome, snapshotGitState } from './host-landing-oracle/git-model.js'; +export { runBrunchHostLandingOracle } from './host-landing-oracle/runner.js'; +export type { + GitStateSnapshot, + HostLandingOracleCheck, + HostLandingOracleReport, + HostLandingScenario, +} from './host-landing-oracle/types.js'; diff --git a/src/dev/execution-comparison/host-landing-oracle/fixture.ts b/src/dev/execution-comparison/host-landing-oracle/fixture.ts new file mode 100644 index 000000000..dcbbb22da --- /dev/null +++ b/src/dev/execution-comparison/host-landing-oracle/fixture.ts @@ -0,0 +1,207 @@ +import { execFile, spawn } from 'node:child_process'; +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { promisify } from 'node:util'; + +import { SessionManager } from '@earendil-works/pi-coding-agent'; + +import { + HOST_LANDING_REVIEW_REF, + HOST_LANDING_RUN_ID, + type HostLandingFixture, + type HostLandingScenario, +} from './types.js'; + +const execFileAsync = promisify(execFile); +const GIT_IDENTITY = [ + '-c', + 'user.name=Brunch Oracle', + '-c', + 'user.email=brunch-oracle@invalid.local', +] as const; + +export async function createHostLandingFixture( + candidateRoot: string, + scenario: HostLandingScenario, + sessionMode: 'settled' | 'fresh', +): Promise { + const root = await mkdtemp(join(tmpdir(), 'brunch-host-landing-oracle-')); + const hostDir = join(root, 'host'); + await mkdir(hostDir); + await git(hostDir, ['init', '-q', '-b', 'main']); + await commitFile(hostDir, 'base.txt', 'base\n', 'host base'); + const hostBaseSha = await gitOutput(hostDir, ['rev-parse', 'HEAD']); + const greenfield = scenario === 'greenfield_success'; + const runRepoDir = greenfield ? join(root, 'greenfield-run') : join(root, 'run-worktree'); + if (greenfield) { + await mkdir(runRepoDir); + await git(runRepoDir, ['init', '-q', '-b', 'main']); + await git(runRepoDir, [...GIT_IDENTITY, 'commit', '--allow-empty', '-q', '-m', 'empty run base']); + } else { + await git(hostDir, ['worktree', 'add', '--quiet', '--detach', runRepoDir, hostBaseSha]); + } + const runBaseSha = await gitOutput(runRepoDir, ['rev-parse', 'HEAD']); + await commitFile(runRepoDir, 'src/a.ts', 'export const a = 1;\n', 'integrate slice a'); + await commitFile(runRepoDir, 'src/b.ts', 'export const b = 2;\n', 'integrate slice b'); + await commitFile(runRepoDir, 'src/c.ts', 'export const c = 3;\n', `promote ${HOST_LANDING_RUN_ID}`); + const completeReviewSha = await gitOutput(runRepoDir, ['rev-parse', 'HEAD']); + const canonicalExpectedTree = await gitOutput(runRepoDir, ['rev-parse', `${completeReviewSha}^{tree}`]); + let reviewSha = completeReviewSha; + if (scenario === 'final_commit_only') { + await git(runRepoDir, ['checkout', '--quiet', '--detach', runBaseSha]); + await commitFile(runRepoDir, 'src/c.ts', 'export const c = 3;\n', 'tip-only rival'); + reviewSha = await gitOutput(runRepoDir, ['rev-parse', 'HEAD']); + } + if (scenario === 'bookkeeping_retained') { + await commitFile(runRepoDir, '.brunch/leak.json', '{"leaked":true}\n', 'bookkeeping rival'); + reviewSha = await gitOutput(runRepoDir, ['rev-parse', 'HEAD']); + } + await git(runRepoDir, ['update-ref', `refs/heads/${HOST_LANDING_REVIEW_REF}`, reviewSha]); + if (scenario === 'dirty_host') await writeFile(join(hostDir, 'base.txt'), 'dirty\n'); + if (scenario === 'conflict') { + await commitFile(hostDir, 'src/a.ts', 'export const a = 999;\n', 'host conflict'); + } + const promotionPath = join(hostDir, '.brunch', 'cook', 'runs', HOST_LANDING_RUN_ID, 'promotion.json'); + const metadataPath = join(hostDir, '.brunch', 'cook', 'runs', HOST_LANDING_RUN_ID, 'run.json'); + await mkdir(dirname(metadataPath), { recursive: true }); + await writeFile( + promotionPath, + `${JSON.stringify({ + runId: HOST_LANDING_RUN_ID, + specId: '1', + promotion: { + status: 'promoted', + commitSha: reviewSha, + reviewBranch: HOST_LANDING_REVIEW_REF, + }, + })}\n`, + ); + await writeFile( + metadataPath, + `${JSON.stringify({ + runId: HOST_LANDING_RUN_ID, + specId: '1', + planPath: join(root, 'plan.json'), + status: 'promotion_prepared', + substrate: greenfield ? 'empty_dir' : 'git_worktree', + worktreeDir: runRepoDir, + runBaseSha, + promotionPath, + promotionCommitSha: reviewSha, + promotionBranch: HOST_LANDING_REVIEW_REF, + })}\n`, + ); + const sessionFile = await createSessionFixture(candidateRoot, hostDir, sessionMode); + return { + root, + hostDir, + runRepoDir, + ...(greenfield ? { targetDir: join(root, 'materialized-target') } : {}), + runBaseSha, + reviewSha, + canonicalExpectedTree, + metadataPath, + sessionFile, + }; +} + +export async function advanceHostLandingReviewRef(fixture: HostLandingFixture): Promise { + await commitFile(fixture.runRepoDir, 'late.txt', 'late\n', 'late rival commit'); + await git(fixture.runRepoDir, [ + 'update-ref', + `refs/heads/${HOST_LANDING_REVIEW_REF}`, + await gitOutput(fixture.runRepoDir, ['rev-parse', 'HEAD']), + fixture.reviewSha, + ]); +} + +async function createSessionFixture( + candidateRoot: string, + cwd: string, + mode: 'settled' | 'fresh', +): Promise { + const responses = await runCandidateRpc(candidateRoot, cwd, [ + { + jsonrpc: '2.0', + id: 1, + method: 'workspace.activate', + params: { decision: { action: 'newSpec', title: 'Host landing oracle' } }, + }, + ]); + const result = responses.find((response) => response['id'] === 1)?.['result']; + if (!record(result) || !record(result['session']) || typeof result['session']['file'] !== 'string') { + throw new Error( + `candidate public RPC did not create a settled-session fixture: ${JSON.stringify(responses)}`, + ); + } + const sessionFile = result['session']['file']; + if (mode === 'fresh') return sessionFile; + const manager = SessionManager.open(sessionFile, dirname(sessionFile), cwd); + manager.appendMessage({ + role: 'assistant', + content: [{ type: 'text', text: 'Settled controller session.' }], + api: 'brunch-oracle', + provider: 'controller', + model: 'none', + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: 'stop', + timestamp: Date.now(), + }); + return sessionFile; +} + +async function runCandidateRpc( + candidateRoot: string, + cwd: string, + requests: readonly Record[], +): Promise[]> { + const child = spawn( + process.execPath, + [join(candidateRoot, 'bin', 'brunch.js'), '--cwd', cwd, '--mode', 'rpc'], + { + cwd: candidateRoot, + env: { ...process.env, PI_OFFLINE: '1', PI_SKIP_VERSION_CHECK: '1' }, + stdio: ['pipe', 'pipe', 'pipe'], + }, + ); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8').on('data', (chunk) => (stdout += chunk)); + child.stderr.setEncoding('utf8').on('data', (chunk) => (stderr += chunk)); + child.stdin.end(`${requests.map((request) => JSON.stringify(request)).join('\n')}\n`); + const exitCode = await new Promise((resolveExit) => child.on('close', resolveExit)); + if (exitCode !== 0) throw new Error(`candidate public RPC failed: ${stderr || stdout}`); + return stdout + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +async function commitFile(cwd: string, path: string, content: string, subject: string): Promise { + const selected = join(cwd, path); + await mkdir(dirname(selected), { recursive: true }); + await writeFile(selected, content); + await git(cwd, ['add', '--', path]); + await git(cwd, [...GIT_IDENTITY, 'commit', '-q', '-m', subject]); +} + +async function git(cwd: string, args: readonly string[]): Promise { + await execFileAsync('git', [...args], { cwd }); +} + +async function gitOutput(cwd: string, args: readonly string[]): Promise { + const result = await execFileAsync('git', [...args], { cwd }); + return result.stdout.trim(); +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/dev/execution-comparison/host-landing-oracle/git-model.ts b/src/dev/execution-comparison/host-landing-oracle/git-model.ts new file mode 100644 index 000000000..c677596b7 --- /dev/null +++ b/src/dev/execution-comparison/host-landing-oracle/git-model.ts @@ -0,0 +1,155 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; +import { promisify } from 'node:util'; + +import type { + GitStateSnapshot, + HostLandingOracleCheck, + HostLandingOracleReport, + HostLandingScenario, +} from './types.js'; + +const execFileAsync = promisify(execFile); +const REQUIRED_CONTENT_PATHS = ['src/a.ts', 'src/b.ts', 'src/c.ts'] as const; + +export async function evaluateHostLandingGitOutcome(input: { + readonly scenario: HostLandingScenario; + readonly hostDir: string; + readonly targetDir?: string; + readonly metadataPath: string; + readonly canonicalExpectedTree: string; + readonly before: GitStateSnapshot; + readonly preConfirm: GitStateSnapshot; + readonly terminalEvidence?: readonly string[]; + readonly providerActivity?: boolean; +}): Promise> { + const destination = input.targetDir ?? input.hostDir; + const after = await snapshotGitState(input.hostDir, input.metadataPath); + const destinationExists = await gitSucceeds(destination, ['rev-parse', '--git-dir']); + const actualTree = destinationExists ? await gitOutput(destination, ['rev-parse', 'HEAD^{tree}']) : ''; + const changedPaths = destinationExists + ? (await gitOutput(destination, ['ls-tree', '-r', '--name-only', 'HEAD'])).split('\n').filter(Boolean) + : []; + const metadata = parseRecord(after.runMetadataBytes); + const preflightPassed = + sameSnapshot(input.before, input.preConfirm) && + input.providerActivity !== true && + (input.terminalEvidence ?? []).some( + (line) => line.includes('complete') || line.includes('Nothing changed'), + ); + const refusal = ['decline', 'dirty_host', 'conflict', 'stale_acceptance'].includes(input.scenario); + const greenfield = input.scenario === 'greenfield_success'; + const fullRangePassed = + !refusal && + !greenfield && + actualTree === input.canonicalExpectedTree && + REQUIRED_CONTENT_PATHS.every((path) => changedPaths.includes(path)) && + !changedPaths.some((path) => path === '.brunch' || path.startsWith('.brunch/')) && + metadata['status'] === 'landed'; + const materializationPassed = + greenfield && + actualTree === input.canonicalExpectedTree && + (await gitOutput(destination, ['rev-list', '--count', 'HEAD'])) === '1' && + (await gitOutput(destination, ['log', '-1', '--format=%an <%ae>'])) === 'brunch ' && + !changedPaths.some((path) => path === '.brunch' || path.startsWith('.brunch/')) && + metadata['status'] === 'landed'; + const refusalPassed = + !refusal || (sameSnapshot(input.before, after) && metadata['status'] === 'promotion_prepared'); + const checks: HostLandingOracleCheck[] = [ + { + id: 'public-tui-preflight', + claims: ['AC1', 'INV1', 'REQ1'], + status: preflightPassed ? 'passed' : 'failed', + evidence: [ + `pre-confirm snapshot ${sameSnapshot(input.before, input.preConfirm) ? 'unchanged' : 'changed'}`, + `provider activity ${input.providerActivity === true ? 'observed' : 'absent'}`, + ], + }, + { + id: 'brownfield-full-range', + claims: ['REQ2'], + status: refusal || greenfield ? 'passed' : fullRangePassed ? 'passed' : 'failed', + evidence: [ + `expected tree ${input.canonicalExpectedTree}`, + `actual tree ${actualTree || '(missing)'}`, + `tracked paths ${changedPaths.join(', ') || '(none)'}`, + ], + }, + { + id: 'greenfield-materialization', + claims: ['REQ3'], + status: greenfield ? (materializationPassed ? 'passed' : 'failed') : 'passed', + evidence: [greenfield ? `materialized tree ${actualTree || '(missing)'}` : 'not selected'], + }, + { + id: 'refusal-safety', + claims: ['REQ4'], + status: refusalPassed ? 'passed' : 'failed', + evidence: [ + refusal ? `host ${sameSnapshot(input.before, after) ? 'unchanged' : 'changed'}` : 'not selected', + ], + }, + ]; + const passed = checks.every(({ status }) => status === 'passed'); + return { + status: passed ? 'passed' : 'assertion_failed', + checks, + terminalEvidence: input.terminalEvidence ?? [], + gitEvidence: { + before: input.before, + preConfirm: input.preConfirm, + after, + expectedTree: input.canonicalExpectedTree, + actualTree, + changedPaths, + }, + }; +} + +export async function snapshotGitState(cwd: string, metadataPath: string): Promise { + const runMetadataBytes = await readFile(metadataPath, 'utf8'); + return { + head: await gitOutput(cwd, ['rev-parse', 'HEAD']), + tree: await gitOutput(cwd, ['rev-parse', 'HEAD^{tree}']), + status: await gitOutput(cwd, ['status', '--porcelain=v1', '--untracked-files=no']), + runMetadataSha256: createHash('sha256').update(runMetadataBytes).digest('hex'), + runMetadataBytes, + }; +} + +export function emptyGitStateSnapshot(): GitStateSnapshot { + return { head: '', tree: '', status: '', runMetadataSha256: '', runMetadataBytes: '' }; +} + +function sameSnapshot(left: GitStateSnapshot, right: GitStateSnapshot): boolean { + return ( + left.head === right.head && + left.tree === right.tree && + left.status === right.status && + left.runMetadataSha256 === right.runMetadataSha256 && + left.runMetadataBytes === right.runMetadataBytes + ); +} + +async function gitOutput(cwd: string, args: readonly string[]): Promise { + const result = await execFileAsync('git', [...args], { cwd }); + return result.stdout.trim(); +} + +async function gitSucceeds(cwd: string, args: readonly string[]): Promise { + try { + await execFileAsync('git', [...args], { cwd }); + return true; + } catch { + return false; + } +} + +function parseRecord(raw: string): Record { + const value = JSON.parse(raw) as unknown; + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('run metadata must be an object'); + } + return value as Record; +} diff --git a/src/dev/execution-comparison/host-landing-oracle/runner.ts b/src/dev/execution-comparison/host-landing-oracle/runner.ts new file mode 100644 index 000000000..14f4905cc --- /dev/null +++ b/src/dev/execution-comparison/host-landing-oracle/runner.ts @@ -0,0 +1,172 @@ +import { randomUUID } from 'node:crypto'; +import { readFile, rm } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +import { + removeSession, + sendKeys, + sendText, + sessionStatus, + startSession, + stopSession, + waitForScreenText, +} from '../../tui-driver.js'; +import { advanceHostLandingReviewRef, createHostLandingFixture } from './fixture.js'; +import { emptyGitStateSnapshot, evaluateHostLandingGitOutcome, snapshotGitState } from './git-model.js'; +import { + HOST_LANDING_CASE_ID, + HOST_LANDING_ORACLE_ID, + HOST_LANDING_RUN_ID, + type HostLandingFixture, + type HostLandingOracleReport, + type HostLandingScenario, +} from './types.js'; + +export async function runBrunchHostLandingOracle(input: { + readonly candidateRoot: string; + readonly scenario?: HostLandingScenario; + readonly sessionMode?: 'settled' | 'fresh'; + readonly keepFixture?: boolean; +}): Promise { + const scenario = input.scenario ?? 'brownfield_success'; + const candidateRoot = resolve(input.candidateRoot); + let fixture: HostLandingFixture | undefined; + try { + fixture = await createHostLandingFixture(candidateRoot, scenario, input.sessionMode ?? 'settled'); + return await driveCandidateTui({ candidateRoot, fixture, scenario }); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + const empty = emptyGitStateSnapshot(); + return { + schemaVersion: 1, + caseId: HOST_LANDING_CASE_ID, + oracleId: HOST_LANDING_ORACLE_ID, + status: 'setup_failed', + scenario, + checks: [], + terminalEvidence: [], + gitEvidence: { + before: empty, + preConfirm: empty, + after: empty, + expectedTree: '', + actualTree: '', + changedPaths: [], + }, + setupFailure: detail, + }; + } finally { + if (fixture && !input.keepFixture) await rm(fixture.root, { recursive: true, force: true }); + } +} + +async function driveCandidateTui(input: { + readonly candidateRoot: string; + readonly fixture: HostLandingFixture; + readonly scenario: HostLandingScenario; +}): Promise { + if (!input.fixture.sessionFile) throw new Error('controller supplied no settled session'); + const sessionBefore = await readFile(input.fixture.sessionFile, 'utf8'); + if (!sessionBefore.includes('"role":"assistant"') && !sessionBefore.includes('"role": "assistant"')) { + throw new Error('controller session fixture is not settled'); + } + const before = await snapshotGitState(input.fixture.hostDir, input.fixture.metadataPath); + const name = `host-land-${randomUUID()}`; + const command = [ + '/usr/bin/env', + 'PI_OFFLINE=1', + 'PI_SKIP_VERSION_CHECK=1', + process.execPath, + join(input.candidateRoot, 'bin', 'brunch.js'), + '--cwd', + input.fixture.hostDir, + '--no-webui', + ]; + await startSession({ name, command, cols: 120, rows: 40, cwd: input.candidateRoot }); + const terminalEvidence: string[] = []; + try { + terminalEvidence.push(...(await wait(name, 'Continue your latest spec and session'))); + sendKeys(name, ['Up', 'Up', 'Up', 'Enter']); + await wait(name, 'What does this specification own?'); + sendKeys(name, ['Down', 'Enter']); + await wait(name, 'Does this build on the existing code here?'); + sendKeys(name, ['Enter']); + await wait(name, 'Choose how Specify mode should work'); + sendKeys(name, ['Esc']); + terminalEvidence.push(...(await wait(name, 'Settled controller session.'))); + const commandText = + input.scenario === 'greenfield_success' + ? `/brunch:land ${HOST_LANDING_RUN_ID} ${input.fixture.targetDir}` + : `/brunch:land ${HOST_LANDING_RUN_ID}`; + terminalEvidence.push(`controller invoked ${commandText}`); + sendText(name, commandText); + sendKeys(name, ['Enter']); + const expectsConfirmation = !['dirty_host', 'conflict'].includes(input.scenario); + if (expectsConfirmation) { + terminalEvidence.push(...(await wait(name, 'Proceed with this host mutation?'))); + } else { + terminalEvidence.push(...(await wait(name, 'Nothing changed'))); + } + const preConfirm = await snapshotGitState(input.fixture.hostDir, input.fixture.metadataPath); + if (input.scenario === 'stale_acceptance') { + await advanceHostLandingReviewRef(input.fixture); + } + if (expectsConfirmation) { + if (input.scenario === 'decline') sendKeys(name, ['Down', 'Enter']); + else sendKeys(name, ['Enter']); + const terminal = await wait( + name, + input.scenario === 'decline' + ? 'declined; nothing changed' + : input.scenario === 'stale_acceptance' + ? 'ref_moved' + : `Landed ${HOST_LANDING_RUN_ID}`, + ); + terminalEvidence.push(...terminal); + } + const sessionAfter = await readFile(input.fixture.sessionFile, 'utf8'); + const providerActivity = + messageCount(sessionAfter) !== messageCount(sessionBefore) || + sessionAfter.includes('"customType":"brunch.kick"') || + sessionAfter.includes('"customType": "brunch.kick"'); + const evaluated = await evaluateHostLandingGitOutcome({ + scenario: input.scenario, + hostDir: input.fixture.hostDir, + ...(input.fixture.targetDir ? { targetDir: input.fixture.targetDir } : {}), + metadataPath: input.fixture.metadataPath, + canonicalExpectedTree: input.fixture.canonicalExpectedTree, + before, + preConfirm, + terminalEvidence, + providerActivity, + }); + return { + schemaVersion: 1, + caseId: HOST_LANDING_CASE_ID, + oracleId: HOST_LANDING_ORACLE_ID, + scenario: input.scenario, + ...evaluated, + }; + } finally { + await stopSession(name); + removeSession(name, { force: true }); + } +} + +async function wait(name: string, text: string): Promise { + const status = sessionStatus(name); + if (!status) throw new Error(`TUI driver session ${name} disappeared`); + const result = await waitForScreenText(status.logPath, status.cols, status.rows, text, { + timeoutMs: 30_000, + }); + if (!result.matched) { + throw new Error(`candidate TUI did not render ${JSON.stringify(text)}\n${result.screen.join('\n')}`); + } + return result.screen; +} + +function messageCount(raw: string): number { + return raw + .split('\n') + .filter((line) => line.includes('"type":"message"') || line.includes('"type": "message"')).length; +} diff --git a/src/dev/execution-comparison/host-landing-oracle/types.ts b/src/dev/execution-comparison/host-landing-oracle/types.ts new file mode 100644 index 000000000..20ff5bfa2 --- /dev/null +++ b/src/dev/execution-comparison/host-landing-oracle/types.ts @@ -0,0 +1,64 @@ +export const HOST_LANDING_CASE_ID = 'brunch-host-landing-v1' as const; +export const HOST_LANDING_ORACLE_ID = 'brunch-host-landing-oracles-v1' as const; +export const HOST_LANDING_RUN_ID = 'run-1'; +export const HOST_LANDING_REVIEW_REF = `brunch/review/${HOST_LANDING_RUN_ID}`; + +export type HostLandingScenario = + | 'brownfield_success' + | 'greenfield_success' + | 'decline' + | 'dirty_host' + | 'conflict' + | 'stale_acceptance' + | 'final_commit_only' + | 'bookkeeping_retained'; + +export interface GitStateSnapshot { + readonly head: string; + readonly tree: string; + readonly status: string; + readonly runMetadataSha256: string; + readonly runMetadataBytes: string; +} + +export interface HostLandingOracleCheck { + readonly id: + | 'public-tui-preflight' + | 'brownfield-full-range' + | 'greenfield-materialization' + | 'refusal-safety'; + readonly claims: readonly string[]; + readonly status: 'passed' | 'failed'; + readonly evidence: readonly string[]; +} + +export interface HostLandingOracleReport { + readonly schemaVersion: 1; + readonly caseId: typeof HOST_LANDING_CASE_ID; + readonly oracleId: typeof HOST_LANDING_ORACLE_ID; + readonly status: 'passed' | 'assertion_failed' | 'setup_failed'; + readonly scenario: HostLandingScenario; + readonly checks: readonly HostLandingOracleCheck[]; + readonly terminalEvidence: readonly string[]; + readonly gitEvidence: { + readonly before: GitStateSnapshot; + readonly preConfirm: GitStateSnapshot; + readonly after: GitStateSnapshot; + readonly expectedTree: string; + readonly actualTree: string; + readonly changedPaths: readonly string[]; + }; + readonly setupFailure?: string; +} + +export interface HostLandingFixture { + readonly root: string; + readonly hostDir: string; + readonly runRepoDir: string; + readonly targetDir?: string; + readonly runBaseSha: string; + readonly reviewSha: string; + readonly canonicalExpectedTree: string; + readonly metadataPath: string; + readonly sessionFile: string; +} diff --git a/src/dev/execution-comparison/operator-cli.ts b/src/dev/execution-comparison/operator-cli.ts index fd70f80cb..f3255f257 100644 --- a/src/dev/execution-comparison/operator-cli.ts +++ b/src/dev/execution-comparison/operator-cli.ts @@ -1,10 +1,19 @@ import { execFile } from 'node:child_process'; -import { lstat, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { isAbsolute, join } from 'node:path'; import { promisify } from 'node:util'; -import { prepareBrunchExecutionWorkspace } from './brunch-lane.js'; -import { loadPublicCasePacket, type PublicCasePacket } from './case-contract.js'; +import type { CommandRunner } from '../../app/command-runner.js'; +import { + preparePinnedExecutionWorkspace, + type PreparedPinnedExecutionWorkspace, +} from '../end-to-end-comparison/pinned-source-preparation.js'; +import { prepareBrunchExecutionWorkspace, seedBrownfieldBrunchExecutionWorkspace } from './brunch-lane.js'; +import { + isPetrinautOptimizationExecutionCaseContract, + loadPublicCasePacket, + type PublicCasePacket, +} from './case-contract.js'; const execFileAsync = promisify(execFile); const SAFE_CASE_ID = /^[a-z0-9][a-z0-9-]*$/u; @@ -27,15 +36,28 @@ export interface ResolvedExecutionCase extends ExecutionCaseSummary { export type PreparedExecutionTarget = | (ResolvedExecutionCase & { + readonly preparation: 'legacy_brunch'; readonly lane: 'brunch'; readonly targetDir: string; readonly specId: number; }) | (ResolvedExecutionCase & { + readonly preparation: 'empty_git'; readonly lane: 'claude_code'; readonly targetDir: string; readonly baseSha: string; - }); + }) + | (ResolvedExecutionCase & + Omit & { + readonly preparation: 'pinned_git'; + readonly lane: 'brunch'; + readonly specId: number; + }) + | (ResolvedExecutionCase & + Omit & { + readonly preparation: 'pinned_git'; + readonly lane: 'claude_code'; + }); export async function listExecutionCases(casesRoot: string): Promise { const entries = await readdir(casesRoot, { withFileTypes: true }); @@ -82,8 +104,72 @@ export async function prepareExecutionTarget(input: { readonly caseReference: string; readonly casesRoot: string; readonly targetDir: string; + readonly controllerRoot?: string; + readonly sourceRepositoryDir?: string; + readonly dependencyInstallRunner?: CommandRunner; }): Promise { const selected = await resolveExecutionCase(input.caseReference, input.casesRoot); + if (isPetrinautOptimizationExecutionCaseContract(selected.packet.contract)) { + if (input.sourceRepositoryDir === undefined) { + throw new Error('pinned execution case requires --source-repository'); + } + if (!isAbsolute(input.sourceRepositoryDir)) { + throw new Error('pinned source repository must be an absolute path'); + } + const sourceRepository = await lstat(input.sourceRepositoryDir); + if (!sourceRepository.isDirectory() || sourceRepository.isSymbolicLink()) { + throw new Error('pinned source repository must be a real directory, not a symlink'); + } + if (input.controllerRoot === undefined || !isAbsolute(input.controllerRoot)) { + throw new Error('pinned execution case requires an absolute controller root'); + } + const prepared = await preparePinnedExecutionWorkspace({ + lane: input.lane, + sourceRepositoryDir: input.sourceRepositoryDir, + sourceCommit: selected.packet.contract.case.repository.parentCommit, + expectedSourceTree: selected.packet.contract.case.repository.parentTree, + targetDir: input.targetDir, + controllerRoot: input.controllerRoot, + specificationPath: join(selected.caseDir, selected.packet.contract.case.specification), + publicContractTemplatePath: join(selected.caseDir, 'public-contract.json'), + ...(input.dependencyInstallRunner === undefined + ? {} + : { dependencyInstallRunner: input.dependencyInstallRunner }), + }); + if (prepared.lane === 'brunch') { + try { + const seeded = await seedBrownfieldBrunchExecutionWorkspace({ + workspaceDir: prepared.targetDir, + }); + const trackedStatus = await gitOutput( + ['status', '--porcelain', '--untracked-files=no'], + prepared.targetDir, + ); + if (trackedStatus.length > 0) { + throw new Error(`Brunch graph preparation modified tracked source: ${trackedStatus}`); + } + return { + ...selected, + ...prepared, + preparation: 'pinned_git', + lane: 'brunch', + specId: seeded.specId, + }; + } catch (error) { + await rm(prepared.targetDir, { recursive: true, force: true }); + throw error; + } + } + return { + ...selected, + ...prepared, + preparation: 'pinned_git', + lane: 'claude_code', + }; + } + if (input.sourceRepositoryDir !== undefined || input.dependencyInstallRunner !== undefined) { + throw new Error('--source-repository is valid only for pinned execution cases'); + } if (input.lane === 'brunch') { const prepared = await prepareBrunchExecutionWorkspace({ workspaceDir: input.targetDir, @@ -91,6 +177,7 @@ export async function prepareExecutionTarget(input: { }); return { ...selected, + preparation: 'legacy_brunch', lane: 'brunch', targetDir: input.targetDir, specId: prepared.specId, @@ -107,6 +194,7 @@ export async function prepareExecutionTarget(input: { const baseSha = await gitOutput(['rev-parse', 'HEAD'], input.targetDir); return { ...selected, + preparation: 'empty_git', lane: 'claude_code', targetDir: input.targetDir, baseSha, diff --git a/src/dev/execution-comparison/oracle-pack.ts b/src/dev/execution-comparison/oracle-pack.ts index ea6d0e59d..be7ef6273 100644 --- a/src/dev/execution-comparison/oracle-pack.ts +++ b/src/dev/execution-comparison/oracle-pack.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; import { readdir, readFile } from 'node:fs/promises'; import { basename, join, relative, sep } from 'node:path'; -export interface ControllerOracleManifest { +export interface PetriControllerOracleManifest { readonly schemaVersion: 1; readonly id: 'minimal-petri-net-editor-oracles-v2'; readonly publicCaseId: 'minimal-petri-net-editor-v1'; @@ -18,6 +18,58 @@ export interface ControllerOracleManifest { readonly catastrophicVisualGate: readonly string[]; } +export interface BrunchHostLandingControllerOracleManifest { + readonly schemaVersion: 1; + readonly id: 'brunch-host-landing-oracles-v1'; + readonly publicCaseId: 'brunch-host-landing-v1'; + readonly runnerVersion: 'brunch-host-landing-v1'; + readonly referenceModelVersion: 'git-full-range-v1'; + readonly checks: readonly { + readonly id: string; + readonly claims: readonly string[]; + }[]; + readonly validityRules: readonly string[]; + readonly replacementRule: string; +} + +export interface PetrinautOptimizationControllerOracleManifest { + readonly schemaVersion: 1; + readonly id: 'petrinaut-optimization-oracles-v1'; + readonly publicCaseId: 'petrinaut-optimization-v1'; + readonly runnerVersion: 'petrinaut-optimization-browser-v1'; + readonly fixtureVersion: 'deterministic-optimizer-v1'; + readonly checks: readonly { + readonly id: + | 'route-and-accessibility' + | 'scenario-configuration' + | 'request-contract' + | 'progress-and-completion' + | 'service-error' + | 'cancel-and-abort' + | 'private-origin-secrecy'; + readonly claims: readonly string[]; + }[]; + readonly validityRules: readonly string[]; + readonly replacementRule: string; +} + +export type ControllerOracleManifest = + | PetriControllerOracleManifest + | BrunchHostLandingControllerOracleManifest + | PetrinautOptimizationControllerOracleManifest; + +export function isPetriControllerOracleManifest( + value: ControllerOracleManifest, +): value is PetriControllerOracleManifest { + return value.id === 'minimal-petri-net-editor-oracles-v2'; +} + +export function isPetrinautOptimizationControllerOracleManifest( + value: ControllerOracleManifest, +): value is PetrinautOptimizationControllerOracleManifest { + return value.id === 'petrinaut-optimization-oracles-v1'; +} + export interface ControllerOraclePack { readonly manifest: ControllerOracleManifest; readonly files: readonly { @@ -72,6 +124,32 @@ export async function loadControllerOracleManifest(caseDir: string): Promise claims))].sort(codePointCompare); +} + +export function assertOracleClaimCoverage( + manifest: ControllerOracleManifest, + requirementClaimIds: readonly string[], +): void { + const assigned = oracleManifestClaimIds(manifest); + const required = [...new Set(requirementClaimIds)].sort(codePointCompare); + if (assigned.length !== required.length || assigned.some((claim, index) => claim !== required[index])) { + throw new Error('controller oracle manifest does not assign every requirement claim exactly'); + } +} + +function parsePetriManifest(value: Record): PetriControllerOracleManifest { const journeys = value['journeys']; if ( value['schemaVersion'] !== 1 || @@ -94,7 +172,109 @@ export function parseControllerOracleManifest(value: unknown): ControllerOracleM ) { invalid(); } - return value as unknown as ControllerOracleManifest; + return value as unknown as PetriControllerOracleManifest; +} + +function parseBrunchHostLandingManifest( + value: Record, +): BrunchHostLandingControllerOracleManifest { + const checks = value['checks']; + if ( + !exactKeys(value, [ + 'schemaVersion', + 'id', + 'publicCaseId', + 'runnerVersion', + 'referenceModelVersion', + 'checks', + 'validityRules', + 'replacementRule', + ]) || + value['schemaVersion'] !== 1 || + value['id'] !== 'brunch-host-landing-oracles-v1' || + value['publicCaseId'] !== 'brunch-host-landing-v1' || + value['runnerVersion'] !== 'brunch-host-landing-v1' || + value['referenceModelVersion'] !== 'git-full-range-v1' || + !Array.isArray(checks) || + checks.length === 0 || + !checks.every( + (check) => + record(check) && + exactKeys(check, ['id', 'claims']) && + nonempty(check['id']) && + nonemptyStrings(check['claims']), + ) || + new Set(checks.map((check) => (check as { id: string }).id)).size !== checks.length || + !nonemptyStrings(value['validityRules']) || + !(value['validityRules'] as string[]).join('\n').includes('promotion_prepared') || + !(value['validityRules'] as string[]).join('\n').includes('landed') || + !nonempty(value['replacementRule']) + ) { + invalid(); + } + return value as unknown as BrunchHostLandingControllerOracleManifest; +} + +const PETRINAUT_OPTIMIZATION_CHECKS = [ + 'route-and-accessibility', + 'scenario-configuration', + 'request-contract', + 'progress-and-completion', + 'service-error', + 'cancel-and-abort', + 'private-origin-secrecy', +] as const; + +function parsePetrinautOptimizationManifest( + value: Record, +): PetrinautOptimizationControllerOracleManifest { + const checks = value['checks']; + if ( + !exactKeys(value, [ + 'schemaVersion', + 'id', + 'publicCaseId', + 'runnerVersion', + 'fixtureVersion', + 'checks', + 'validityRules', + 'replacementRule', + ]) || + value['schemaVersion'] !== 1 || + value['id'] !== 'petrinaut-optimization-oracles-v1' || + value['publicCaseId'] !== 'petrinaut-optimization-v1' || + value['runnerVersion'] !== 'petrinaut-optimization-browser-v1' || + value['fixtureVersion'] !== 'deterministic-optimizer-v1' || + !Array.isArray(checks) || + checks.length !== PETRINAUT_OPTIMIZATION_CHECKS.length || + !checks.every( + (check, index) => + record(check) && + exactKeys(check, ['id', 'claims']) && + check['id'] === PETRINAUT_OPTIMIZATION_CHECKS[index] && + nonemptyStrings(check['claims']), + ) || + !nonemptyStrings(value['validityRules']) || + !petrinautValidityRules(value['validityRules'] as string[]) || + !nonempty(value['replacementRule']) + ) { + invalid(); + } + return value as unknown as PetrinautOptimizationControllerOracleManifest; +} + +function petrinautValidityRules(rules: readonly string[]): boolean { + const joined = rules.join('\n'); + return ( + joined.includes('Before candidate execution') && + joined.includes('compiled immutable dependency preparation') && + joined.includes('package-registry network') && + joined.includes('network-denied candidate execution lane') && + joined.includes('promotion_prepared') && + joined.includes('After lane termination') && + joined.includes('compiled focused build preparation') && + joined.includes('/optimization') + ); } async function listFiles(root: string): Promise { @@ -142,6 +322,12 @@ function record(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } +function exactKeys(value: Record, expected: readonly string[]): boolean { + const actual = Object.keys(value).sort(codePointCompare); + const wanted = [...expected].sort(codePointCompare); + return actual.length === wanted.length && actual.every((key, index) => key === wanted[index]); +} + function invalid(): never { throw new Error('invalid fixed controller oracle manifest'); } diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle.ts new file mode 100644 index 000000000..cedc9cd60 --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle.ts @@ -0,0 +1,9 @@ +export { + PETRINAUT_FOCUSED_PREPARATION, + runPetrinautOptimizationOracle, +} from './petrinaut-optimization-oracle/runner.js'; +export type { + PetrinautOptimizationCheckId, + PetrinautOptimizationOracleCheck, + PetrinautOptimizationOracleReport, +} from './petrinaut-optimization-oracle/types.js'; diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts new file mode 100644 index 000000000..7e6c8cf4f --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts @@ -0,0 +1,423 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { access } from 'node:fs/promises'; +import { createServer } from 'node:http'; + +import { chromium, type Browser, type Page } from 'playwright-core'; + +import type { PetrinautOptimizationExecutionCasePublicContract } from '../case-contract.js'; +import type { PetrinautOptimizationControllerOracleManifest } from '../oracle-pack.js'; +import { requirePetrinautFocusedObservation } from './claims.js'; +import { startDeterministicFakeOptimizer } from './fake-optimizer.js'; +import type { PetrinautOptimizationOracleCheck } from './types.js'; + +export async function runPetrinautBrowserChecks(input: { + readonly candidateRoot: string; + readonly contract: PetrinautOptimizationExecutionCasePublicContract; + readonly manifest: PetrinautOptimizationControllerOracleManifest; +}): Promise<{ + readonly checks: readonly PetrinautOptimizationOracleCheck[]; + readonly consoleErrors: readonly string[]; + readonly failedRequests: readonly string[]; +}> { + const fake = await startDeterministicFakeOptimizer(); + const port = await availablePort(); + const candidateOrigin = `http://127.0.0.1:${port}`; + const processEvidence: string[] = []; + const candidate = spawn( + 'yarn', + ['workspace', '@apps/petrinaut-website', 'dev', '--host', '127.0.0.1', '--port', String(port)], + { + cwd: input.candidateRoot, + env: { + ...process.env, + PETRINAUT_OPT_ORIGIN: fake.origin, + VITE_PETRINAUT_OPT_PROVIDER: 'service', + }, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + candidate.stdout?.on('data', (chunk: Buffer) => processEvidence.push(chunk.toString('utf8'))); + candidate.stderr?.on('data', (chunk: Buffer) => processEvidence.push(chunk.toString('utf8'))); + let browser: Browser | undefined; + const consoleErrors: string[] = []; + const failedRequests: string[] = []; + try { + await waitForRoute( + `${candidateOrigin}${input.contract.acceptance.publicRoute}`, + candidate, + processEvidence, + ); + browser = await chromium.launch({ executablePath: await resolveChromeExecutable(), headless: true }); + const definitions = checkDefinitions({ + contract: input.contract, + candidateOrigin, + fakeOrigin: fake.origin, + requests: fake.requests, + }); + const checks: PetrinautOptimizationOracleCheck[] = []; + for (const declared of input.manifest.checks) { + const definition = definitions.get(declared.id); + if (definition === undefined) throw new Error(`missing Petrinaut check implementation: ${declared.id}`); + const context = await browser.newContext(); + const page = await context.newPage(); + page.setDefaultTimeout(5_000); + page.on('console', (message) => { + if (message.type() === 'error') consoleErrors.push(message.text()); + }); + page.on('pageerror', (error) => consoleErrors.push(error.message)); + page.on('requestfailed', (request) => + declared.id === 'cancel-and-abort' && request.failure()?.errorText === 'net::ERR_ABORTED' + ? undefined + : failedRequests.push( + `${request.method()} ${request.url()}: ${request.failure()?.errorText ?? 'failed'}`, + ), + ); + try { + await page.goto(`${candidateOrigin}${input.contract.acceptance.publicRoute}`, { + waitUntil: 'networkidle', + }); + const evidence = await definition(page); + checks.push({ id: declared.id, claims: declared.claims, status: 'passed', evidence }); + } catch (error) { + checks.push({ + id: declared.id, + claims: declared.claims, + status: 'failed', + evidence: [error instanceof Error ? error.message : String(error)], + }); + } finally { + await context.close(); + } + } + return { checks, consoleErrors, failedRequests }; + } finally { + await browser?.close(); + await stopChild(candidate); + await fake.close(); + } +} + +type CheckDefinition = (page: Page) => Promise; + +function checkDefinitions(input: { + readonly contract: PetrinautOptimizationExecutionCasePublicContract; + readonly candidateOrigin: string; + readonly fakeOrigin: string; + readonly requests: { readonly body: unknown; readonly aborted: boolean }[]; +}): ReadonlyMap { + return new Map([ + [ + 'route-and-accessibility', + async (page) => { + await requireCount(page.getByRole('heading', { name: 'Optimizations', exact: true }), 1, 'view'); + await requireCount(page.getByRole('tab', { name: 'Optimizations', exact: true }), 1, 'tab'); + await page.getByRole('button', { name: 'Create optimization', exact: true }).click(); + await requireCount(page.getByRole('combobox', { name: 'Scenario', exact: true }), 1, 'scenario'); + await page.getByRole('combobox', { name: 'Scenario', exact: true }).selectOption('baseline'); + let controlsReachable = true; + for (const [role, name] of [ + ['combobox', 'Objective metric'], + ['combobox', 'Objective direction'], + ['button', 'Run optimization'], + ] as const) { + const locator = page.getByRole(role, { name, exact: true }); + await locator.focus(); + controlsReachable &&= await locator.evaluate((element) => element === document.activeElement); + } + requirePetrinautFocusedObservation({ + check: 'route-and-accessibility', + pathname: new URL(page.url()).pathname, + expectedPathname: input.contract.acceptance.publicRoute, + controlsReachable, + }); + return [ + 'public /optimization route ready', + 'required controls expose stable roles and keyboard focus', + ]; + }, + ], + [ + 'scenario-configuration', + async (page) => { + await openConfiguration(page); + const optimize = page.getByRole('checkbox', { name: 'Optimize rate', exact: true }); + await optimize.check(); + await page.getByRole('spinbutton', { name: 'rate minimum', exact: true }).fill('2'); + await page + .getByRole('combobox', { name: 'Objective direction', exact: true }) + .selectOption('minimize'); + await page.getByRole('combobox', { name: 'Scenario', exact: true }).selectOption('surge'); + assert(!(await optimize.isChecked()), 'scenario change retained optimized binding'); + assert( + (await page.getByRole('spinbutton', { name: 'rate fixed value', exact: true }).inputValue()) === + '8', + 'scenario change did not reset fixed value', + ); + assert( + (await page.getByRole('combobox', { name: 'Objective direction', exact: true }).inputValue()) === + 'maximize', + 'scenario change retained metric direction', + ); + return [ + 'configuration hidden until scenario selection', + 'scenario change reset bindings and objective', + ]; + }, + ], + [ + 'request-contract', + async (page) => { + await openConfiguration(page); + const savedRequestIndex = input.requests.length; + await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); + await status(page, /Complete/u); + const savedBody = input.requests[savedRequestIndex]?.body; + assert(record(savedBody), 'fake optimizer did not capture the saved-metric request'); + const savedObjective = savedBody['objective']; + assert( + record(savedObjective) && + savedObjective['direction'] === 'maximize' && + record(savedObjective['metric']) && + savedObjective['metric']['source'] === 'saved', + 'saved objective missing', + ); + + const customRequestIndex = input.requests.length; + await page.getByRole('textbox', { name: 'Optimization name', exact: true }).fill('request proof'); + await page.getByRole('checkbox', { name: 'Optimize rate', exact: true }).check(); + await page.getByRole('spinbutton', { name: 'rate minimum', exact: true }).fill('2'); + await page.getByRole('spinbutton', { name: 'rate maximum', exact: true }).fill('9'); + await page.getByRole('spinbutton', { name: 'demand fixed value', exact: true }).fill('7'); + await page.getByRole('combobox', { name: 'Objective metric', exact: true }).selectOption('custom'); + await page.getByRole('textbox', { name: 'Custom metric code', exact: true }).fill('return 42;'); + await page + .getByRole('combobox', { name: 'Objective direction', exact: true }) + .selectOption('minimize'); + await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); + await status(page, /Complete/u); + const body = input.requests[customRequestIndex]?.body; + assert(record(body), 'fake optimizer did not capture a JSON request'); + assert( + record(body['scenario']) && body['scenario']['id'] === 'baseline', + 'scenario missing from request', + ); + const bindings = record(body['scenario']) ? body['scenario']['parameterBindings'] : undefined; + assert(record(bindings), 'parameter bindings missing from request'); + assert( + record(bindings['rate']) && bindings['rate']['kind'] === 'optimize', + 'optimized binding missing', + ); + assert(record(bindings['demand']) && bindings['demand']['value'] === 7, 'fixed binding missing'); + const objective = body['objective']; + assert(record(objective) && objective['direction'] === 'minimize', 'objective direction missing'); + assert( + record(objective['metric']) && + objective['metric']['source'] === 'custom' && + objective['metric']['code'] === 'return 42;', + 'custom objective missing', + ); + return [ + 'captured flat fixed/optimized bindings', + 'captured saved and custom objectives with direction', + ]; + }, + ], + [ + 'progress-and-completion', + async (page) => { + await openConfiguration(page); + await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); + await page.getByText('Trial 1: 12', { exact: true }).waitFor(); + await page.getByText('Best so far: 12', { exact: true }).first().waitFor(); + await status(page, /Complete/u); + requirePetrinautFocusedObservation({ + check: 'progress-and-completion', + progressiveTrialCount: await page.getByText(/^Trial \d+: /u).count(), + bestSoFarVisible: (await page.getByText(/^Best so far: /u).count()) > 0, + completionVisible: await page + .getByRole('status', { name: 'Optimization status', exact: true }) + .filter({ hasText: /Complete/u }) + .isVisible(), + }); + return ['progressive trial rendered', 'best-so-far rendered', 'completion rendered']; + }, + ], + [ + 'service-error', + async (page) => { + await openConfiguration(page); + await page.getByRole('textbox', { name: 'Optimization name', exact: true }).fill('service failure'); + await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); + await status(page, /Error: Deterministic optimizer failure/u); + return ['service error rendered distinctly']; + }, + ], + [ + 'cancel-and-abort', + async (page) => { + const requestIndex = input.requests.length; + await openConfiguration(page); + await page.getByRole('textbox', { name: 'Optimization name', exact: true }).fill('cancel proof'); + await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); + const cancel = page.getByRole('button', { name: 'Cancel optimization', exact: true }); + await cancel.waitFor(); + await cancel.focus(); + assert( + await cancel.evaluate((element) => element === document.activeElement), + 'cancel is not focusable', + ); + await cancel.press('Enter'); + await status(page, /Cancelled/u); + await waitFor( + () => input.requests[requestIndex]?.aborted === true, + 'upstream request was not aborted', + ); + requirePetrinautFocusedObservation({ + check: 'cancel-and-abort', + cancelControlVisible: true, + hostRequestAborted: input.requests[requestIndex]?.aborted === true, + cancelledVisible: await page + .getByRole('status', { name: 'Optimization status', exact: true }) + .filter({ hasText: /Cancelled/u }) + .isVisible(), + }); + return ['cancel keyboard control rendered', 'host request aborted', 'cancelled state rendered']; + }, + ], + [ + 'private-origin-secrecy', + async (page) => { + const browserRequests: string[] = []; + page.on('request', (request) => browserRequests.push(request.url())); + await openConfiguration(page); + await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); + await status(page, /Complete/u); + requirePetrinautFocusedObservation({ + check: 'private-origin-secrecy', + candidateOrigin: input.candidateOrigin, + browserRequestUrls: browserRequests, + domText: await page.locator('body').innerText(), + privateOrigin: input.fakeOrigin, + }); + return ['browser traffic remained same-origin', 'DOM omitted private optimizer origin']; + }, + ], + ]); +} + +async function openConfiguration(page: Page): Promise { + await page.getByRole('button', { name: 'Create optimization', exact: true }).click(); + const scenario = page.getByRole('combobox', { name: 'Scenario', exact: true }); + assert( + (await page.getByRole('button', { name: 'Run optimization', exact: true }).count()) === 0, + 'configuration appeared before scenario selection', + ); + await scenario.selectOption('baseline'); +} + +async function status(page: Page, pattern: RegExp): Promise { + await page + .getByRole('status', { name: 'Optimization status', exact: true }) + .filter({ hasText: pattern }) + .waitFor(); +} + +async function requireCount( + locator: ReturnType, + expected: number, + label: string, +): Promise { + const actual = await locator.count(); + assert(actual === expected, `${label}: expected ${expected}, received ${actual}`); +} + +async function availablePort(): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('port reservation failed'); + const port = address.port; + await new Promise((resolveClose, reject) => { + server.close((error) => (error === undefined ? resolveClose() : reject(error))); + }); + return port; +} + +async function waitForRoute(url: string, child: ChildProcess, evidence: readonly string[]): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (child.exitCode !== null) { + throw new Error(`candidate process exited ${child.exitCode}: ${evidence.join('')}`); + } + try { + const response = await fetch(url); + if (response.ok) return; + } catch { + // Candidate is still starting. + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`candidate /optimization route did not become ready: ${evidence.join('')}`); +} + +async function stopChild(child: ChildProcess): Promise { + if (child.exitCode !== null) return; + signalChildTree(child, 'SIGTERM'); + await Promise.race([ + new Promise((resolve) => child.once('exit', () => resolve())), + new Promise((resolve) => setTimeout(resolve, 2_000)), + ]); + if (child.exitCode === null) signalChildTree(child, 'SIGKILL'); +} + +function signalChildTree(child: ChildProcess, signal: NodeJS.Signals): void { + if (process.platform !== 'win32' && child.pid !== undefined) { + try { + process.kill(-child.pid, signal); + return; + } catch { + // The process group may already have exited. + } + } + child.kill(signal); +} + +async function resolveChromeExecutable(): Promise { + const candidates = [ + process.env['BRUNCH_CHROME_PATH'], + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/usr/bin/google-chrome', + '/usr/bin/chromium', + ].filter((candidate): candidate is string => candidate !== undefined && candidate.length > 0); + for (const candidate of candidates) { + try { + await access(candidate); + return candidate; + } catch { + // Continue through closed platform candidates. + } + } + throw new Error('no Chrome executable found; set BRUNCH_CHROME_PATH'); +} + +async function waitFor(condition: () => boolean, message: string): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (condition()) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error(message); +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/claims.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/claims.ts new file mode 100644 index 000000000..fdb204c68 --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/claims.ts @@ -0,0 +1,62 @@ +export type PetrinautFocusedObservation = + | { + readonly check: 'route-and-accessibility'; + readonly pathname: string; + readonly expectedPathname: string; + readonly controlsReachable: boolean; + } + | { + readonly check: 'progress-and-completion'; + readonly progressiveTrialCount: number; + readonly bestSoFarVisible: boolean; + readonly completionVisible: boolean; + } + | { + readonly check: 'cancel-and-abort'; + readonly cancelControlVisible: boolean; + readonly cancelledVisible: boolean; + readonly hostRequestAborted: boolean; + } + | { + readonly check: 'private-origin-secrecy'; + readonly candidateOrigin: string; + readonly browserRequestUrls: readonly string[]; + readonly domText: string; + readonly privateOrigin: string; + }; + +export function assessPetrinautFocusedObservation( + observation: PetrinautFocusedObservation, +): readonly string[] { + switch (observation.check) { + case 'route-and-accessibility': + return [ + ...(observation.pathname === observation.expectedPathname ? [] : ['focused route missing']), + ...(observation.controlsReachable ? [] : ['required controls are not keyboard reachable']), + ]; + case 'progress-and-completion': + return [ + ...(observation.progressiveTrialCount > 0 ? [] : ['progressive trials missing']), + ...(observation.bestSoFarVisible ? [] : ['best-so-far missing']), + ...(observation.completionVisible ? [] : ['completion missing']), + ]; + case 'cancel-and-abort': + return [ + ...(observation.cancelControlVisible ? [] : ['cancel control missing']), + ...(observation.cancelledVisible ? [] : ['cancelled state missing']), + ...(observation.hostRequestAborted ? [] : ['host request was not aborted']), + ]; + case 'private-origin-secrecy': + return [ + ...(observation.browserRequestUrls.every((url) => new URL(url).origin === observation.candidateOrigin) + ? [] + : ['browser contacted private origin']), + ...(observation.domText.includes(observation.privateOrigin) ? ['DOM exposed private origin'] : []), + ]; + } +} + +export function requirePetrinautFocusedObservation(observation: PetrinautFocusedObservation): void { + const failures = assessPetrinautFocusedObservation(observation); + if (failures.length > 0) throw new Error(failures.join('; ')); +} diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts new file mode 100644 index 000000000..301e3a522 --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts @@ -0,0 +1,107 @@ +import { createServer, type ServerResponse } from 'node:http'; + +export interface CapturedOptimizationRequest { + readonly body: unknown; + aborted: boolean; +} + +export async function startDeterministicFakeOptimizer(): Promise<{ + readonly origin: string; + readonly requests: CapturedOptimizationRequest[]; + readonly close: () => Promise; +}> { + const requests: CapturedOptimizationRequest[] = []; + const openResponses = new Set(); + const server = createServer(async (request, response) => { + if (request.method !== 'POST' || request.url !== '/optimize/all') { + response.writeHead(404).end('Not found'); + return; + } + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + let body: unknown; + try { + body = JSON.parse(Buffer.concat(chunks).toString('utf8')) as unknown; + } catch { + response.writeHead(400).end('invalid JSON'); + return; + } + const captured: CapturedOptimizationRequest = { body, aborted: false }; + requests.push(captured); + openResponses.add(response); + const markAborted = () => { + if (!response.writableEnded) captured.aborted = true; + openResponses.delete(response); + }; + request.on('aborted', markAborted); + response.on('close', markAborted); + response.writeHead(200, { 'content-type': 'application/x-ndjson' }); + const name = record(body) && typeof body['name'] === 'string' ? body['name'] : ''; + writeEvent(response, { type: 'started', requestedTrials: 2 }); + if (name.includes('cancel')) return; + if (name.includes('failure')) { + writeEvent(response, { + type: 'error', + code: 'deterministic_failure', + message: 'Deterministic optimizer failure', + retryable: false, + }); + response.end(); + return; + } + const best = { trial: 0, parameters: { rate: 4 }, objective: 12 }; + writeEvent(response, { + type: 'trial', + trial: 0, + parameters: { rate: 4 }, + objective: 12, + state: 'complete', + best, + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + writeEvent(response, { + type: 'trial', + trial: 1, + parameters: { rate: 6 }, + objective: 10, + state: 'complete', + best, + }); + writeEvent(response, { + type: 'complete', + requestedTrials: 2, + completedTrials: 2, + prunedTrials: 0, + failedTrials: 0, + best, + }); + response.end(); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('fake optimizer has no TCP address'); + return { + origin: `http://127.0.0.1:${address.port}`, + requests, + close: async () => { + for (const response of openResponses) response.destroy(); + await new Promise((resolveClose, reject) => { + server.close((error) => (error === undefined ? resolveClose() : reject(error))); + }); + }, + }; +} + +function writeEvent(response: ServerResponse, event: unknown): void { + response.write(`${JSON.stringify(event)}\n`); +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/fixture.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/fixture.ts new file mode 100644 index 000000000..30052f764 --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/fixture.ts @@ -0,0 +1,263 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +const PACKAGE_SCRIPTS = [ + ['packages/ds-components', '@hashintel/ds-components', ['codegen', 'build']], + ['packages/petrinaut-core', '@hashintel/petrinaut-core', ['build']], + ['packages/optimizer-client', '@local/petrinaut-optimizer-client', ['build']], + ['packages/petrinaut', '@hashintel/petrinaut', ['build']], +] as const; + +export async function createKnownGoodPetrinautCandidate(root: string): Promise { + await mkdir(join(root, 'scripts'), { recursive: true }); + await writeFile( + join(root, 'package.json'), + `${JSON.stringify( + { + private: true, + workspaces: ['packages/*', 'apps/*'], + }, + null, + 2, + )}\n`, + ); + await writeFile(join(root, 'scripts', 'mark.mjs'), preparationMarkerSource()); + await writeFile(join(root, 'scripts', 'server.mjs'), candidateServerSource()); + for (const [path, name, scripts] of PACKAGE_SCRIPTS) { + await mkdir(join(root, path), { recursive: true }); + await writeFile( + join(root, path, 'package.json'), + `${JSON.stringify({ + name, + version: '0.0.0', + private: true, + scripts: Object.fromEntries( + scripts.map((script) => [script, `node ../../scripts/mark.mjs ${name}:${script}`]), + ), + })}\n`, + ); + } + await mkdir(join(root, 'apps', 'petrinaut-website'), { recursive: true }); + await writeFile( + join(root, 'apps', 'petrinaut-website', 'package.json'), + `${JSON.stringify({ + name: '@apps/petrinaut-website', + version: '0.0.0', + private: true, + scripts: { dev: 'node ../../scripts/server.mjs' }, + })}\n`, + ); +} + +function preparationMarkerSource(): string { + return String.raw` +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +const root = resolve(import.meta.dirname, ".."); +const path = resolve(root, ".focused-preparation.json"); +const expected = [ + "@hashintel/ds-components:codegen", + "@hashintel/ds-components:build", + "@hashintel/petrinaut-core:build", + "@local/petrinaut-optimizer-client:build", + "@hashintel/petrinaut:build", +]; +let completed = []; +try { + completed = JSON.parse(await readFile(path, "utf8")); +} catch {} +const next = process.argv[2]; +if (next !== expected[completed.length]) { + throw new Error("focused preparation ran out of order: " + next); +} +completed.push(next); +await writeFile(path, JSON.stringify(completed)); +`; +} + +function candidateServerSource(): string { + return String.raw` +import { createServer, request as httpRequest } from "node:http"; + +const args = new Map(); +for (let index = 2; index < process.argv.length; index += 2) { + args.set(process.argv[index], process.argv[index + 1]); +} +const host = args.get("--host") ?? "127.0.0.1"; +const port = Number(args.get("--port")); +const optimizerOrigin = new URL(process.env.PETRINAUT_OPT_ORIGIN); +const optimizerProvider = process.env.VITE_PETRINAUT_OPT_PROVIDER; +const requiredPreparation = 5; + +const html = String.raw${'`'} + +
+
+

Optimizations

+ + +
Idle
+
+
+ +${'`'}; + +const server = createServer(async (request, response) => { + if (request.method === "GET" && request.url === "/favicon.ico") { + response.writeHead(204).end(); + return; + } + if (request.method === "GET" && request.url === "/optimization") { + if (optimizerProvider !== "service") { + response.writeHead(404).end("Optimization provider disabled"); + return; + } + response.writeHead(200, { "content-type": "text/html; charset=utf-8" }).end(html); + return; + } + if (request.method === "POST" && request.url === "/api/petrinaut-opt/optimize/all") { + const upstream = httpRequest(new URL("/optimize/all", optimizerOrigin), { + method: "POST", + headers: { "content-type": "application/json" }, + }, (upstreamResponse) => { + response.writeHead(upstreamResponse.statusCode ?? 500, { "content-type": "application/x-ndjson" }); + upstreamResponse.pipe(response); + }); + request.pipe(upstream); + request.on("aborted", () => upstream.destroy()); + response.on("close", () => { + if (!response.writableEnded) upstream.destroy(); + }); + return; + } + response.writeHead(404).end("Not found"); +}); + +server.listen(port, host, async () => { + const { readFile } = await import("node:fs/promises"); + let prepared = []; + try { prepared = JSON.parse(await readFile(new URL("../.focused-preparation.json", import.meta.url), "utf8")); } catch {} + if (prepared.length !== requiredPreparation) { + console.error("focused preparation incomplete"); + process.exit(1); + } + console.log("PETRINAUT_READY"); +}); +`; +} diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/runner.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/runner.ts new file mode 100644 index 000000000..af0394b97 --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/runner.ts @@ -0,0 +1,109 @@ +import { runCommand } from '../../../app/command-runner.js'; +import { isPetrinautOptimizationExecutionCaseContract, loadPublicCasePacket } from '../case-contract.js'; +import { + isPetrinautOptimizationControllerOracleManifest, + loadControllerOracleManifest, +} from '../oracle-pack.js'; +import { runPetrinautBrowserChecks } from './browser.js'; +import type { PetrinautOptimizationOracleReport } from './types.js'; + +export const PETRINAUT_FOCUSED_PREPARATION = [ + { + id: 'design-system-codegen', + command: 'yarn', + args: ['workspace', '@hashintel/ds-components', 'codegen'], + }, + { + id: 'design-system-build', + command: 'yarn', + args: ['workspace', '@hashintel/ds-components', 'build'], + }, + { + id: 'petrinaut-core-build', + command: 'yarn', + args: ['workspace', '@hashintel/petrinaut-core', 'build'], + }, + { + id: 'optimizer-client-build', + command: 'yarn', + args: ['workspace', '@local/petrinaut-optimizer-client', 'build'], + }, + { + id: 'petrinaut-ui-build', + command: 'yarn', + args: ['workspace', '@hashintel/petrinaut', 'build'], + }, +] as const; + +export async function runPetrinautOptimizationOracle(input: { + readonly candidateRoot: string; + readonly caseDir: string; +}): Promise { + const [packet, manifest] = await Promise.all([ + loadPublicCasePacket(input.caseDir), + loadControllerOracleManifest(input.caseDir), + ]); + if ( + !isPetrinautOptimizationExecutionCaseContract(packet.contract) || + !isPetrinautOptimizationControllerOracleManifest(manifest) + ) { + throw new Error('Petrinaut optimization oracle received a different compiled case'); + } + + const preparation: PetrinautOptimizationOracleReport['preparation'][number][] = []; + for (const step of PETRINAUT_FOCUSED_PREPARATION) { + const result = await runCommand(step.command, step.args, { + cwd: input.candidateRoot, + timeoutMs: 10 * 60_000, + maxOutputBytes: 256 * 1024, + }); + preparation.push({ + id: step.id, + status: result.exitCode === 0 ? 'passed' : 'failed', + exitCode: result.exitCode, + }); + if (result.exitCode !== 0) { + return { + schemaVersion: 1, + caseId: packet.contract.case.id, + oracleId: manifest.id, + status: 'setup_failed', + preparation, + checks: [], + setupFailure: `${step.id} exited ${result.exitCode}`, + consoleErrors: [], + failedRequests: [], + }; + } + } + + try { + const browser = await runPetrinautBrowserChecks({ + candidateRoot: input.candidateRoot, + contract: packet.contract, + manifest, + }); + return { + schemaVersion: 1, + caseId: packet.contract.case.id, + oracleId: manifest.id, + status: browser.checks.every(({ status }) => status === 'passed') ? 'passed' : 'assertion_failed', + preparation, + checks: browser.checks, + consoleErrors: browser.consoleErrors, + failedRequests: browser.failedRequests, + }; + } catch (error) { + return { + schemaVersion: 1, + caseId: packet.contract.case.id, + oracleId: manifest.id, + status: 'setup_failed', + preparation, + checks: [], + setupFailure: error instanceof Error ? error.message : String(error), + consoleErrors: [], + failedRequests: [], + }; + } +} diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/types.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/types.ts new file mode 100644 index 000000000..6692a63cd --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/types.ts @@ -0,0 +1,31 @@ +export type PetrinautOptimizationCheckId = + | 'route-and-accessibility' + | 'scenario-configuration' + | 'request-contract' + | 'progress-and-completion' + | 'service-error' + | 'cancel-and-abort' + | 'private-origin-secrecy'; + +export interface PetrinautOptimizationOracleCheck { + readonly id: PetrinautOptimizationCheckId; + readonly claims: readonly string[]; + readonly status: 'passed' | 'failed' | 'setup_failed'; + readonly evidence: readonly string[]; +} + +export interface PetrinautOptimizationOracleReport { + readonly schemaVersion: 1; + readonly caseId: 'petrinaut-optimization-v1'; + readonly oracleId: 'petrinaut-optimization-oracles-v1'; + readonly status: 'passed' | 'setup_failed' | 'assertion_failed'; + readonly preparation: readonly { + readonly id: string; + readonly status: 'passed' | 'failed'; + readonly exitCode: number; + }[]; + readonly checks: readonly PetrinautOptimizationOracleCheck[]; + readonly setupFailure?: string; + readonly consoleErrors: readonly string[]; + readonly failedRequests: readonly string[]; +} diff --git a/src/session/__tests__/active-branch-reader-inventory.test.ts b/src/session/__tests__/active-branch-reader-inventory.test.ts index 5fd9e01c1..efe3def86 100644 --- a/src/session/__tests__/active-branch-reader-inventory.test.ts +++ b/src/session/__tests__/active-branch-reader-inventory.test.ts @@ -71,6 +71,10 @@ const NON_SESSION_JSON_PARSERS: Record = { rationale: 'parses the dev-only normalized trajectory NDJSON artifact, never Pi session JSONL', requiredOwner: 'readEvents', }, + 'src/dev/execution-comparison/host-landing-oracle/fixture.ts': { + rationale: 'parses public candidate JSON-RPC stdout responses, never Pi session JSONL', + requiredOwner: 'runCandidateRpc', + }, }; const ALL_CLASSIFICATIONS: Record = { diff --git a/testing/comparisons/missions/brunch-host-landing.md b/testing/comparisons/missions/brunch-host-landing.md new file mode 100644 index 000000000..7b71df99e --- /dev/null +++ b/testing/comparisons/missions/brunch-host-landing.md @@ -0,0 +1,5 @@ +# Safely land a prepared run + +Add a user-confirmed workflow that can land a prepared implementation run into its host repository. +The workflow must show the complete proposed change before mutation, preserve repository safety on +decline or refusal, and make successful landing durable and inspectable. diff --git a/testing/comparisons/missions/petrinaut-optimization.md b/testing/comparisons/missions/petrinaut-optimization.md new file mode 100644 index 000000000..2b527eb19 --- /dev/null +++ b/testing/comparisons/missions/petrinaut-optimization.md @@ -0,0 +1,5 @@ +# Optimize a Petrinaut scenario + +Add a focused optimization interface to Petrinaut. A user must be able to choose a scenario, configure a +flat parameter search space and one objective metric, start an optimization, follow progressive results, +and cancel a run. Keep the optimizer address private behind the website's same-origin API. diff --git a/testing/end-to-end-comparisons/cases/brunch-host-landing/controller/requirement-registry.json b/testing/end-to-end-comparisons/cases/brunch-host-landing/controller/requirement-registry.json new file mode 100644 index 000000000..6d8dd4a31 --- /dev/null +++ b/testing/end-to-end-comparisons/cases/brunch-host-landing/controller/requirement-registry.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": 1, + "caseId": "brunch-host-landing-v1", + "rows": [ + { + "id": "REQ1", + "publicConcern": "Complete-range preflight", + "origin": "public_baseline", + "publicWording": "Review the complete prepared change before host mutation.", + "controller": { + "wording": "The pre-confirmation snapshot names every commit and changed path in runBaseSha..reviewSha.", + "expectedState": "Host HEAD, tree, worktree, and run metadata are byte-identical before confirmation." + } + }, + { + "id": "REQ2", + "publicConcern": "Complete brownfield landing", + "origin": "controller_only", + "controller": { + "wording": "Every non-bookkeeping content change in the complete promoted range lands.", + "expectedState": "The host tree matches the controller Git model, .brunch is absent, and metadata is landed." + } + }, + { + "id": "REQ3", + "publicConcern": "Greenfield materialization", + "origin": "public_baseline", + "publicWording": "A greenfield run can land into a missing or empty target.", + "controller": { + "wording": "The complete review-tip tree becomes one Brunch-authored root commit.", + "expectedState": "The target has one commit and its tree equals reviewSha^{tree} without .brunch paths." + } + }, + { + "id": "REQ4", + "publicConcern": "Refusal safety", + "origin": "controller_only", + "controller": { + "wording": "Decline, dirty-host, conflict, and stale acceptance are inert.", + "expectedState": "The host snapshot is byte-identical and run status remains promotion_prepared." + } + }, + { + "id": "INV1", + "publicConcern": "User mutation authority", + "origin": "public_baseline", + "publicWording": "Landing requires confirmation after a read-only preflight.", + "controller": { + "wording": "No candidate or agent-callable surface mutates the host before confirmation.", + "expectedState": "Mutation begins only after the controller confirms the exact displayed review tip." + } + }, + { + "id": "AC1", + "publicConcern": "Public TUI address", + "origin": "public_baseline", + "publicWording": "The workflow is available as /brunch:land in the public TUI.", + "controller": { + "wording": "A resumed settled session reaches confirmation under PI_OFFLINE=1 without provider activity.", + "expectedState": "The real TUI renders the preflight and no fresh-session kick is observed." + } + } + ] +} diff --git a/testing/end-to-end-comparisons/cases/brunch-host-landing/shared-baseline.md b/testing/end-to-end-comparisons/cases/brunch-host-landing/shared-baseline.md new file mode 100644 index 000000000..4ef9bd840 --- /dev/null +++ b/testing/end-to-end-comparisons/cases/brunch-host-landing/shared-baseline.md @@ -0,0 +1,14 @@ +# Shared baseline: prepared-run landing + +The repository contains an execution system that produces immutable run metadata and a promoted review +reference. A successful execution lane stops at `promotion_prepared`; it does not mutate the host branch. + +The user-facing Brunch TUI is the public interaction surface. The landing workflow is addressed as +`/brunch:land` and requires explicit confirmation after a read-only preflight. + +Landing must work for both: + +- a brownfield run rooted at an existing host commit; and +- a greenfield run whose complete promoted tree is materialized into a missing or empty target. + +Implementation structure, module names, ports, and internal APIs are not part of the contract. diff --git a/testing/end-to-end-comparisons/cases/brunch-host-landing/study-contract.json b/testing/end-to-end-comparisons/cases/brunch-host-landing/study-contract.json new file mode 100644 index 000000000..65ae330d7 --- /dev/null +++ b/testing/end-to-end-comparisons/cases/brunch-host-landing/study-contract.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 1, + "id": "brunch-host-landing-e2e-v1", + "caseId": "brunch-host-landing-v1", + "mission": { + "path": "testing/comparisons/missions/brunch-host-landing.md", + "sha256": "sha256:1c9925dffbb4afcdb2b209558134d73bd6b0fbe4e78abe21a15d289e1308d89e" + }, + "sharedBaseline": { + "path": "testing/end-to-end-comparisons/cases/brunch-host-landing/shared-baseline.md", + "sha256": "sha256:286e5d6afbfc75a1662d5915647f1b130698b726b00fec377b92d01964baa8f3" + }, + "requirementRegistry": { + "path": "testing/end-to-end-comparisons/cases/brunch-host-landing/controller/requirement-registry.json", + "sha256": "sha256:b21214b0fa9b7dc6e5d602ebaae10fa2e0f136bab49b5e9205af9160e7b0c2e5" + }, + "executionContractTemplate": { + "path": "testing/execution-comparisons/cases/brunch-host-landing/public-contract.json", + "sha256": "sha256:48292ac96c44d91a487292ab9403dcd13adfb70848c8c8b3b9bbdd3aa2c0da18" + }, + "oracle": { + "id": "brunch-host-landing-oracles-v1", + "manifestPath": "testing/execution-comparisons/cases/brunch-host-landing/controller/oracle-manifest.json", + "manifestSha256": "sha256:ae5b9a0a4d7170471583471bd29ad619419d7190be33116d74aabfb487e07c21" + }, + "source": { + "parentCommit": "f5a423b19f76cf345d88053456870a126e451618", + "parentTree": "a5709715a07faef0b96d3e05a7b6f9f8d693dd38" + }, + "budgets": { + "elicitation": { + "qualifyingQuestions": 8, + "targetTurns": 16, + "elapsedMinutes": 45, + "mechanicalInterventions": 2 + }, + "execution": { + "elapsedMinutes": 90, + "mechanicalInterventions": 2, + "substantiveHumanInterventions": 0 + } + }, + "actorRecipes": { + "elicitation": "agent-as-user-comparison/v1", + "execution": { + "brunch": "brunch-pinned-git/v1", + "claude_code": "claude-code-pinned-git/v1" + } + }, + "specSources": ["brunch_spec", "claude_spec"], + "executors": ["brunch", "claude_code"] +} diff --git a/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json b/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json new file mode 100644 index 000000000..1847d94ca --- /dev/null +++ b/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json @@ -0,0 +1,90 @@ +{ + "schemaVersion": 1, + "caseId": "petrinaut-optimization-v1", + "rows": [ + { + "id": "REQ1", + "publicConcern": "Scenario-first configuration", + "origin": "public_baseline", + "publicWording": "Select a scenario before configuration; changing it resets scenario-specific choices.", + "controller": { + "wording": "Configuration controls are unavailable before a scenario and reset after selecting another.", + "expectedState": "No stale parameter binding or metric remains after the scenario changes." + } + }, + { + "id": "REQ2", + "publicConcern": "Flat parameter bindings", + "origin": "controller_only", + "controller": { + "wording": "One fixed binding and one optimized binding with typed bounds enter the request.", + "expectedState": "The request contains the exact flat binding map configured through public controls." + } + }, + { + "id": "REQ3", + "publicConcern": "Objective metric", + "origin": "controller_only", + "controller": { + "wording": "Saved and custom metric paths each preserve one explicit objective direction.", + "expectedState": "The request contains one metric objective and maximize or minimize direction." + } + }, + { + "id": "REQ4", + "publicConcern": "Request construction", + "origin": "controller_only", + "controller": { + "wording": "The browser sends the complete configured request only to the same-origin proxy.", + "expectedState": "Captured JSON matches the selected scenario, bindings, metric, direction, execution, and study settings." + } + }, + { + "id": "REQ5", + "publicConcern": "Progressive results", + "origin": "controller_only", + "controller": { + "wording": "Deterministic trial, best-so-far, and complete events remain visibly distinct.", + "expectedState": "Progressive trials and current best render before the completed result." + } + }, + { + "id": "REQ6", + "publicConcern": "Service failure", + "origin": "controller_only", + "controller": { + "wording": "A deterministic service failure produces an error state rather than completion.", + "expectedState": "The rendered error is distinct from completed and cancelled results." + } + }, + { + "id": "REQ7", + "publicConcern": "Cancellation", + "origin": "controller_only", + "controller": { + "wording": "Cancel aborts the active proxy request and renders cancellation.", + "expectedState": "The controller observes request abortion and the DOM reports cancelled." + } + }, + { + "id": "REQ8", + "publicConcern": "Private upstream origin", + "origin": "public_baseline", + "publicWording": "Keep the optimizer address private behind the website's same-origin API.", + "controller": { + "wording": "No browser request or rendered text contains the private fake-optimizer origin.", + "expectedState": "Only the candidate origin appears in browser traffic and DOM evidence." + } + }, + { + "id": "AC1", + "publicConcern": "Accessible public workflow", + "origin": "public_baseline", + "publicWording": "Expose stable named controls and result regions on /optimization.", + "controller": { + "wording": "Tab, form, start/cancel, and result controls have stable roles/names and accept keyboard focus.", + "expectedState": "Every required interactive control is rendered once and is keyboard reachable." + } + } + ] +} diff --git a/testing/end-to-end-comparisons/cases/petrinaut-optimization/shared-baseline.md b/testing/end-to-end-comparisons/cases/petrinaut-optimization/shared-baseline.md new file mode 100644 index 000000000..9dc9175be --- /dev/null +++ b/testing/end-to-end-comparisons/cases/petrinaut-optimization/shared-baseline.md @@ -0,0 +1,12 @@ +# Shared baseline: Petrinaut optimization + +The repository contains Petrinaut's editor, scenarios, parameter schemas, saved and custom metrics, and a +pre-existing optimizer API/client protocol. These capabilities are available to inspect and reuse; adding +the optimization user interface is the implementation task. + +The public mechanical address is the Petrinaut website's `/optimization` route. Its browser calls a +same-origin API path which may proxy to a separately configured optimizer. The upstream optimizer origin +must not appear in browser requests or rendered content. + +The execution scope is one frontend feature in the full repository. It does not require the authenticated +HASH application shell, an extracted Petrinaut repository, or a new optimizer backend. diff --git a/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json b/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json new file mode 100644 index 000000000..bd11a20f1 --- /dev/null +++ b/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 1, + "id": "petrinaut-optimization-e2e-v1", + "caseId": "petrinaut-optimization-v1", + "mission": { + "path": "testing/comparisons/missions/petrinaut-optimization.md", + "sha256": "sha256:89daf3507df732405bad528e507328aede01b5125383c499aef8b97fff8b92b0" + }, + "sharedBaseline": { + "path": "testing/end-to-end-comparisons/cases/petrinaut-optimization/shared-baseline.md", + "sha256": "sha256:977947acf91d236397174a247436b77db0164f5b0deaf63ce78d6d8f12a46b9e" + }, + "requirementRegistry": { + "path": "testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json", + "sha256": "sha256:395f28cad01954be1b68be799f659a2292f28dc267c26eb50ebb68e9fce8ed05" + }, + "executionContractTemplate": { + "path": "testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json", + "sha256": "sha256:1d8e9190749bd4107831aeaf50da35c73fec6196a2f902a8a57eaa64ba47d864" + }, + "oracle": { + "id": "petrinaut-optimization-oracles-v1", + "manifestPath": "testing/execution-comparisons/cases/petrinaut-optimization/controller/oracle-manifest.json", + "manifestSha256": "sha256:7f5050a0bae848023b01b68b22993934fe3bac691a90748659d894bfc76dd939" + }, + "source": { + "parentCommit": "5c7a2d9db5caa851c38938f4b1bac19005b0e978", + "parentTree": "a3e08cf75e00cc9016c931f4665341506e03533e" + }, + "budgets": { + "elicitation": { + "qualifyingQuestions": 8, + "targetTurns": 16, + "elapsedMinutes": 45, + "mechanicalInterventions": 2 + }, + "execution": { + "elapsedMinutes": 90, + "mechanicalInterventions": 2, + "substantiveHumanInterventions": 0 + } + }, + "actorRecipes": { + "elicitation": "agent-as-user-comparison/v1", + "execution": { + "brunch": "brunch-pinned-git/v1", + "claude_code": "claude-code-pinned-git/v1" + } + }, + "specSources": ["brunch_spec", "claude_spec"], + "executors": ["brunch", "claude_code"] +} diff --git a/testing/execution-comparisons/cases/brunch-host-landing/controller/oracle-manifest.json b/testing/execution-comparisons/cases/brunch-host-landing/controller/oracle-manifest.json new file mode 100644 index 000000000..63ebd0188 --- /dev/null +++ b/testing/execution-comparisons/cases/brunch-host-landing/controller/oracle-manifest.json @@ -0,0 +1,32 @@ +{ + "schemaVersion": 1, + "id": "brunch-host-landing-oracles-v1", + "publicCaseId": "brunch-host-landing-v1", + "runnerVersion": "brunch-host-landing-v1", + "referenceModelVersion": "git-full-range-v1", + "checks": [ + { + "id": "public-tui-preflight", + "claims": ["AC1", "INV1", "REQ1"] + }, + { + "id": "brownfield-full-range", + "claims": ["REQ2"] + }, + { + "id": "greenfield-materialization", + "claims": ["REQ3"] + }, + { + "id": "refusal-safety", + "claims": ["REQ4"] + } + ], + "validityRules": [ + "The candidate lane terminates at promotion_prepared before the controller creates any landing repository.", + "The controller resumes a supplied settled session under PI_OFFLINE=1; a fresh session or provider turn is setup_failed.", + "The controller invokes only the built candidate public TUI and inspects terminal, filesystem, Git, and retained run metadata.", + "Only disposable controller-created repositories may reach landed." + ], + "replacementRule": "Retain every report. Replace only harness setup invalidity under the same frozen packets; never replace an assertion failure." +} diff --git a/testing/execution-comparisons/cases/brunch-host-landing/public-contract.json b/testing/execution-comparisons/cases/brunch-host-landing/public-contract.json new file mode 100644 index 000000000..b8bebad7e --- /dev/null +++ b/testing/execution-comparisons/cases/brunch-host-landing/public-contract.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 1, + "case": { + "id": "brunch-host-landing-v1", + "specification": "spec.md", + "specificationSha256": "80dc03a071ee5bd03740afcecd90387e6d5e01c7d163d9ddf5d870599be181f3", + "provider": "anthropic", + "model": "claude-opus-4-8", + "product": "brunch", + "mode": "brownfield", + "scope": "single_feature", + "surface": "backend", + "repository": { + "substrate": "pinned_git", + "parentCommit": "f5a423b19f76cf345d88053456870a126e451618", + "parentTree": "a5709715a07faef0b96d3e05a7b6f9f8d693dd38" + } + }, + "budgets": { + "elapsedMinutes": 90, + "mechanicalInterventions": 2, + "substantiveHumanInterventions": 0 + }, + "delivery": { + "runtimeNetwork": "forbidden", + "dependencyInstallNetwork": "forbidden" + }, + "acceptance": { + "publicCommand": "/brunch:land", + "executionTerminal": "promotion_prepared" + }, + "rules": [ + "Work only in the target repository.", + "Do not inspect controller paths or seek hidden comparison material.", + "Do not use network access or external historical references.", + "Stop after the run reaches promotion_prepared; do not land it." + ] +} diff --git a/testing/execution-comparisons/cases/brunch-host-landing/spec.md b/testing/execution-comparisons/cases/brunch-host-landing/spec.md new file mode 100644 index 000000000..e5366912c --- /dev/null +++ b/testing/execution-comparisons/cases/brunch-host-landing/spec.md @@ -0,0 +1,35 @@ +# Host landing + +### G1 Land a prepared run safely + +Give the user one explicit TUI workflow for reviewing and landing a prepared run without allowing the +execution lane or an agent-callable tool to mutate the host. + +### REQ1 Complete-range review + +Before confirmation, show the full commit and path range from the run base through the promoted review +tip. Inspection must leave the host branch, tree, worktree, and run metadata unchanged. + +### REQ2 Complete brownfield landing + +After confirmation, land every content change in the promoted range into the active repository. Preserve +the review reference, exclude `.brunch` bookkeeping from tracked output, and record the run as landed. + +### REQ3 Greenfield materialization + +For a greenfield run, materialize the complete promoted tip tree into a missing or empty target as one +clean Brunch-authored initial commit. + +### REQ4 Refusal safety + +Decline, dirty-host, conflict, and stale-acceptance outcomes leave the host unchanged and do not record +the run as landed. + +### INV1 User authority + +Host mutation occurs only after an interactive confirmation bound to the exact promoted commit shown by +preflight. + +### AC1 Public TUI behavior + +The workflow is exercised through `/brunch:land` in the real public TUI without a provider turn. diff --git a/testing/execution-comparisons/cases/petrinaut-optimization/controller/oracle-manifest.json b/testing/execution-comparisons/cases/petrinaut-optimization/controller/oracle-manifest.json new file mode 100644 index 000000000..6181698be --- /dev/null +++ b/testing/execution-comparisons/cases/petrinaut-optimization/controller/oracle-manifest.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "id": "petrinaut-optimization-oracles-v1", + "publicCaseId": "petrinaut-optimization-v1", + "runnerVersion": "petrinaut-optimization-browser-v1", + "fixtureVersion": "deterministic-optimizer-v1", + "checks": [ + { + "id": "route-and-accessibility", + "claims": ["AC1"] + }, + { + "id": "scenario-configuration", + "claims": ["REQ1"] + }, + { + "id": "request-contract", + "claims": ["REQ2", "REQ3", "REQ4"] + }, + { + "id": "progress-and-completion", + "claims": ["REQ5"] + }, + { + "id": "service-error", + "claims": ["REQ6"] + }, + { + "id": "cancel-and-abort", + "claims": ["REQ7"] + }, + { + "id": "private-origin-secrecy", + "claims": ["REQ8"] + } + ], + "validityRules": [ + "Before candidate execution, the controller materializes the pinned source and completes the compiled immutable dependency preparation; only that declared step may use package-registry network.", + "The network-denied candidate execution lane terminates at promotion_prepared before browser launch.", + "After lane termination, the controller runs only the compiled focused build preparation and starts the candidate Petrinaut website.", + "The browser opens /optimization and sends optimization traffic only through the candidate same-origin proxy.", + "Controller fixtures and expected requests are created after lane termination in roots disjoint from the candidate." + ], + "replacementRule": "Retain every report. Replace only setup invalidity under the same frozen packets; never replace an assertion failure." +} diff --git a/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json b/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json new file mode 100644 index 000000000..387b40788 --- /dev/null +++ b/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json @@ -0,0 +1,51 @@ +{ + "schemaVersion": 1, + "case": { + "id": "petrinaut-optimization-v1", + "specification": "spec.md", + "specificationSha256": "a949cdeeca3093c51f36d42169d582141280f3561c9770de2b6665e28fac3c15", + "provider": "anthropic", + "model": "claude-opus-4-8", + "product": "petrinaut", + "mode": "brownfield", + "scope": "single_feature", + "surface": "frontend", + "repository": { + "substrate": "pinned_git", + "parentCommit": "5c7a2d9db5caa851c38938f4b1bac19005b0e978", + "parentTree": "a3e08cf75e00cc9016c931f4665341506e03533e" + } + }, + "budgets": { + "elapsedMinutes": 90, + "mechanicalInterventions": 2, + "substantiveHumanInterventions": 0 + }, + "delivery": { + "runtimeNetwork": "forbidden", + "dependencyInstallNetwork": "controller_only" + }, + "acceptance": { + "publicRoute": "/optimization", + "sameOriginApi": "/api/petrinaut-opt/optimize/all", + "executionTerminal": "promotion_prepared" + }, + "accessibility": { + "view": { "role": "heading", "name": "Optimizations" }, + "tab": { "role": "tab", "name": "Optimizations" }, + "create": { "role": "button", "name": "Create optimization" }, + "scenario": { "role": "combobox", "name": "Scenario" }, + "metric": { "role": "combobox", "name": "Objective metric" }, + "direction": { "role": "combobox", "name": "Objective direction" }, + "run": { "role": "button", "name": "Run optimization" }, + "cancel": { "role": "button", "name": "Cancel optimization" }, + "status": { "role": "status", "name": "Optimization status" }, + "results": { "role": "region", "name": "Optimization results" } + }, + "rules": [ + "Work only in the target repository.", + "Do not inspect controller paths or seek hidden comparison material.", + "Do not use network access or external references during implementation or verification.", + "Preserve the full repository and stop at promotion_prepared without landing." + ] +} diff --git a/testing/execution-comparisons/cases/petrinaut-optimization/spec.md b/testing/execution-comparisons/cases/petrinaut-optimization/spec.md new file mode 100644 index 000000000..6233b5e59 --- /dev/null +++ b/testing/execution-comparisons/cases/petrinaut-optimization/spec.md @@ -0,0 +1,50 @@ +# Petrinaut optimization + +### G1 Configure and run an optimization + +Add a capability-present Optimizations view to Petrinaut's focused website route. The workflow must make +scenario selection explicit before optimization configuration and expose run progress without revealing +the private optimizer service. + +### REQ1 Scenario-first configuration + +Require the user to select a scenario before configuring an optimization. Changing the scenario resets +parameter and metric configuration that belongs to the previous scenario. + +### REQ2 Flat parameter bindings + +Show every selected scenario parameter in one flat configuration. Each parameter can be fixed at one +valid typed value or optimized over a valid typed domain. + +### REQ3 Objective metric + +Allow exactly one saved model metric or run-local custom metric as the objective, with an explicit +maximize or minimize direction. + +### REQ4 Request construction + +Starting a run sends the chosen scenario, fixed and optimized parameter bindings, metric objective and +direction, and execution/study settings through the website's same-origin optimization API. + +### REQ5 Progressive results + +Render streamed trials, best-so-far values, and successful completion as distinct progressive states. + +### REQ6 Service failure + +Render an optimizer service failure distinctly from successful completion and user cancellation. + +### REQ7 Cancellation + +Allow an active run to be cancelled. Cancellation aborts the browser's in-flight same-origin request and +renders a cancelled state. + +### REQ8 Private upstream origin + +Browser traffic and rendered content must not expose or contact the private upstream optimizer origin. + +### AC1 Accessible public workflow + +The `/optimization` route exposes a named Optimizations view. Its tab, scenario selector, parameter and +metric form controls, start/cancel controls, and result regions use stable accessible roles and names and +are keyboard reachable. From 2b0033935cf37f6d4fedfb48ee936f79196d0a6e Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 11:51:31 +0200 Subject: [PATCH 02/15] FE-1241: Admit pinned historical replay targets Make strict two-commit admission the sole lane-ready preparation path and harden its dependency and network contracts. Co-authored-by: Cursor --- memory/PLAN.md | 14 +- memory/SPEC.md | 3 +- ...son-cases--admission-contract-hardening.md | 100 +++ ...ses--historical-replay-target-admission.md | 173 ++++ ...ses--petrinaut-optimization-case-oracle.md | 177 ++++ src/dev/TOPOLOGY.md | 6 +- src/dev/end-to-end-comparison.ts | 3 - .../__tests__/case-profile.test.ts | 32 +- .../__tests__/solution-isolation.test.ts | 43 +- .../end-to-end-comparison/brunch-adapter.ts | 42 +- .../end-to-end-comparison/claude-adapter.ts | 152 ++-- .../pinned-source-preparation.ts | 156 ---- .../solution-isolation.ts | 95 ++- .../historical-replay-target.test.ts | 756 ++++++++++++++++++ .../__tests__/operator-cli.test.ts | 212 ++--- .../pinned-source-preparation.test.ts | 189 ----- src/dev/execution-comparison/case-contract.ts | 7 + .../historical-replay-target.ts | 367 +++++++++ src/dev/execution-comparison/operator-cli.ts | 116 +-- 19 files changed, 1941 insertions(+), 702 deletions(-) create mode 100644 memory/cards/brownfield-comparison-cases--admission-contract-hardening.md create mode 100644 memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md create mode 100644 memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md delete mode 100644 src/dev/end-to-end-comparison/pinned-source-preparation.ts create mode 100644 src/dev/execution-comparison/__tests__/historical-replay-target.test.ts delete mode 100644 src/dev/execution-comparison/__tests__/pinned-source-preparation.test.ts create mode 100644 src/dev/execution-comparison/historical-replay-target.ts diff --git a/memory/PLAN.md b/memory/PLAN.md index 052f151cd..260de00a7 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -117,9 +117,13 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand - `execution-comparison-tracer` ([FE-1230](https://linear.app/hash/issue/FE-1230/greenfield-execution-comparison-tracer)) — **active on `ka/fe-1230-independent-oracle-journeys`:** the frozen case, immutable-attempt contract, retained Brunch/Claude v1 pair, and versioned independent browser-oracle journeys are built. Next: restore the exact retained output paths, replay `petri-editor-browser-v2` unchanged against both, then review and promote the bounded evidence; no mutants, judging campaign, or repetitions in this frontier. - `end-to-end-comparison-tracer` ([FE-1239](https://linear.app/hash/issue/FE-1239/trace-elicitation-through-execution)) — **complete on `ka/fe-1239-end-to-end-comparison-tracer`, stacked on FE-1230:** two rigorous Petri-editor elicitation outputs crossed exact immutable handoffs into the staged 2×2 Brunch/Claude matrix; all four valid failed cells, unchanged-oracle evidence, and the validity-first requirement ledger are promoted. Definition and retained witness below. - `executor-slice-admission-parity` ([FE-1240](https://linear.app/hash/issue/FE-1240/prevent-invalid-scoped-slices-from-reaching-execution)) — **complete on `ka/fe-1240-slice-admission-parity`, restacked on `next` after FE-1239 landed:** incomplete scoped-slice worker context is rejected during deterministic plan admission, exact findings enter bounded repair before `slice_execute`, and the execution boundary remains fail-closed. Definition below. +<<<<<<< HEAD - `executor-plan-coherence` ([FE-1250](https://linear.app/hash/issue/FE-1250/build-coherent-execution-plans)) — **implementation complete on `ka/fe-1250-coherent-execution-plans`, based on FE-1240 via `next`:** frontier-verified multi-slice epics now require one ordinary terminal member over every sibling, planner conduct establishes shared foundations, and workers preserve cumulative public contracts for the canonical harness. Fast verification passes; next is the separately authorized unchanged Petri comparison rerun. No new browser gate, durable plan kind, or executor lifecycle phase. Definition below. - `comparison-publication-workflow` ([FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports)) — **active on `ka/fe-1251-comparison-publication`, stacked on FE-1250:** capture immutable controller and release provenance before comparison lanes start, then explicitly publish retained validity-first reports into the canonical Notion database through a guarded idempotent skill. Definition below. - `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases)) — **active on `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240:** A49-L's versioned fail-closed historical-replay admission path and the frozen Brunch backend host-landing profile/controller oracle are complete. Preserve the minimal Petri-net editor as the only greenfield case, then replay Petrinaut optimization UI PR #9051 in the full `hashintel/hash` repository. A50-L is invalidated and D136-L selects the focused `/optimization` route for the common hard gate. Next: build the revised Petrinaut profile/oracle before any historical provider run. +======= +- `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases)) — **active on `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240:** both frozen brownfield profiles/oracles and D137-L's admitted, lane-ready two-commit historical replay path are built and contract-hardened. Final admission now follows D136-L's tracked-clean dependency boundary, pinned case recipes are exhaustive with no default, and the production deep operation has a reachable-network rival. Preserve the minimal Petri-net editor as the only greenfield case. Next: scope and run the fresh historical provider preflight. +>>>>>>> 5e1ed22b (FE-1241: Admit pinned historical replay targets) - **Carved from FE-1167 (2026-07-13):** the Execute-mode evidence sub-list — Execute entry beats on thin vs rich seeds (assessment honesty: Ask on thin, Proceed on rich) and the FE-1107/KA residue (close-or-narrow, demo/walkthrough session via `TESTING_PLAN.md`, post-KA plan pass). The former sticky-posture question is no longer KA residue: FE-1187 `remediation-4` owns the persistent Specify elicitation-style audit/SPEC revision, and its Continue lexical audit owns the old `continue` ambiguity. Full context in the archived FE-1167 definition (`docs/archive/PLAN_HISTORY.md`). - `planning-process-model` — **moved to the KA stream 2026-07-13; reshaped by D126-L**: the durable scope handoff is settled, so this item now owns only plan projection and epistemic-horizon questions beyond committed scopes. Definition below. - **[1.x data-model handoff owed by FE-1187](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md):** a concise colleague-facing note must distinguish current canon from directional vocabulary before the consolidated outer checkpoint. Current: `{milestone, frontier, scope}` with executor-derived slices; basis (`explicit | implicit`) orthogonal to settlement (`advisory | settled`); no persisted readiness grade or spec-global elicitation-gap table; active-branch Pi JSONL reads; one CommandExecutor mutation authority with spec-local LSN/change log; no new projected `vv_obligation` (legacy rows remain readable). Directional only: explain stored `thesis` as pitch/concept without renaming it yet, and avoid new coupling to spec-local `term` while its possible workspace lift remains future work. Persisted judgment-shaped reconciliation needs remain current, but are YAGNI-suspect: add no kinds/consumers/orchestration dependency without fresh evidence; re-evaluate derivation/removal when KA work first needs that table. Delivered note: [`Brunch 1.x data-model handoff`](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md); it does not block the deterministic TUI queue. @@ -463,22 +467,22 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Linear / branch:** [FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases), child of FE-1211; `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240. - **Kind:** structural evaluation expansion — parameterized case/substrate/oracle composition plus two repository-backed end-to-end cases; one frontier with several scoped slices, not one frontier per case. - **Certainty:** proving. -- **Status:** active 2026-07-22; the fail-closed brownfield materialization/solution-isolation tracer is complete and A49-L is retired. Both frozen case/oracle slices are complete: Brunch uses black-box `/brunch:land` plus an independent Git model, and Petrinaut's operator now materializes the full pinned HASH tree from an explicit controller source, freezes the exact handoff, and performs the closed immutable install before each network-denied lane; D136-L's focused builds and standalone `/optimization` fake-optimizer oracle run only after lane termination. A50-L remains invalidated and fully provisioned HASH `/processes/draft` host/iframe integration remains non-gating outer evidence. Historical Petrinaut provider runs remain unbuilt. +- **Status:** active 2026-07-22; D137-L is materialized as the sole pinned preparation path and its admission contracts are hardened. The production operation validates the declared root plus packet-only child, chooses from an exhaustive compile-time recipe map with no default, admits the complete prefix under the strict asymmetric Claude/Brunch policy pair, and returns lane-ready launch metadata; Brunch additionally returns an Execute-plan-ready brownfield `specId`. D136-L's compiled immutable install may leave non-ignored untracked dependency artifacts, while tracked source or packet mutation still rejects. Self-contained rivals retain wrong-parent, packet-drift, third-commit, remote/ref, path-escape, raw-verifier, and reachable-network rejection before launch. Both frozen case/oracle slices remain built; A50-L remains invalidated; D136-L's standalone `/optimization` hard gate and non-gating `/processes/draft` outer evidence remain unchanged. Historical provider runs remain unexecuted. - **Objective:** prove that FE-1239's exact elicitation → immutable handoff → crossed Brunch/Claude execution → requirement-ledger path works against real pinned brownfield repositories without weakening controller isolation or turning the Petri tracer into generic campaign machinery. - **Lights up:** pinned repository + private feature mission + shared public baseline → two codebase-aware elicitation lanes → two exact handoffs applied to the same base tree → four isolated execution cells → repository-specific controller oracle → validity-first requirement ledger, once for Brunch and once for Petrinaut. -- **Stabilizes:** one data-driven case profile across `greenfield` and `brownfield`; immutable repository/base-tree identity beside immutable spec identity; clean same-base materialization for every lane; a closed controller-only immutable dependency install before runtime denial; compile-time oracle dispatch; and diff/evidence collection that excludes harness seed files without widening FE-1230's `ExecutionAttempt`. +- **Stabilizes:** one data-driven case profile across `greenfield` and `brownfield`; immutable repository/base-tree identity beside immutable spec identity; clean same-base materialization for every lane; an exhaustive code-owned dependency recipe per pinned case, including the closed controller-only immutable install before runtime denial; pinned Brunch graph seeding and launch identity without tracked-source mutation; compile-time oracle dispatch; and diff/evidence collection that excludes harness seed files without widening FE-1230's `ExecutionAttempt`. - **Selected replay — Brunch backend:** source FE-1201 / [PR #336](https://github.com/hashintel/brunch/pull/336), base `f5a423b19f76cf345d88053456870a126e451618`, merged reference `0092a5498d22603eeb22529e8823b365ff59b505`. Derive only the host-landing backend behavior and its temp-repository safety contract; TUI/RPC/web presentation changes in the historical PR are outside the mission, so the whole reference diff is not a golden patch. - **Selected replay — Petrinaut frontend:** source FE-1162 UI slice / [PR #9051](https://github.com/hashintel/hash/pull/9051), full `hashintel/hash` base `5c7a2d9db5caa851c38938f4b1bac19005b0e978`, merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0`. Replay only the optimization UI; optimizer backend/API capabilities already present at the base commit are shared public baseline, not implementation work or elicitation gain. -- **A49-L retired (2026-07-21):** the live Claude probes established provider continuity while arbitrary target network and forbidden-root reads failed; the integrated v1 admission path records source commit/tree identity, produces one declared synthetic target commit with no worktree residue, remotes, or later refs, and rejects target/symlink escape, missing/weakened policy, unbranded verifier, or unsupported host. Claude uses strict empty MCP, explicit non-web tools, `dontAsk` plus an empty network allowlist, canonical forbidden-root denial, empty ambient setting sources/plugin settings, and `failIfUnavailable`; Brunch removes web/research/Specify-subagent surfaces, retains planner/worker, rejects lexical and symlink escape from foreground/child file tools, and injects a verifier that denies runtime network plus sealed-root reads/writes. The selected historical patches require no package dependency absent from their parent trees, so controller-side preparation can precede runtime denial. +- **A49-L retired (2026-07-21), clarified by D137-L (2026-07-22):** the live Claude probes established provider continuity while arbitrary target network and forbidden-root reads failed. Historical isolation excludes source-history reachability; it does not require a literal one-commit execution base. The declared synthetic prefix is one source-materialization root plus one packet-only exact-handoff child, with no tracked worktree mutation, extra commits, remotes, or refs; D136-L dependency artifacts may remain untracked after controller preparation. Admission rejects target/symlink escape, packet drift, missing/weakened policy, reachable required network probe, unbranded verifier, or unsupported host. Claude uses strict empty MCP, explicit non-web tools, `dontAsk` plus an empty network allowlist, canonical forbidden-root denial, empty ambient setting sources/plugin settings, and `failIfUnavailable`; Brunch removes web/research/Specify-subagent surfaces, retains planner/worker, rejects lexical and symlink escape from foreground/child file tools, and injects a verifier that denies runtime network plus sealed-root reads/writes. - **Retired:** A49-L; later historical case launches must consume this admission path and may not fall back to prompt-only conduct. - **Why now / unlocks:** FE-1239 proved the full chain only on an empty greenfield browser app, and FE-1240 closes the slice-admission defect that stopped both Brunch execution cells. The next load-bearing unknown is whether the same contracts survive repository inspection, pinned pre-existing code, and feature-delta verification. - **Boundary:** retain `minimal-petri-net-editor` unchanged as the sole `greenfield / whole_application / frontend` case; add exactly the derived FE-1201 host-landing slice as `brunch / brownfield / single_feature / backend` and PR #9051 as `petrinaut / brownfield / single_feature / frontend`. Petrinaut receives the full pinned HASH monorepo, not an extracted library fixture. The framework may parameterize case id, frozen axes, repository substrate, delivery contract, and sealed oracle id, but it must not accept arbitrary manifest shell commands or make controller material or historical solution refs target-visible. -- **Acceptance:** the existing Petri case, hashes, browser-oracle behavior, and promoted witness remain valid; both brownfield cases freeze content-addressed mission/baseline/registry/execution/oracle packets; every elicitation and execution lane starts from a separate clean checkout of the same declared base tree with no later refs/remotes or reachable historical-solution services; exact approved spec and contract bytes cross unchanged; pinned cases require an explicit safe controller source and record the compiled immutable install without tracked-source mutation before runtime network denial; harness seed commits are identified separately from implementation diffs; controller roots remain unreachable; a compile-time registry dispatches each known oracle; known-good fixtures pass and one focused negative fixture per case proves oracle sensitivity; each real case retains exactly two elicitation handoffs and four execution attempts with complete cleanup and requirement evidence or `not_assessable`. +- **Acceptance:** the existing Petri case, hashes, browser-oracle behavior, and promoted witness remain valid; both brownfield cases freeze content-addressed mission/baseline/registry/execution/oracle packets; every elicitation and execution lane starts from a separate clean checkout of the same declared base tree with no later refs/remotes or reachable historical-solution services; exact approved spec and contract bytes cross unchanged; pinned cases require an explicit safe controller source and record the compiled immutable install without tracked-source mutation before runtime network denial; pinned Brunch preparation returns a launchable `specId` whose brownfield graph preserves the exact approved bytes and is Execute-plan ready; harness seed commits are identified separately from implementation diffs; controller roots remain unreachable; a compile-time registry dispatches each known oracle; known-good fixtures pass and one focused negative fixture per case proves oracle sensitivity; each real case retains exactly two elicitation handoffs and four execution attempts with complete cleanup and requirement evidence or `not_assessable`. - **Verification:** per `memory/SPEC.md` §Verification Design. Inner — case-profile, base/tree identity, materialization, strict solution-isolation, path-isolation, oracle-dispatch, and handoff/matrix contracts. Middle — unchanged Petri regression; Brunch black-box `/brunch:land` TUI journeys resume a settled session under `PI_OFFLINE=1` and are judged by an independent full-range Git model over disposable repositories; Petrinaut deterministic fake-provider browser, host/iframe bridge, and accessibility journeys; focused final-commit-only/behavioral rivals prove sensitivity. Outer — optional real-optimizer smoke, masked Petrinaut visual review, masked code-quality review, then one fresh staged 2×2 run and promoted bounded report per brownfield case. Historical code/diff/architecture similarity is never an oracle. - **Cross-cutting obligations:** public baseline controls only addressability needed by the common oracle and is never credited as elicitation gain; existing repository knowledge may be inspected equally by both elicitation lanes; private mission/reveal/oracle files never enter target workspaces; failed and invalid attempts remain retained; Brunch stops at `promotion_prepared` and never lands into either source repository; provider/model/budget/intervention rules remain frozen per study. - **Explicitly out:** Clay/Pi Campaign Machine; any second greenfield case; whole-repository rewrites; evolving-plan studies; reliability repetitions; automatic scoring or cross-case winner; live production deployment; Cursor/Codex lanes; arbitrary oracle plugins; production `/compare-end-to-end`; landing provider outputs into either source repository; and FE-1230 `ExecutionAttempt` schema widening. - **Traceability:** D70-L fixture taxonomy; D134-L/I67-L controller/mission isolation; D40-L/D120-L/I62-L execution and no-landing boundaries; FE-1210/FE-1230/FE-1232/FE-1239/FE-1240; [`docs/praxis/comparison-runs.md`](../docs/praxis/comparison-runs.md); Notion `brunch Test Plan` axes. -- **Current execution pointer:** [`memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md`](cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md) is complete, including the operator-level pinned materialization and pre-lane dependency-install correction. Next scope the first Petrinaut historical provider run and separately owned host-integration outer evidence; neither may weaken or replace the frozen standalone mechanical hard gate. +- **Current execution pointer:** none; the admission-contract hardening card is consumed. Next scope the fresh historical provider preflight for actual HASH commit/tree resolution, real immutable install, and focused-route setup validity; separately named host-integration evidence remains non-gating. ### comparison-reporting-skills diff --git a/memory/SPEC.md b/memory/SPEC.md index 224da5602..d1665a027 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -127,7 +127,7 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | A46-L | A closed Zod schema can represent every `SessionPresentationDelta`, including `OpenAsk` questions with questionnaire questions present or exactly absent. **Validated 2026-07-15:** the contract composes the owned question schemas as exact alternatives and round-trips all delta variants without a cast or restated owner type. | medium | validated | D133-L; `src/rpc/__tests__/standalone-web-session-host.contract.test.ts` | | A47-L | Pi's valuable `InteractiveMode` TUI behavior can be preserved while one independent cwd-scoped Brunch session host owns the sole writable sealed Pi runtime, JSONL session manager, graph command authority, and semantic live-event fan-out used by both TUI and React clients. The unknown is the TUI attachment seam: Pi exports `InteractiveMode` over an in-process `AgentSessionRuntime`, not a remote TUI client. Validation must prove a real TUI + browser target without a second writable runtime, raw-Pi browser contract, or permanent second relay. Frontier: `shared-session-host-tracer`; retirement/cutover: `shared-session-host-cutover`. | medium | open | D39-L, D132-L, D133-L; I64-L, I65-L | | A48-L | A read-only semantic preflight can improve orientation-menu availability over deterministic graph-fact heuristics while returning within a ≤3-second interaction budget often enough to justify a model-backed path. Admission is limited to the configured soft recommended evaluator model; other foreground selections, missing evaluator auth, timeout, malformed output, or failure use the deterministic safe subset without restricting Pi-native `/model`. The path must consume no foreground turn, write no transcript/graph/session truth, and cache only in process by `{specId, lsn, operationalMode}`. A Brunch-owned reconciliation-blocker reader participates in gating now with an explicit empty implementation; injected non-empty blockers veto availability so future derived/persisted blocker wiring cannot be forgotten. Validation: a tracked human-approved contrastive catalog plus structured scratch tracer report; first run three uncached feasibility calls, then only if plausible run ten uncached labeled cases, requiring ≥8/10 within 3 seconds, exact flags on every completed response, at least one named deterministic-fallback miss corrected, and zero false-positive moves. If the budget or quality gate fails, retire only the model path, not the deterministic fallback. | low | open | D74-L, D109-L, D123-L; `walkthrough-remediation-2` | - + | A50-L | The real HASH `/processes/draft` host/iframe route can be the Petrinaut comparison's common hard-gate address under focused controller preparation, with unrelated auth/backend behavior cheaply stubbed. **Invalidated 2026-07-21:** the route source accepts a backend-free draft, but the Next application shell fails before rendering without generated Panda and GraphQL artifacts plus built workspace dependencies. An immutable install fetched 3,570 packages (2.79 GiB); frontend dependency preparation fanned into 61 packages including Rust/toolchain work and stopped on a pinned mise-version mismatch. By contrast, focused design-system/Petrinaut builds brought the merged standalone `/optimization` route to an accessible Optimizations view without HASH backend state. | high | invalidated | D136-L; `brownfield-comparison-cases` | ### Active Decisions @@ -307,6 +307,7 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D133-L | Web and TUI share transport-neutral presentation semantics, not platform components or separately-authored domain decoders (2026-07-14). Validated Brunch `toolResult.details` project to a shared semantic presentation model; LLM-context, TUI, and React adapters render that meaning for their audiences. Web hydrates from a named JSONL-derived product projection and overlays target-addressed live events, then refetches canonical truth at settlement/reconnect. The host emits neither ANSI/TUI strings nor ready-made HTML, and no chat mirror/event store is introduced. FE-1200 materialized the full required persisted family inventory: ordinary text; free-text, choice, choices, and bounded-questionnaire terminal read-back; candidate, review-set, and digest offers/continuations; and receipt-bearing review settlement. Live React controls answer free text and listed single/multi choices; bounded questionnaires remain answerable headlessly through D38-L's schema-tagged string/JSON envelope, without a dedicated React questionnaire form. Depends on: A44-L, D17-L, D19-L, D104-L; req 12, req 17, req 32. | [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — full required-family coverage materialized 2026-07-15 | | D134-L | The approachable `/compare-specs` control topology is one top-level project Pi session acting as the simulated user and driving exactly one comparison-harness interactive subshell at a time. It does not spawn a Pi actor that then opens a nested interactive shell. The top-level agent alone receives the private mission; each harness receives only its approved minimal framing and the user's natural messages. Shared top-level context and lane order are acceptable and disclosed for this exploratory workflow; isolation-sensitive studies continue to use FE-1210's separate rigorous campaign recipe. Ordinary text interaction is the stock-Pi baseline for choices and approvals; a custom structured-question tool may enhance presentation but is never required. Setup checks are bounded to actual selected-harness prerequisites—no throwaway Pi/Claude provider turns or synthetic actor launches on every run. Supersedes: FE-1215's unmaterialized fresh-nested-actor design note. | [`.pi/prompts/compare-specs.md`](../.pi/prompts/compare-specs.md), [`testing/comparisons/missions/README.md`](../testing/comparisons/missions/README.md); PLAN `operator-comparison-workflow` | active — remediation materialized and focused Brunch smoke witnessed 2026-07-17; full comparison witness deferred | | D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller preparation boundary. Before the network-denied candidate lane, the controller materializes the declared parent tree and exact handoff, then runs only `corepack yarn install --immutable --mode=skip-build`; this closed immutable install is the sole package-registry-network allowance and must leave tracked source clean. After the lane terminates at `promotion_prepared`, the controller runs the closed focused package builds and launches the standalone website `/optimization` route. A deterministic loopback fake optimizer grades capability-present UI, scenario-first configuration, fixed/optimized flat bindings, one saved/custom metric plus direction, request construction, progressive trials/best/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and accessibility. The real HASH `/processes/draft` host/iframe integration is not a common hard gate: its global Next shell requires unrelated GraphQL/codegen and broad 61-package platform/toolchain preparation, so one fully provisioned host/iframe smoke plus masked code/visual review remain explicit non-gating outer evidence and report `not_assessable` when unavailable. Depends on: A50-L, D134-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases`; Petrinaut scope card | active — materialized 2026-07-22 | +| D137-L | Historical replay preparation has one deep public operation that owns the security-sensitive order from pinned-source materialization through exact handoff, closed case-owned dependency preparation, strict admission, and lane finalization (2026-07-22). Its final Git authority is the declared two-commit synthetic prefix clarified under A49-L; admission validates the root source identity, packet-only child, strict Claude/Brunch policy pair, network/path probes, and absence of any third history, remote, ref, residue, or escaping symlink. Public callers receive only a lane-discriminated ready descriptor (`Brunch` includes the seeded brownfield `specId`; Claude does not); private stage types may enforce internal ordering, but no public phase machine, arbitrary command/plugin surface, or serializable security token is introduced. A compile-time case registry selects either no dependency preparation for Brunch host landing or D136-L's sole immutable Petrinaut recipe. Depends on: A49-L retirement, D136-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases` | active — materialized 2026-07-22 | | D60-L | Agent context splits into pull / projection / render / surface, distinguishes graph-truth from active-context reads, and keeps `workspace.state` separate. Agent context (what the agent reasons over) spans `cwd` (filesystem kickoff heuristic — `.brunch?`, session count/length, README/markdown sizes, file counts), `graph` (overview/list/query), and `node` (variable-hop neighborhood). ... | [`src/graph/TOPOLOGY.md`](../src/graph/TOPOLOGY.md), [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md) | active | | D83-L | Context-render house style: a markdown frame (md-pen) with TOON for uniform data and a fenced ASCII tree for hierarchy, wrapped in `
` tags; agent context clusters into `` / `` / `` scopes. Refines D60-L's RENDER stage. LLM-facing agent-context renders adopt one consistent dialect instead of ad-hoc `[bracket]` + bullet lists: - **Audience scope.** `src/agents/contexts/` owns the LLM agent-context dialect (the `
` scope-clustering plus TOON data blocks and t ... | See archive snapshot for full rationale. | active | | D85-L | Suspended prompt-resource axis model: strategy/lens/method are no longer runtime state. A 2026-06-18 grill consolidation of the `agents/skills/` topology and the D58-L manifest axes, implemented across FE-893, FE-861, and FE-898, produced useful prompt-resource content and path topology. ... | [`src/agents/TOPOLOGY.md`](../src/agents/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md) | retired/suspended | diff --git a/memory/cards/brownfield-comparison-cases--admission-contract-hardening.md b/memory/cards/brownfield-comparison-cases--admission-contract-hardening.md new file mode 100644 index 000000000..8d68ed0de --- /dev/null +++ b/memory/cards/brownfield-comparison-cases--admission-contract-hardening.md @@ -0,0 +1,100 @@ +# Harden historical replay admission contracts + +Frontier: brownfield-comparison-cases +Status: done +Mode: single +Created: 2026-07-22 + +Posture: proving (inherited from `brownfield-comparison-cases`). + +## Objective + +Historical replay admission enforces D136-L and D137-L without rejecting declared dependency artifacts or defaulting an undecided pinned case. + +## Cold-start reads + +- `memory/SPEC.md` — D136-L and D137-L +- `memory/PLAN.md` — frontier `brownfield-comparison-cases` +- `src/dev/TOPOLOGY.md` — Historical Replay Isolation +- `memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md` — completed tracer and preserved invariants +- `src/dev/execution-comparison/historical-replay-target.ts` — dependency selection and deep-operation composition +- `src/dev/end-to-end-comparison/solution-isolation.ts` — final-prefix cleanliness and network probes + +## Acceptance Criteria + +```text +admission contract hardening +├── ✓ historical-replay-target.test.ts — a dependency recipe may leave untracked artifacts while tracked source remains clean +├── ✓ historical-replay-target.test.ts — tracked dependency mutation still fails during dependency_preparation +├── ✓ historical-replay-target.test.ts — a branded verifier that reaches a required network probe fails during admission with network_probe_reachable +├── ✓ historical-replay-target.test.ts + TypeScript build — every pinned case id has one explicit code-owned dependency recipe and no default branch +├── ✓ solution-isolation.test.ts — packet drift, third commits, remotes, refs, and tracked worktree mutation remain rejected +└── ✓ card/link check — completed Petrinaut evidence points to the live historical-replay tests rather than deleted preparation tests +``` + +## Completion evidence + +| Leaf | Outcome | Evidence | +| ---- | ------- | -------- | +| Non-ignored untracked dependency artifacts remain admissible | met | `historical-replay-target.test.ts` runs the compiled Petrinaut install seam, leaves `.pnp.cjs` untracked, and reaches Claude readiness; final admission now asks Git only for tracked worktree changes | +| Tracked source or packet mutation still fails | met | the Petrinaut tracked-package rival fails in `dependency_preparation`; packet drift fails in `admission`; `solution-isolation.test.ts` now mutates tracked `package.json` and retains `git_worktree_changes_present` | +| Reachable required network probe prevents readiness | met | a deep-operation branded-verifier rival reaches `https://github.com`, fails in `admission` with `network_probe_reachable`, returns no descriptor, and removes the owned target | +| Every pinned case id has one explicit code-owned recipe | met | `PinnedExecutionCaseId` projects the contract owner union; `DEPENDENCY_RECIPE_BY_PINNED_CASE satisfies Record<...>` names Brunch `none` and Petrinaut immutable Yarn with no default; the TypeScript production build passes | +| Existing topology and isolation rivals remain rejected | met | focused historical-replay and solution-isolation suites retain wrong-parent, packet-drift, third-commit, remote/ref, tracked-worktree, path/symlink, policy, brand, and host rejection | +| Petrinaut evidence points to live tests | met | the completed Petrinaut card now cites `historical-replay-target.test.ts` and `solution-isolation.test.ts`; no `pinned-source-preparation.test.ts` reference remains | + +Untracked-artifact red: `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts -t "admits non-ignored untracked Petrinaut"` — 1 failed in `admission` with `git_worktree_changes_present` for `?? .pnp.cjs`. Green: the same command — 1 passed, 12 skipped by the name filter. + +Reachable-network sensitivity red: with the required `https://github.com` probe temporarily removed, `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts -t "branded verifier reaches a required network probe"` — 1 failed because preparation incorrectly resolved `ready`. Green after restoring the required probe: the same command — 1 passed, 14 skipped by the name filter. + +Exhaustive-map red: a temporary source sentinel failed while the production module still used the Petrinaut type guard/default-none branch. After the exhaustive `satisfies Record` map and runtime coverage for both current cases landed, the sentinel was removed as a representation lock; the TypeScript production build is the durable exhaustiveness oracle. + +Focused green before representation-only sentinel removal: `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts src/dev/execution-comparison/__tests__/operator-cli.test.ts src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts src/dev/end-to-end-comparison/__tests__/case-profile.test.ts src/dev/execution-comparison/__tests__/case-contract.test.ts src/dev/execution-comparison/__tests__/brunch-lane.test.ts` — 7 files, 45 tests passed. + +Checkpoint green after refactor: `npm run verify:full` — default 328 files/2,572 tests passed with 1 file/2 tests skipped; slow 9 files/68 tests passed; TypeScript and production builds passed. `npm run check` and `git diff --check` passed. + +Remaining provider-preflight debt: resolve the declared commit/tree against a real HASH checkout, run the real immutable install and retain its exact result, then prove the focused standalone `/optimization` route reaches setup-valid readiness before interpreting any candidate outcome. The separately named `/processes/draft` host/iframe evidence remains non-gating. + +## Verification Approach + +- **Inner:** contrastive deep-operation tests for allowed untracked dependency output, forbidden tracked mutation, reachable network, and exhaustive case-recipe selection. +- **Middle:** existing solution-isolation adversarial suite and full D137 historical-replay target suite. +- **Outer:** none for this hardening slice. The first fresh Petrinaut provider preflight owns actual HASH commit/tree resolution, real immutable install, and focused-route setup validity before any result is interpreted. +- **Checkpoint:** `npm run verify:full`, `npm run check`, `git diff --check`, and skipped-test delta versus `20709cdd`. + +## Cross-cutting obligations + +- Dependency preparation remains one compiled recipe; public contracts and manifests never supply commands. +- Only controller-owned pre-lane preparation may create dependency artifacts; source and packet files remain immutable. +- Production composition still creates the real fail-if-unavailable verifier; test composition may inject only a branded verifier factory. +- Setup rejection remains separate from candidate assertion failure. +- Brunch no-install behavior, strict asymmetric policies, no-landing, and the sole greenfield Petri path remain unchanged. + +## Assumption dependency + +None — this slice reconciles two already-settled decisions and adds the missing adversarial oracle. + +## Explicitly Out + +- A real HASH checkout/install/browser run; owned by the next provider preflight. +- General dependency plugins, persisted phase receipts, or a third brownfield case. +- Broader oracle registry or replay-module refactoring. + +## Expected touched paths (tentative) + +```text +memory/ +├── PLAN.md ~ +└── cards/ + ├── brownfield-comparison-cases--admission-contract-hardening.md + ├── brownfield-comparison-cases--historical-replay-target-admission.md ~ + └── brownfield-comparison-cases--petrinaut-optimization-case-oracle.md ~ +src/dev/ +├── TOPOLOGY.md ~ +├── end-to-end-comparison/ +│ ├── solution-isolation.ts ~ +│ └── __tests__/solution-isolation.test.ts ~ +└── execution-comparison/ + ├── historical-replay-target.ts ~ + └── __tests__/historical-replay-target.test.ts ~ +``` diff --git a/memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md b/memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md new file mode 100644 index 000000000..649f92161 --- /dev/null +++ b/memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md @@ -0,0 +1,173 @@ +# Admit lane-ready historical replay targets + +Frontier: brownfield-comparison-cases +Status: done +Mode: single +Created: 2026-07-22 + +Posture: proving (inherited from `brownfield-comparison-cases`). + +## Orientation + +- **Containing seam:** D137-L historical replay preparation/admission between frozen case selection and Brunch/Claude execution launch. +- **Frontier:** FE-1241 `brownfield-comparison-cases`; keep this work on `ka/fe-1241-brownfield-comparison-cases`. +- **Volatile state:** the Petrinaut profile/oracle and pinned Brunch graph seed are built; review found `admitHistoricalReplay` test-only, one-commit-only, and absent from the production prepare path. +- **Main risk:** weakening A49-L while reconciling its source-materialization proof with FE-1239's separate exact-handoff commit. + +## Target Behavior + +Every pinned brownfield execution lane is returned only after D137-L's deep operation admits its complete declared synthetic prefix. + +## Cold-start reads + +- `memory/SPEC.md` — A49-L retirement clarification; D136-L; D137-L +- `memory/PLAN.md` — frontier `brownfield-comparison-cases` +- `src/dev/TOPOLOGY.md` — Historical Replay Isolation and Brownfield Comparison Oracles +- `src/dev/end-to-end-comparison/solution-isolation.ts` — current admission checks and one-commit mismatch +- `src/dev/end-to-end-comparison/pinned-source-preparation.ts` — current two-commit preparation and dependency install +- `src/dev/execution-comparison/operator-cli.ts` — current Petrinaut-only pinned dispatch +- `src/dev/end-to-end-comparison/{brunch-adapter,claude-adapter}.ts` — current greenfield execution launch adapters + +## Boundary Crossings + +```text +→ frozen case contract + lane + source/target/controller/forbidden roots +→ compile-time historical-replay profile resolution +→ pinned source root commit +→ exact packet-only handoff child commit +→ case-owned dependency preparation or explicit no-op +→ strict policy-pair and runtime-boundary admission +→ lane finalization +→ BrunchReady { baseSha, specId, launch } | ClaudeReady { baseSha, launch } +``` + +## Completion evidence + +| Leaf | Outcome | Evidence | +| ---- | ------- | -------- | +| One source root plus one packet-only child is admitted | met | `historical-replay-target.test.ts` known-good temporary Git fixture; `solution-isolation.test.ts` two-commit admission | +| `HEAD` and returned `baseSha` equal the handoff child | met | `historical-replay-target.test.ts` exact `HEAD`, parent, count, and packet-delta assertions | +| Wrong parent, packet drift, or third commit rejects before readiness | met | `historical-replay-target.test.ts` production-operation rivals retain admission reasons and remove the partial target | +| Brunch host landing performs no dependency install | met | self-contained Brunch production-operation test supplies a fail-if-called install runner and reaches readiness | +| Petrinaut selects only immutable Yarn install and rejects tracked mutation | met | self-contained synthetic pinned-Git tests assert exact `corepack yarn install --immutable --mode=skip-build`, admit its non-ignored untracked dependency artifact under D136-L, and fail tracked mutation in `dependency_preparation` | +| Strict policy/path/network/ref/symlink isolation prevents readiness | met | production-operation reachable-network, remote/ref, symlink, and forbidden-root rivals plus `solution-isolation.test.ts` weakened-policy, readable-root, tracked-worktree, network, and branded-verifier rivals | +| Pinned production callers cannot bypass the deep operation | met | `operator-cli.ts` dispatches every parsed `pinned_git` contract to `prepareHistoricalReplayTarget`; obsolete half-ready preparation module/export removed | +| Brunch returns positive brownfield Execute-plan-ready `specId` | met | self-contained target test queries the exact handoff requirement, projects brownfield Execute state, and passes `assertExecuteProjectionPlanReady` | +| Claude has no `specId` and carries strict launch policy | met | production dispatch test asserts the lane discriminator, strict policy, empty MCP/settings/plugins/network allowance, and absence of `specId` | +| Ready descriptor reaches execution while Petri remains unchanged | met-with-divergence | The Claude runner directly consumes the ready descriptor and retains output from the packet-child `baseSha`; Brunch readiness carries its adapter-produced launch descriptor; no automatic pinned actor-recipe dispatcher is implemented or claimed; greenfield `execution-adapters.test.ts` remains green | +| Linux CI can exercise the production operation without weakening production defaults | met | every deep-operation test injects a factory whose verifier is branded by `createNetworkDeniedCommandRunner`; the portability oracle executes the complete ready path with a process-independent sandbox fake, raw substitution rejects, and the default unsupported-host oracle remains fail-closed | +| Setup rejection is structured and leaves no launchable partial target | met | `HistoricalReplayTargetPreparationError` carries `status`, `phase`, and admission reasons; failure tests prove owned-target removal and pre-existing-target preservation | + +Portability red: `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts -t "injected branded verifier factory"` — 1 failed because the production operation ignored the injected factory (`expected 0 to be 1`). + +Portability green: the same command — 1 passed, 12 skipped by the name filter. The paired unsupported-host oracle also passed: `npm test -- src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts -t "fails closed when the host cannot provide"` — 1 passed, 3 skipped by the name filter. + +Focused green: `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts src/dev/execution-comparison/__tests__/operator-cli.test.ts src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts src/dev/end-to-end-comparison/__tests__/case-profile.test.ts src/dev/execution-comparison/__tests__/case-contract.test.ts src/dev/execution-comparison/__tests__/brunch-lane.test.ts` — 7 files, 43 tests passed. + +Checkpoint green: `npm run verify:full` — default 328 files/2,571 tests passed with 1 file/2 tests skipped; slow 9 files/68 tests passed; build passed. `npm run check` and `git diff --check` passed. Skipped-test delta versus `20709cdd`: 0 changed `.skip`/`.todo` markers. + +## Risks and Assumptions + +- **RISK:** changing `commitCount === 1` to `2` could admit arbitrary synthetic history. + - **MITIGATION:** validate the exact root/child relationship, source identity, packet-only child delta, `HEAD === baseSha`, and absence of any third commit. +- **RISK:** generalizing pinned dispatch could run Petrinaut's install for Brunch host landing. + - **MITIGATION:** an exhaustive code-owned map gives every pinned case id exactly one `none` or `petrinaut-yarn-immutable-v1` recipe with no default; the public contract never supplies commands. +- **RISK:** tests could pass only because the developer has a sibling HASH checkout. + - **MITIGATION:** the admission tracer must use self-contained temporary Git fixtures at the deep-module boundary; optional real-repository evidence cannot be the only oracle. +- **ASSUMPTION:** the exact-handoff child can remain independently attributable while the two commits are admitted as one closed synthetic prefix. + - **IMPACT IF FALSE:** D137-L's selected shape fails and preparation must collapse to one commit or adopt a different execution-base model. + - **VALIDATE:** one known-good two-commit fixture plus packet-drift, wrong-parent, and third-commit rivals through the production operation. + - **SPEC:** settled by D137-L; this slice is the falsifier. + +## Posture check + +This tracer scores on all three proving axes: + +- **Proof of life:** a prepared pinned target reaches a real lane launch descriptor only through production admission. +- **Invariant:** A49-L isolation is enforced over the final execution base rather than a pre-handoff intermediate. +- **Uncertainty:** contrastive Git-topology rivals can falsify D137-L before provider work. + +No spike is cheaper: the production slice itself is a temporary-repository experiment with no provider, browser, or external service. + +## Acceptance Criteria + +```text +historical replay target +├── declared prefix +│ ├── ✓ historical-replay-target.test.ts — one source root plus one packet-only child is admitted +│ ├── ✓ historical-replay-target.test.ts — HEAD and returned baseSha equal the handoff child +│ └── ✓ historical-replay-target.test.ts — wrong parent, packet drift, or any third commit fails before lane readiness +├── closed preparation +│ ├── ✓ historical-replay-target.test.ts — Brunch host landing performs no dependency install +│ └── ✓ historical-replay-target.test.ts — Petrinaut selects exactly corepack yarn install --immutable --mode=skip-build and rejects tracked mutation +├── strict isolation +│ ├── ✓ historical-replay-target.test.ts — weakened policy, readable forbidden root, reachable network probe, remote/ref, or escaping symlink prevents a ready result +│ └── ✓ production-call-site guard — no pinned production caller can bypass the deep operation for a half-ready workspace +├── lane finalization +│ ├── ✓ historical-replay-target.test.ts — Brunch returns a positive specId whose exact spec graph is brownfield and Execute-plan ready +│ ├── ✓ historical-replay-target.test.ts — Claude returns no specId and carries the strict launch policy +│ └── △ execution adapters — Claude directly consumes its ready descriptor and Brunch carries adapter-produced launch metadata; no automatic actor-recipe dispatcher is claimed; Petri greenfield remains unchanged +└── failure hygiene + └── ✓ historical-replay-target.test.ts — rejection retains structured phase/reason evidence and leaves no launchable partial target +``` + +## Invariants preserved + +- Exact approved `spec.md` and `public-contract.json` bytes cross unchanged — guarded by: existing handoff/public-packet tests plus `historical-replay-target.test.ts`. +- No historical source refs, remotes, solution services, controller roots, or case-private oracle material become target-reachable — guarded by: `solution-isolation.test.ts` and the new production-path rivals. +- Brunch and Claude retain their asymmetric strict policies; neither is reduced to a lowest-common-denominator sandbox — guarded by: `solution-isolation.test.ts` and launch-descriptor assertions. +- Brunch stops at `promotion_prepared` and never lands provider output into a source repository — guarded by: existing execution-adapter and no-landing suites. +- `ExecutionAttempt` remains unchanged — guarded by: existing end-to-end matrix/attempt tests. +- Minimal Petri remains the sole greenfield case and its preparation path remains byte-identical — guarded by: existing case-profile and execution-adapter tests. +- Stop the line: a red on exact handoff bytes, controller isolation, strict policy shape, or no-landing is a respec signal, not a fixture update. + +## Verification Approach + +- **Inner:** self-contained temporary-Git state-machine rivals through the public deep operation; focused case-policy, operator, adapter, and launch-descriptor tests. +- **Middle:** existing full solution-isolation adversarial suite plus pinned Brunch graph/preflight integration; no provider invocation. +- **Outer:** none for this infrastructure slice. FE-1241 owns the later fresh Petrinaut provider matrix and separately named host-integration evidence after this admission gate closes. +- **Checkpoint:** `npm run verify:full`, `npm run check`, `git diff --check`, and skipped-test delta versus `20709cdd`. + +## Cross-cutting obligations + +- D137-L exposes one deep operation, not a public phase machine or serializable security token. +- Dependency commands remain code-owned and case-closed; no manifest command/plugin escape hatch. +- Admission validates exactly one strict Claude/Brunch policy pair even though the returned descriptor is lane-specific. +- Setup rejection remains distinct from candidate assertion failure and happens before provider work. +- Failed/invalid evidence remains inspectable while incomplete targets cannot be launched. + +## Explicitly Out + +- Real Petrinaut provider runs, full HASH `/processes/draft`, and optional real optimizer evidence. +- Closing the separate review finding that the synthetic Petrinaut browser fixture does not prove the real pinned HASH route. +- General replay plugins, resumable persisted phase engines, arbitrary dependency recipes, or a third brownfield case. +- Refactoring the oracle registry or unrelated cleanup/retry infrastructure. + +## Expected touched paths (tentative) + +```text +memory/ +├── SPEC.md ~ +├── PLAN.md ~ +└── cards/brownfield-comparison-cases--historical-replay-target-admission.md +src/dev/ +├── TOPOLOGY.md ~ +├── end-to-end-comparison.ts ~ +├── end-to-end-comparison/ +│ ├── solution-isolation.ts ~ +│ ├── pinned-source-preparation.ts -|~ +│ ├── brunch-adapter.ts ~ +│ ├── claude-adapter.ts ~ +│ └── __tests__/ +│ ├── solution-isolation.test.ts ~ +│ └── execution-adapters.test.ts ~ +└── execution-comparison/ + ├── historical-replay-target.ts + + ├── historical-replay-target/ ? + ├── case-contract.ts ~ + ├── operator-cli.ts ~ + ├── brunch-lane.ts ~ + └── __tests__/ + ├── historical-replay-target.test.ts + + └── operator-cli.test.ts ~ +``` diff --git a/memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md b/memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md new file mode 100644 index 000000000..b124dfb9c --- /dev/null +++ b/memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md @@ -0,0 +1,177 @@ +# Petrinaut optimization brownfield case oracle + +Frontier: brownfield-comparison-cases +Status: done +Mode: single +Created: 2026-07-21 + +## Orientation + +- Containing seam: controller-owned brownfield case profiles, pinned-source preparation, compile-time oracle dispatch, and FE-1239 exact handoffs over unchanged FE-1230 attempt leaves. +- Frontier: `brownfield-comparison-cases` (FE-1241), on `ka/fe-1241-brownfield-comparison-cases`; this is another slice on the existing issue/branch, not a new frontier. +- Volatile state: A49-L isolation and the Brunch host-landing profile/oracle are complete. The Petri editor remains the sole greenfield case. Petrinaut packets, target preparation, and oracle code do not exist yet; no historical provider run is authorized. +- Spike verdict: A50-L is invalidated and D136-L is active. The full HASH `/processes/draft` Next shell is not the common hard-gate address; the focused standalone `/optimization` route is. The authenticated HASH host/iframe integration remains explicit non-gating outer evidence. + +Posture: proving (inherited from `brownfield-comparison-cases`). + +Cross-cutting obligations: + +- consume the A49-L history-free source identity and fail-closed Claude/Brunch policies without duplicating or weakening them; +- give every elicitation and execution cell a separate materialization of the same full HASH parent tree `a3e08cf75e00cc9016c931f4665341506e03533e` at commit `5c7a2d9db5caa851c38938f4b1bac19005b0e978`; +- preserve FE-1239 exact approved-spec bytes and FE-1230 `ExecutionAttempt` without schema widening; +- keep historical ids, URLs, merged refs, solution code, controller fixtures, expected requests/events, and oracle source outside target-visible packets and roots; +- retain `minimal-petri-net-editor` unchanged as the sole greenfield case and the completed Brunch case/oracle unchanged; +- judge public behavior, proxy/request provenance, and accessible rendered outcomes—never historical code, module placement, diff shape, or architecture. + +## Target Behavior + +A frozen Petrinaut brownfield case produces claim-linked controller verdicts from deterministic fake-optimizer browser journeys through the candidate's public `/optimization` route. + +## Cold-start reads + +- `memory/SPEC.md` — A50-L, D136-L; Brownfield assessment; §Verification Design rows for case identity, Petrinaut standalone browser/accessibility, and outer evidence; historical-replay boundary and blind spots +- `memory/PLAN.md` — frontier: `brownfield-comparison-cases` +- `src/dev/TOPOLOGY.md` — historical replay isolation, compiled oracle ownership, and public-root/private-subtree layout +- `docs/praxis/comparison-runs.md` — controller isolation, attempt validity, retention, and no-landing rules +- `src/dev/end-to-end-comparison/{study-contract,solution-isolation}.ts` — closed case identity and A49-L materialization/admission +- `src/dev/execution-comparison/{case-contract,operator-cli,oracle-pack,browser-oracle}.ts` and `src/dev/execution-comparison-operator.ts` — current profile, preparation, browser journey, report, and compile-time dispatch seams +- `testing/{end-to-end-comparisons,execution-comparisons}/cases/{minimal-petri-net-editor,brunch-host-landing}/` — immutable predecessor packet shapes; do not copy browser-specific fields into the backend case +- PR #9051 at parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978` and merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0` — bootstrap behavior/rivals only; no historical file is oracle authority or target-visible input +- Base-tree `@hashintel/petrinaut-core` optimization schemas — public pre-existing backend/API capability; do not credit it as UI implementation work + +## Boundary Crossings + +```text +content-addressed Petrinaut brownfield study + -> closed frontend profile + full HASH parent commit/tree + -> A49-L history-free materialization per lane + -> exact public specification/contract seed + -> controller runs corepack yarn install --immutable --mode=skip-build + -> tracked source remains clean; dependency result is retained outside ExecutionAttempt + -> network-denied execution lane starts and terminates at promotion_prepared + -> execution attempt is retained unchanged + -> controller runs focused package builds after lane termination + -> controller starts the candidate Petrinaut website + -> /optimization exposes the capability-present optimization UI + -> controller fakes optimizer responses behind the same-origin proxy + -> browser drives scenario, parameters, metric, stream, error, and cancel + -> proxy/request provenance + accessibility + rendered-state assertions + <- claim-linked report with setup_failed distinct from assertion_failed +``` + +## Risks and Assumptions + +- RISK: the oracle imports PR #9051 modules or requires their file layout and becomes an architecture matcher → MITIGATION: launch only declared candidate commands and observe browser roles/names, HTTP requests, aborts, and rendered outcomes; source guards scan the whole oracle subtree. +- RISK: the standalone route passes while the authenticated HASH host/iframe bridge is absent → MITIGATION: do not claim the common gate proves that bridge. Before the first provider matrix, the frontier owns one fully provisioned `/processes/draft` host/iframe smoke plus masked code review; unavailable evidence is `not_assessable`. +- RISK: target code can read fake events, expected manifests, wrong rivals, or historical solution material → MITIGATION: create controller fixtures after lane termination in disjoint roots and retain only aggregate claim results in audience-safe packets. +- RISK: browser/Vite startup or fixture import failure is mistaken for product failure → MITIGATION: report `setup_failed` separately with focused-build, process, route, and console evidence; assertion failures begin only after the declared app and fixture are ready. +- RISK: full HASH dependency preparation widens runtime network → MITIGATION: after materialization and exact handoff seeding, run only the compiled immutable Yarn install as a controller-owned pre-lane step, reject tracked-source mutation, then admit the target under A49-L; execution and verifier commands remain network-denied, while the post-lane oracle permits loopback only. +- RISK: visual polish is overclaimed by DOM assertions → MITIGATION: mechanical accessibility/interaction checks gate this card; masked qualitative visual review remains separately owned outer evidence. +- ASSUMPTION: the spike's focused preparation set—design-system codegen/build plus Petrinaut core, optimizer-client, and UI builds—is sufficient for history-free candidate checkouts without broad HASH codegen/toolchain work. + → IMPACT IF FALSE: candidate setup is invalid and the focused hard gate must be respecified before provider work; do not fall back to the broad 61-package Next-shell preparation. + → VALIDATE: operator preparation must record the exact immutable install before either lane, and the known-good slow fixture must execute only the closed post-lane focused builds before `/optimization` becomes ready. + +## Posture check + +- Proof of life: lights the first full-monorepo frontend case from pinned source identity through a focused public optimization route to a controller verdict. +- Invariant: establishes one compile-time third case/oracle variant without arbitrary commands, manifest paths, plugins, or historical references. +- Uncertainty: locks the spike-selected focused preparation boundary and proves it is sufficient in a history-free candidate checkout. + +## Acceptance Criteria + +```text +Petrinaut case/oracle acceptance +├── frozen profile +│ ├── case-profile.test.ts → exact mission/baseline/registry/contract/oracle hashes and parent commit/tree +│ ├── case-profile.test.ts → target-visible bytes omit PR/issue ids, URLs, merged ref, expected events, and solution wording +│ └── case-contract.test.ts → only petrinaut/brownfield/single_feature/frontend/pinned_git parses +├── same-base preparation +│ ├── operator-cli.test.ts → the real prepare path requires an explicit safe source and routes both lanes through pinned HASH materialization +│ ├── historical-replay-target.test.ts → the deep operation materializes the declared source identity plus exact packet-only child before readiness +│ ├── historical-replay-target.test.ts → exact immutable install is recorded before lane execution, tracked source stays clean, and declared untracked dependency artifacts remain admissible +│ └── historical-replay-target.test.ts + solution-isolation.test.ts → no third commit, remotes, later refs, tracked residue, or packet drift; source identity remains separate from implementation diff +├── closed oracle +│ ├── oracle-pack.test.ts → third compiled manifest, exact check ids, complete requirement assignment +│ ├── operator-oracle-dispatch.test.ts → known id selects compiled code; unknown id fails before launch +│ └── petrinaut-optimization-oracle.test.ts → no historical/candidate-internal imports or runtime implementation selectors +├── public browser behavior +│ ├── petrinaut-optimization-oracle.slow.test.ts → focused preparation launches `/optimization` with an accessible Optimizations view +│ ├── petrinaut-optimization-oracle.slow.test.ts → scenario is explicit and changing it resets configuration +│ ├── petrinaut-optimization-oracle.slow.test.ts → flat fixed/optimized bindings and one saved/custom metric direction reach the request +│ ├── petrinaut-optimization-oracle.slow.test.ts → progressive trials, best-so-far, completion, and service failure render distinctly +│ ├── petrinaut-optimization-oracle.slow.test.ts → cancel aborts the host request and renders cancelled +│ ├── petrinaut-optimization-oracle.slow.test.ts → browser traffic stays on the same-origin proxy; requests/DOM expose no private upstream origin +│ └── petrinaut-optimization-oracle.slow.test.ts → required tab/form/result/cancel controls are keyboard-reachable with stable roles/names +├── sensitivity +│ └── petrinaut-optimization-oracle.test.ts → missing-route, final-only, direct-private-origin, and UI-only-cancel rivals fail focused claims +└── regressions + ├── existing Petri + Brunch case/profile/oracle suites remain unchanged and green + ├── exact handoff, four-cell matrix, immutable attempt, isolation, cleanup, and no-landing suites remain green + └── npm run verify:full passes +``` + +## Invariants preserved + +- Historical code/diff/architecture similarity is never an oracle — guarded by: whole-subtree dependency/source guard and browser-only candidate interaction. +- Full `hashintel/hash` is the substrate; no Petrinaut library extraction masquerades as brownfield context — guarded by: parent commit/tree and same-base preparation tests. +- Target-visible material contains no controller fixture, hidden expected request/event, private optimizer origin, or historical-solution locator — guarded by: profile redaction/hash tests plus root-disjointness admission. +- Brunch comparison execution still stops at `promotion_prepared`; controller browser execution begins only after lane termination — guarded by: existing execution terminal suites and oracle report chronology. +- `ExecutionAttempt` remains unchanged and case/oracle identity stays beside it — guarded by: `artifact-contract.test.ts`, `execution-cell.test.ts`, and operator report tests. +- Petri remains the sole greenfield case and the completed Brunch profile/oracle remains byte-identical — guarded by: existing case/study/oracle packet hash tests and both predecessor slow suites. + +## Verification Approach + +- Inner: closed schema, packet hash, source identity, A49-L preparation, claim coverage, compile-time registry, path isolation, and focused wrong-rival tests. +- Middle: operator-owned pinned materialization and immutable dependency installation precede the sealed lane; controller-owned `/optimization` browser journeys run the closed focused builds after lane termination against a synthetic contract fixture using deterministic same-origin proxy responses, no live optimizer. +- Outer: owned by `brownfield-comparison-cases` before its first Petrinaut 2×2 provider run; trigger after this mechanical oracle is frozen and one candidate output exists, then run one fully provisioned `/processes/draft` host/iframe integration smoke, identity-masked visual/code review, and an optional real-optimizer adapter smoke. Outer findings are non-gating, are `not_assessable` when unavailable, and cannot repair a mechanical failure. + +## Expected touched paths (tentative) + +```text +testing/ +├── comparisons/missions/petrinaut-optimization.md + +├── end-to-end-comparisons/cases/petrinaut-optimization/ + +└── execution-comparisons/cases/petrinaut-optimization/ + +src/dev/ +├── TOPOLOGY.md ~ +├── end-to-end-comparison/ +│ ├── study-contract.ts ~ +│ └── __tests__/case-profile.test.ts ~ +├── execution-comparison-operator.ts ~ +└── execution-comparison/ + ├── case-contract.ts ~ + ├── operator-cli.ts ~ + ├── oracle-pack.ts ~ + ├── petrinaut-optimization-oracle.ts + + ├── petrinaut-optimization-oracle/ + + └── __tests__/ + ├── case-contract.test.ts ~ + ├── historical-replay-target.test.ts ~ + ├── oracle-pack.test.ts ~ + ├── operator-oracle-dispatch.test.ts ~ + ├── petrinaut-optimization-oracle.test.ts + + └── petrinaut-optimization-oracle.slow.test.ts + +memory/ +├── PLAN.md ~ +└── SPEC.md ? +``` + +## Completion evidence + +| Leaf | Outcome | Evidence | +| --- | --- | --- | +| Frozen profile | met | `case-profile.test.ts` and `case-contract.test.ts`; exact packet hashes, pinned HASH commit/tree, closed brownfield axes, and target-visible redaction | +| Same-base preparation | met | `operator-cli.test.ts`, `historical-replay-target.test.ts`, and `solution-isolation.test.ts`; production dispatch requires a safe explicit source and routes every pinned lane through the one deep operation, which preserves exact handoff bytes over the declared two-commit prefix, records only `corepack yarn install --immutable --mode=skip-build` for Petrinaut, permits non-ignored untracked dependency artifacts while rejecting tracked source/packet mutation, and retains no third commit, remotes, or later refs. The Brunch-ready branch returns a positive Execute-plan-ready brownfield `specId`; Claude readiness has no `specId`. | +| Closed oracle | met | `oracle-pack.test.ts`, `operator-oracle-dispatch.test.ts`, and `petrinaut-optimization-oracle.test.ts`; exact manifest claims, compile-time dispatch, immutable implementation pack, and source isolation | +| Public browser behavior | met | `petrinaut-optimization-oracle.slow.test.ts`; focused builds, standalone `/optimization`, scenario reset, fixed/optimized bindings, saved/custom objectives, progressive/best/completion/error, cancel/abort, same-origin secrecy, and accessible controls | +| Sensitivity | met | `petrinaut-optimization-oracle.test.ts`; missing-route, final-only, direct-private-origin, and UI-only-cancel rivals each fail their focused claim | +| Existing invariants | met | unchanged Petri/Brunch and exact-handoff/matrix/isolation/no-landing suites passed in the full repository gate; `ExecutionAttempt` remained unchanged | +| Full repository gate | met | `npm run verify:full`: default 328 files passed and 1 skipped, 2,562 tests passed and 2 skipped; slow 9 files/68 tests passed; TypeScript and production builds passed | + +The first full gate exposed two integration regressions: the operator case-list expectation still named two cases, and concurrent temporary Git cleanup could hit a transient macOS `ENOTEMPTY`. The closed list now includes Petrinaut and generated-root cleanup retries the transient removal without changing target behavior. + +Independent review then exposed that the low-level pinned-source proof was not reached by production operator preparation and that a fresh archive lacked dependencies. The corrected operator dispatch now makes the source checkout explicit, preserves exact packet bytes, records the closed controller-only immutable install before each lane, and leaves the existing post-lane focused-build/browser oracle—including required service-provider mode—unchanged. + +The interrupted pinned-Brunch follow-through is also closed: `seedBrownfieldBrunchExecutionWorkspace` seeds the already-materialized full repository without Petri-specific execution wording or tracked-source mutation, establishes the stored brownfield posture, and returns the `specId` consumed by the network-denied comparison TUI preflight. + +Skipped-test-count delta vs the preceding FE-1241 full gate: zero; both retained two skipped tests. The fully provisioned `/processes/draft` host/iframe smoke, masked review, and optional real-optimizer smoke remain explicitly non-gating outer evidence for the later historical provider-run scope. diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index 8fa71f773..74fd2fdd5 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -21,13 +21,15 @@ It does not own published CLI behavior, public RPC contracts, database imports f ## Historical Replay Isolation -`end-to-end-comparison/solution-isolation.ts` owns the versioned fail-closed admission seam for pinned brownfield source trees. It records source commit/tree identity, materializes one declared synthetic target commit without worktree residue, remotes, or later refs, requires exactly one strict Claude/Brunch policy pair, rejects target and symlink escapes, and admits declared local checks only through the branded macOS verifier that denies runtime network plus sealed-root reads/writes. Claude uses native sandbox denial with no ambient MCP/settings/plugins or web tools; Brunch uses target-bounded foreground and planner/worker child file tools. Unsupported hosts stop before provider work. Controller-owned preparation may populate declared dependencies in the target only before isolation admission; its recipe and result metadata stay controller-owned, and case-private packets, historical references, fixtures, and oracles never enter the target. +`execution-comparison/historical-replay-target.ts` is D137-L's one deep public operation: callers provide the frozen case, lane, source, target, controller, and forbidden roots and receive only a lane-ready descriptor after every phase succeeds. The operation owns a closed two-commit synthetic prefix—one root materializes and identifies the pinned source tree, its sole child contains only the exact approved spec and public contract and becomes the execution base—then selects from an exhaustive compile-time map with one explicit code-owned dependency recipe for every pinned case id, performs strict admission, and finalizes the lane. There is no default recipe: adding a pinned case without deciding its preparation fails the TypeScript build. Brunch readiness includes a positive brownfield `specId`; Claude readiness omits `specId`; both include adapter-produced launch metadata. No public phase machine, arbitrary command/plugin seam, or half-ready preparation export remains. + +`end-to-end-comparison/solution-isolation.ts` owns the versioned fail-closed admission mechanics. Admission validates both declared commits and their parent/diff relationship, requires exactly one strict Claude/Brunch policy pair, rejects any third history, tracked worktree mutation, remote, later ref, target/symlink escape, forbidden-root overlap, raw verifier substitution, reachable required network probe, or unsupported host, and admits declared local checks only through the branded macOS verifier that denies runtime network plus sealed-root reads/writes. D136-L deliberately permits controller-created non-ignored untracked dependency artifacts after tracked-source cleanliness passes; exact packet bytes are still rehashed independently during final admission. Production composition creates the verifier from the real host and therefore fails closed when unavailable; test composition may inject only a factory returning the same runtime-branded verifier, so Linux CI can exercise the production deep operation without making a verifier part of its domain input or admitting raw substitutes. Claude uses native sandbox denial with no ambient MCP/settings/plugins or web tools; Brunch uses target-bounded foreground and planner/worker child file tools. Controller-owned preparation may run only the compiled Petrinaut immutable-install recipe before admission; Brunch host landing performs no dependency install. Recipe/result metadata stay controller-owned, and case-private packets, historical references, fixtures, and oracles never enter the target. ## Brownfield Comparison Oracles `execution-comparison/host-landing-oracle.ts` is the public Brunch controller entry point; its same-named private subtree owns disposable Git fixtures, settled-session public-TUI actuation, the independent Git outcome model, and report types. The oracle resumes only controller-supplied settled sessions under `PI_OFFLINE=1`, invokes `/brunch:land` through the built candidate, and distinguishes setup invalidity from claim failure. -Petrinaut preparation selected by D136-L is split across the operator and oracle. `execution-comparison/operator-cli.ts` requires an explicit controller-owned source repository, materializes the frozen full HASH tree and exact handoff per lane, then runs the one compiled immutable install (`corepack yarn install --immutable --mode=skip-build`) before the network-denied lane; install failure or tracked-source mutation removes the target. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against the standalone `/optimization` route. The browser sees only the candidate origin and same-origin proxy, while controller fixtures and the private optimizer origin remain outside target-visible roots. The fully provisioned `/processes/draft` host/iframe journey is non-gating outer evidence and does not alter this mechanical hard gate. +Petrinaut preparation selected by D136-L is split across the historical-replay operation and oracle. `execution-comparison/operator-cli.ts` resolves every `pinned_git` contract into the deep operation; that operation materializes the frozen full HASH tree and exact handoff per lane, then runs the one compiled immutable install (`corepack yarn install --immutable --mode=skip-build`) before the network-denied lane. Install failure or tracked-source mutation removes only the newly owned target. The pinned Brunch branch then seeds the exact specification as one opaque settled requirement, establishes brownfield posture, and returns the `specId` consumed by `execution-comparison-brunch.ts`; Claude receives the same source and handoff without Brunch graph state. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against the standalone `/optimization` route. The browser sees only the candidate origin and same-origin proxy, while controller fixtures and the private optimizer origin remain outside target-visible roots. The fully provisioned `/processes/draft` host/iframe journey is non-gating outer evidence and does not alter this mechanical hard gate. `execution-comparison-operator.ts` dispatches both brownfield oracles and the retained Petri browser oracle through a closed compile-time registry and includes every implementation module in the immutable oracle-pack hash; manifests cannot select paths, commands, or plugins at runtime. diff --git a/src/dev/end-to-end-comparison.ts b/src/dev/end-to-end-comparison.ts index 4e583bffa..1d425f04e 100644 --- a/src/dev/end-to-end-comparison.ts +++ b/src/dev/end-to-end-comparison.ts @@ -53,17 +53,14 @@ export { } from './end-to-end-comparison/claude-adapter.js'; export { retainExecutionCell } from './end-to-end-comparison/execution-cell.js'; export { - admitHistoricalReplay, assertTargetBoundedPath, createBrunchSolutionIsolationPolicy, createClaudeSolutionIsolationPolicy, createNetworkDeniedCommandRunner, - materializePinnedSourceTree, SolutionIsolationAdmissionError, type BrunchSolutionIsolationPolicy, type ClaudeSolutionIsolationPolicy, type IsolationAdmissionReason, - type MaterializedPinnedSourceTree, type NetworkDeniedCommandRunner, type SolutionIsolationPolicy, } from './end-to-end-comparison/solution-isolation.js'; diff --git a/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts b/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts index 385043bd9..cd4a171ac 100644 --- a/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts @@ -64,30 +64,32 @@ describe('compiled end-to-end comparison case profiles', () => { mode: 'brownfield', scope: 'single_feature', surface: 'backend', - repository: { substrate: 'pinned_git' }, + repository: { + substrate: 'pinned_git', + parentCommit: brunch.contract.source?.parentCommit, + parentTree: brunch.contract.source?.parentTree, + }, }); expect(petrinautPacket.contract.case).toMatchObject({ product: 'petrinaut', mode: 'brownfield', scope: 'single_feature', surface: 'frontend', - repository: { substrate: 'pinned_git' }, + repository: { + substrate: 'pinned_git', + parentCommit: petrinaut.contract.source?.parentCommit, + parentTree: petrinaut.contract.source?.parentTree, + }, }); - const sources = [brunch.contract.source, petrinaut.contract.source]; - if (sources.some((source) => source === undefined)) { + if (brunch.contract.source === undefined || petrinaut.contract.source === undefined) { throw new Error('brownfield profiles must declare source identity'); } - for (const source of sources) { - const actualTree = ( - await execFileAsync('git', ['rev-parse', `${source!.parentCommit}^{tree}`], { - cwd: - source!.parentCommit === '5c7a2d9db5caa851c38938f4b1bac19005b0e978' - ? fileURLToPath(new URL('../../../../../hash/', import.meta.url)) - : repositoryRoot, - }) - ).stdout.trim(); - expect(actualTree).toBe(source!.parentTree); - } + const actualBrunchTree = ( + await execFileAsync('git', ['rev-parse', `${brunch.contract.source.parentCommit}^{tree}`], { + cwd: repositoryRoot, + }) + ).stdout.trim(); + expect(actualBrunchTree).toBe(brunch.contract.source.parentTree); }); it('keeps historical solution locators and controller expectations out of target-visible artifacts', async () => { diff --git a/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts b/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts index aa1c5facb..58b4e3413 100644 --- a/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'; import { createServer, type Server } from 'node:http'; import { tmpdir } from 'node:os'; @@ -13,6 +14,8 @@ import { createNetworkDeniedCommandRunner, materializePinnedSourceTree, SolutionIsolationAdmissionError, + type MaterializedHistoricalReplayPrefix, + type MaterializedPinnedSourceTree, type NetworkDeniedCommandRunner, type SolutionIsolationPolicy, } from '../solution-isolation.js'; @@ -131,6 +134,7 @@ describe('brownfield historical-solution isolation', () => { sourceCommit: source.sourceCommit, targetDir, }); + const prefix = await freezePacketChild(materialized); const probeUrl = await localProbeUrl(); const forbiddenRoots = [ source.repositoryDir, @@ -141,7 +145,7 @@ describe('brownfield historical-solution isolation', () => { await Promise.all(forbiddenRoots.slice(1).map(async (path) => await mkdir(path))); const admission = await admitHistoricalReplay({ - materialized, + prefix, policies: strictPolicies(targetDir, forbiddenRoots), forbiddenRoots, networkProbeUrls: [probeUrl, 'https://github.com', 'https://linear.app', 'https://www.notion.so'], @@ -169,11 +173,12 @@ describe('brownfield historical-solution isolation', () => { sourceCommit: source.sourceCommit, targetDir, }); + const prefix = await freezePacketChild(materialized); const probeUrl = await localProbeUrl(); const forbiddenRoots = [source.repositoryDir, join(root, 'controller')]; await mkdir(forbiddenRoots[1]!); const baseInput = { - materialized, + prefix, forbiddenRoots, networkProbeUrls: [probeUrl, 'https://github.com', 'https://linear.app', 'https://www.notion.so'], verifier: createNetworkDeniedCommandRunner({ forbiddenReadRoots: forbiddenRoots }), @@ -204,7 +209,7 @@ describe('brownfield historical-solution isolation', () => { reasons: [expect.objectContaining({ code: 'policy_weakened' })], }); - await writeFile(join(targetDir, 'historical-reference.patch'), 'must not be target-visible\n'); + await writeFile(join(targetDir, 'package.json'), '{"scripts":{"verify":"node changed.mjs"}}\n'); await expect( admitHistoricalReplay({ ...baseInput, @@ -213,7 +218,7 @@ describe('brownfield historical-solution isolation', () => { ).rejects.toMatchObject({ reasons: [expect.objectContaining({ code: 'git_worktree_changes_present' })], }); - await rm(join(targetDir, 'historical-reference.patch')); + await writeFile(join(targetDir, 'package.json'), '{"scripts":{"verify":"node verify.mjs"}}\n'); await git(targetDir, ['remote', 'add', 'origin', 'https://github.com/hashintel/brunch.git']); await git(targetDir, ['branch', 'later-solution']); @@ -289,3 +294,33 @@ describe('brownfield historical-solution isolation', () => { ); }); }); + +async function freezePacketChild( + materialized: MaterializedPinnedSourceTree, +): Promise { + const packetFiles = [ + { path: 'public-contract.json' as const, bytes: Buffer.from('{"schemaVersion":1}\n') }, + { path: 'spec.md' as const, bytes: Buffer.from('# Exact packet\n') }, + ]; + for (const file of packetFiles) { + await writeFile(join(materialized.targetDir, file.path), file.bytes); + } + await git(materialized.targetDir, ['add', '--', ...packetFiles.map(({ path }) => path)]); + await git(materialized.targetDir, [ + '-c', + 'user.name=Isolation Test', + '-c', + 'user.email=isolation@invalid.local', + 'commit', + '-m', + 'Freeze exact packet', + ]); + return { + ...materialized, + baseSha: await git(materialized.targetDir, ['rev-parse', 'HEAD']), + packetFiles: packetFiles.map(({ path, bytes }) => ({ + path, + sha256: `sha256:${createHash('sha256').update(bytes).digest('hex')}`, + })), + }; +} diff --git a/src/dev/end-to-end-comparison/brunch-adapter.ts b/src/dev/end-to-end-comparison/brunch-adapter.ts index b84c88a83..f242f1e71 100644 --- a/src/dev/end-to-end-comparison/brunch-adapter.ts +++ b/src/dev/end-to-end-comparison/brunch-adapter.ts @@ -15,6 +15,28 @@ export interface ExecutionLaunch { readonly cwd: string; } +export function createBrunchExecutionLaunch(input: { + readonly workspaceDir: string; + readonly specId: number; + readonly forbiddenRoots: readonly string[]; +}): ExecutionLaunch { + return { + command: 'npx', + args: [ + 'tsx', + 'src/dev/execution-comparison-brunch.ts', + '--workspace', + input.workspaceDir, + '--spec-id', + String(input.specId), + '--solution-isolation', + 'v1', + ...input.forbiddenRoots.flatMap((root) => ['--forbidden-root', root]), + ], + cwd: repositoryRoot(), + }; +} + export async function prepareBrunchExecutionCell(input: { readonly cellRoot: string; readonly workspaceDir: string; @@ -47,21 +69,11 @@ export async function prepareBrunchExecutionCell(input: { ); return { prepared, - launch: { - command: 'npx', - args: [ - 'tsx', - 'src/dev/execution-comparison-brunch.ts', - '--workspace', - input.workspaceDir, - '--spec-id', - String(prepared.specId), - '--solution-isolation', - 'v1', - ...forbiddenRoots.flatMap((root) => ['--forbidden-root', root]), - ], - cwd: repositoryRoot(), - }, + launch: createBrunchExecutionLaunch({ + workspaceDir: input.workspaceDir, + specId: prepared.specId, + forbiddenRoots, + }), }; } diff --git a/src/dev/end-to-end-comparison/claude-adapter.ts b/src/dev/end-to-end-comparison/claude-adapter.ts index 086f4fe6d..ba1b11b1f 100644 --- a/src/dev/end-to-end-comparison/claude-adapter.ts +++ b/src/dev/end-to-end-comparison/claude-adapter.ts @@ -5,7 +5,10 @@ import { fileURLToPath } from 'node:url'; import { runCommand, type CommandResult, type CommandRunner } from '../../app/command-runner.js'; import type { ExecutionLaunch } from './brunch-adapter.js'; import { materializeExactExecutionPacket } from './public-packet.js'; -import { createClaudeSolutionIsolationPolicy } from './solution-isolation.js'; +import { + createClaudeSolutionIsolationPolicy, + type ClaudeSolutionIsolationPolicy, +} from './solution-isolation.js'; import { assertControllerIsolation } from './study-contract.js'; import { containedPath } from './validation.js'; @@ -22,6 +25,13 @@ export interface PreparedClaudeExecutionWorkspace { readonly launch: ExecutionLaunch; } +export interface ClaudeLaneReadyExecutionTarget { + readonly lane: 'claude_code'; + readonly targetDir: string; + readonly baseSha: string; + readonly launch: ExecutionLaunch; +} + export interface ClaudeExecutionRun { readonly startedAt: string; readonly endedAt: string; @@ -40,6 +50,65 @@ export interface ClaudeExecutionRun { }; } +export function createClaudeExecutionLaunch(input: { + readonly workspaceDir: string; + readonly isolationPolicy: ClaudeSolutionIsolationPolicy; +}): ExecutionLaunch { + return { + command: 'claude', + args: [ + '--print', + '--verbose', + '--output-format', + 'stream-json', + '--model', + 'claude-opus-4-8', + '--effort', + 'max', + '--permission-mode', + input.isolationPolicy.permissionMode, + '--no-session-persistence', + '--disable-slash-commands', + '--no-chrome', + '--strict-mcp-config', + '--mcp-config', + '{"mcpServers":{}}', + '--setting-sources', + '', + '--tools', + input.isolationPolicy.allowedTools.join(','), + '--allowedTools', + ...input.isolationPolicy.allowedTools, + '--disallowedTools', + 'WebFetch', + 'WebSearch', + '--settings', + JSON.stringify({ + enabledPlugins: {}, + permissions: { + allow: input.isolationPolicy.allowedTools, + deny: ['WebFetch', 'WebSearch'], + }, + sandbox: { + enabled: input.isolationPolicy.nativeSandbox.enabled, + failIfUnavailable: input.isolationPolicy.nativeSandbox.failIfUnavailable, + autoAllowBashIfSandboxed: true, + allowUnsandboxedCommands: false, + filesystem: { + denyRead: input.isolationPolicy.nativeSandbox.deniedReadRoots, + }, + network: { + allowedDomains: input.isolationPolicy.nativeSandbox.allowedDomains, + deniedDomains: input.isolationPolicy.nativeSandbox.deniedDomains, + }, + }, + }), + implementationPrompt(), + ], + cwd: input.workspaceDir, + }; +} + export async function prepareClaudeExecutionWorkspace( input: { readonly workspaceDir: string; @@ -79,59 +148,10 @@ export async function prepareClaudeExecutionWorkspace( return { workspaceDir: input.workspaceDir, baseSha, - launch: { - command: 'claude', - args: [ - '--print', - '--verbose', - '--output-format', - 'stream-json', - '--model', - 'claude-opus-4-8', - '--effort', - 'max', - '--permission-mode', - isolationPolicy.permissionMode, - '--no-session-persistence', - '--disable-slash-commands', - '--no-chrome', - '--strict-mcp-config', - '--mcp-config', - '{"mcpServers":{}}', - '--setting-sources', - '', - '--tools', - isolationPolicy.allowedTools.join(','), - '--allowedTools', - ...isolationPolicy.allowedTools, - '--disallowedTools', - 'WebFetch', - 'WebSearch', - '--settings', - JSON.stringify({ - enabledPlugins: {}, - permissions: { - allow: isolationPolicy.allowedTools, - deny: ['WebFetch', 'WebSearch'], - }, - sandbox: { - enabled: isolationPolicy.nativeSandbox.enabled, - failIfUnavailable: isolationPolicy.nativeSandbox.failIfUnavailable, - autoAllowBashIfSandboxed: true, - allowUnsandboxedCommands: false, - filesystem: { - denyRead: isolationPolicy.nativeSandbox.deniedReadRoots, - }, - network: { - allowedDomains: isolationPolicy.nativeSandbox.allowedDomains, - deniedDomains: isolationPolicy.nativeSandbox.deniedDomains, - }, - }, - }), - implementationPrompt(), - ], - cwd: input.workspaceDir, - }, + launch: createClaudeExecutionLaunch({ + workspaceDir: input.workspaceDir, + isolationPolicy, + }), }; } @@ -141,7 +161,7 @@ function repositoryRoot(): string { export async function runClaudeExecutionWorkspace( input: { - readonly prepared: PreparedClaudeExecutionWorkspace; + readonly prepared: PreparedClaudeExecutionWorkspace | ClaudeLaneReadyExecutionTarget; readonly evidenceDir: string; readonly elapsedMinutes: number; }, @@ -151,10 +171,11 @@ export async function runClaudeExecutionWorkspace( throw new Error('Claude execution budget must be a positive whole number of minutes'); } await mkdir(input.evidenceDir); + const workspaceDir = 'targetDir' in input.prepared ? input.prepared.targetDir : input.prepared.workspaceDir; const startedAt = new Date().toISOString(); // ceiling: retain at most 10 MiB per provider stream; raise or stream to disk if real runs exceed it. const result = await runner(input.prepared.launch.command, input.prepared.launch.args, { - cwd: input.prepared.workspaceDir, + cwd: workspaceDir, timeoutMs: input.elapsedMinutes * 60_000, maxOutputBytes: 10 * 1024 * 1024, }); @@ -166,7 +187,7 @@ export async function runClaudeExecutionWorkspace( let repository: ClaudeExecutionRun['repository']; try { repository = await finalizeClaudeExecutionWorkspace( - { workspaceDir: input.prepared.workspaceDir }, + { workspaceDir, baseSha: input.prepared.baseSha }, runner, ); } catch { @@ -186,6 +207,7 @@ export async function runClaudeExecutionWorkspace( export async function finalizeClaudeExecutionWorkspace( input: { readonly workspaceDir: string; + readonly baseSha?: string; }, runner: CommandRunner = runCommand, ): Promise<{ @@ -193,11 +215,17 @@ export async function finalizeClaudeExecutionWorkspace( readonly reviewSha: string; readonly finalGitRange: string; }> { - const baseSha = ( - await gitChecked(runner, input.workspaceDir, ['rev-list', '--max-parents=0', 'HEAD']) - ).stdout - .trim() - .split('\n')[0]!; + const baseSha = + input.baseSha ?? + (await gitChecked(runner, input.workspaceDir, ['rev-list', '--max-parents=0', 'HEAD'])).stdout + .trim() + .split('\n')[0]!; + const resolvedBase = ( + await gitChecked(runner, input.workspaceDir, ['rev-parse', '--verify', `${baseSha}^{commit}`]) + ).stdout.trim(); + if (resolvedBase !== baseSha) { + throw new Error('Claude execution base does not match the lane-ready descriptor'); + } const status = await gitChecked(runner, input.workspaceDir, ['status', '--porcelain']); if (status.stdout.trim().length > 0) { await gitChecked(runner, input.workspaceDir, ['add', '--all']); diff --git a/src/dev/end-to-end-comparison/pinned-source-preparation.ts b/src/dev/end-to-end-comparison/pinned-source-preparation.ts deleted file mode 100644 index f49791e00..000000000 --- a/src/dev/end-to-end-comparison/pinned-source-preparation.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { copyFile, mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; - -import { runCommand, type CommandRunner } from '../../app/command-runner.js'; -import { materializeExactExecutionPacket } from './public-packet.js'; -import { materializePinnedSourceTree } from './solution-isolation.js'; -import { assertControllerIsolation } from './study-contract.js'; - -const COMPARISON_GIT_IDENTITY = [ - '-c', - 'user.name=Brunch Comparison', - '-c', - 'user.email=brunch-comparison@invalid.local', -] as const; - -export const PINNED_DEPENDENCY_PREPARATION = { - command: 'corepack', - args: ['yarn', 'install', '--immutable', '--mode=skip-build'], -} as const; - -export interface PreparedPinnedExecutionWorkspace { - readonly lane: 'brunch' | 'claude_code'; - readonly targetDir: string; - readonly sourceCommit: string; - readonly sourceTree: string; - readonly materializedCommit: string; - readonly baseSha: string; - readonly dependencyPreparation: { - readonly command: typeof PINNED_DEPENDENCY_PREPARATION.command; - readonly args: typeof PINNED_DEPENDENCY_PREPARATION.args; - readonly status: 'passed'; - readonly exitCode: 0; - }; -} - -export async function preparePinnedExecutionWorkspace( - input: { - readonly lane: PreparedPinnedExecutionWorkspace['lane']; - readonly sourceRepositoryDir: string; - readonly sourceCommit: string; - readonly expectedSourceTree: string; - readonly targetDir: string; - readonly controllerRoot: string; - readonly specificationPath: string; - readonly publicContractTemplatePath: string; - readonly dependencyInstallRunner?: CommandRunner; - }, - runner: CommandRunner = runCommand, -): Promise { - assertControllerIsolation({ - controllerRoot: input.controllerRoot, - targetRoots: [input.targetDir], - }); - assertControllerIsolation({ - controllerRoot: input.sourceRepositoryDir, - targetRoots: [input.targetDir], - }); - const materialized = await materializePinnedSourceTree({ - sourceRepositoryDir: input.sourceRepositoryDir, - sourceCommit: input.sourceCommit, - targetDir: input.targetDir, - runner, - }); - if (materialized.sourceTree !== input.expectedSourceTree) { - await rm(input.targetDir, { recursive: true, force: true }); - throw new Error('pinned source tree does not match the frozen execution contract'); - } - - const packetRoot = await mkdtemp(join(tmpdir(), 'brunch-pinned-execution-packet-')); - try { - const packet = await materializeExactExecutionPacket({ - specificationPath: input.specificationPath, - publicContractTemplatePath: input.publicContractTemplatePath, - packetDir: join(packetRoot, 'packet'), - }); - for (const file of packet.packet.files) { - await copyFile(join(packet.packetDir, file.path), join(input.targetDir, file.path)); - } - await gitChecked(runner, input.targetDir, ['add', '--', 'public-contract.json', 'spec.md']); - await gitChecked(runner, input.targetDir, [ - ...COMPARISON_GIT_IDENTITY, - 'commit', - '-m', - 'Freeze exact comparison handoff', - ]); - const baseSha = (await gitChecked(runner, input.targetDir, ['rev-parse', 'HEAD'])).stdout.trim(); - const dependencyInstallRunner = input.dependencyInstallRunner ?? runCommand; - const installed = await dependencyInstallRunner( - PINNED_DEPENDENCY_PREPARATION.command, - PINNED_DEPENDENCY_PREPARATION.args, - { - cwd: input.targetDir, - timeoutMs: 30 * 60_000, - maxOutputBytes: 256 * 1024, - }, - ); - if (installed.exitCode !== 0) { - throw new Error( - `${PINNED_DEPENDENCY_PREPARATION.command} ${PINNED_DEPENDENCY_PREPARATION.args.join( - ' ', - )} controller dependency preparation failed (${installed.exitCode}): ${ - installed.stderr || installed.stdout - }`, - ); - } - const trackedStatus = ( - await gitChecked(runner, input.targetDir, ['status', '--porcelain', '--untracked-files=no']) - ).stdout.trim(); - if (trackedStatus.length > 0) { - throw new Error(`controller dependency preparation modified tracked source: ${trackedStatus}`); - } - const sourceIdentity = JSON.parse( - await readFile(join(input.targetDir, '.comparison-source.json'), 'utf8'), - ) as { - sourceCommit?: string; - sourceTree?: string; - }; - if ( - sourceIdentity.sourceCommit !== materialized.sourceCommit || - sourceIdentity.sourceTree !== materialized.sourceTree - ) { - throw new Error('materialized source identity drifted while freezing the execution handoff'); - } - return { - lane: input.lane, - targetDir: input.targetDir, - sourceCommit: materialized.sourceCommit, - sourceTree: materialized.sourceTree, - materializedCommit: materialized.syntheticCommit, - baseSha, - dependencyPreparation: { - ...PINNED_DEPENDENCY_PREPARATION, - status: 'passed', - exitCode: 0, - }, - }; - } catch (error) { - await rm(input.targetDir, { recursive: true, force: true }); - throw error; - } finally { - await rm(packetRoot, { recursive: true, force: true }); - } -} - -async function gitChecked( - runner: CommandRunner, - cwd: string, - args: readonly string[], -): ReturnType { - const result = await runner('git', args, { cwd }); - if (result.exitCode !== 0) { - throw new Error(`git ${args.join(' ')} failed: ${result.stderr || result.stdout}`); - } - return result; -} diff --git a/src/dev/end-to-end-comparison/solution-isolation.ts b/src/dev/end-to-end-comparison/solution-isolation.ts index 80f8e0386..415e6e60f 100644 --- a/src/dev/end-to-end-comparison/solution-isolation.ts +++ b/src/dev/end-to-end-comparison/solution-isolation.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { realpathSync } from 'node:fs'; import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -39,6 +40,14 @@ export interface MaterializedPinnedSourceTree { readonly syntheticCommit: string; } +export interface MaterializedHistoricalReplayPrefix extends MaterializedPinnedSourceTree { + readonly baseSha: string; + readonly packetFiles: readonly { + readonly path: 'public-contract.json' | 'spec.md'; + readonly sha256: string; + }[]; +} + export interface ClaudeSolutionIsolationPolicy { readonly executor: 'claude_code'; readonly recipeVersion: typeof RECIPE_VERSION; @@ -124,9 +133,11 @@ export async function materializePinnedSourceTree(input: { ).stdout.trim(); const archiveDir = await mkdtemp(join(tmpdir(), 'brunch-pinned-source-')); const archivePath = join(archiveDir, 'source.tar'); + let targetCreated = false; try { await mkdir(dirname(input.targetDir), { recursive: true }); await mkdir(input.targetDir); + targetCreated = true; await commandChecked(runner, input.sourceRepositoryDir, 'git', [ 'archive', '--format=tar', @@ -156,7 +167,9 @@ export async function materializePinnedSourceTree(input: { syntheticCommit, }; } catch (error) { - await rm(input.targetDir, { recursive: true, force: true }); + if (targetCreated) { + await rm(input.targetDir, { recursive: true, force: true }); + } throw error; } finally { await rm(archiveDir, { recursive: true, force: true }); @@ -237,7 +250,7 @@ export function assertTargetBoundedPath(policy: SolutionIsolationPolicy, request } export async function admitHistoricalReplay(input: { - readonly materialized: MaterializedPinnedSourceTree; + readonly prefix: MaterializedHistoricalReplayPrefix; readonly policies: readonly SolutionIsolationPolicy[]; readonly forbiddenRoots: readonly string[]; readonly networkProbeUrls: readonly string[]; @@ -259,15 +272,15 @@ export async function admitHistoricalReplay(input: { const runner = input.runner ?? runCommand; const reasons: IsolationAdmissionReason[] = []; inspectVerifier(input.verifier, input.forbiddenRoots, reasons); - inspectPolicies(input.policies, input.materialized.targetDir, input.forbiddenRoots, reasons); + inspectPolicies(input.policies, input.prefix.targetDir, input.forbiddenRoots, reasons); inspectForbiddenRoots(input.policies, input.forbiddenRoots, reasons); - await inspectMaterializedRepository(input.materialized, runner, reasons); - await inspectTargetSymlinks(input.materialized.targetDir, reasons); + await inspectMaterializedRepository(input.prefix, runner, reasons); + await inspectTargetSymlinks(input.prefix.targetDir, reasons); if (reasons.length > 0) throw new SolutionIsolationAdmissionError(reasons); for (const forbiddenRoot of input.forbiddenRoots) { const result = await input.verifier.run('/bin/ls', ['-la', forbiddenRoot], { - cwd: input.materialized.targetDir, + cwd: input.prefix.targetDir, timeoutMs: 15_000, maxOutputBytes: 16 * 1024, }); @@ -282,7 +295,7 @@ export async function admitHistoricalReplay(input: { } const curlReady = await input.verifier.run('/usr/bin/curl', ['--version'], { - cwd: input.materialized.targetDir, + cwd: input.prefix.targetDir, timeoutMs: 15_000, maxOutputBytes: 16 * 1024, }); @@ -294,7 +307,7 @@ export async function admitHistoricalReplay(input: { for (const url of new Set([...input.networkProbeUrls, ...REQUIRED_SOLUTION_PROBE_URLS])) { const result = await input.verifier.run('/usr/bin/curl', ['--fail', '--silent', '--show-error', url], { - cwd: input.materialized.targetDir, + cwd: input.prefix.targetDir, timeoutMs: 15_000, maxOutputBytes: 16 * 1024, }); @@ -306,7 +319,7 @@ export async function admitHistoricalReplay(input: { const localChecks = []; for (const check of input.localChecks) { const result = await input.verifier.run(check.command, check.args, { - cwd: input.materialized.targetDir, + cwd: input.prefix.targetDir, timeoutMs: 10 * 60_000, maxOutputBytes: 128 * 1024, }); @@ -324,8 +337,8 @@ export async function admitHistoricalReplay(input: { return { status: 'admitted', recipeVersion: RECIPE_VERSION, - sourceCommit: input.materialized.sourceCommit, - sourceTree: input.materialized.sourceTree, + sourceCommit: input.prefix.sourceCommit, + sourceTree: input.prefix.sourceTree, executors: input.policies.map(({ executor }) => executor), localChecks, }; @@ -452,7 +465,7 @@ function inspectForbiddenRoots( } async function inspectMaterializedRepository( - materialized: MaterializedPinnedSourceTree, + materialized: MaterializedHistoricalReplayPrefix, runner: CommandRunner, reasons: IsolationAdmissionReason[], ): Promise { @@ -472,16 +485,66 @@ async function inspectMaterializedRepository( const commitCount = ( await gitChecked(runner, materialized.targetDir, ['rev-list', '--count', 'HEAD']) ).stdout.trim(); - if (commitCount !== '1') { + if (commitCount !== '2') { reasons.push({ code: 'git_history_present', detail: `target has ${commitCount} reachable commits` }); } const head = (await gitChecked(runner, materialized.targetDir, ['rev-parse', 'HEAD'])).stdout.trim(); - if (head !== materialized.syntheticCommit) { + if (head !== materialized.baseSha) { reasons.push({ code: 'identity_mismatch', - detail: 'materialized target commit does not match declaration', + detail: 'historical replay HEAD does not match the declared packet child', }); } + const roots = ( + await gitChecked(runner, materialized.targetDir, ['rev-list', '--max-parents=0', materialized.baseSha]) + ).stdout + .trim() + .split('\n') + .filter(Boolean); + if (roots.length !== 1 || roots[0] !== materialized.syntheticCommit) { + reasons.push({ + code: 'identity_mismatch', + detail: 'historical replay root does not match the declared materialized source commit', + }); + } + const parent = ( + await gitChecked(runner, materialized.targetDir, ['rev-parse', `${materialized.baseSha}^`]) + ).stdout.trim(); + if (parent !== materialized.syntheticCommit) { + reasons.push({ + code: 'identity_mismatch', + detail: 'historical replay packet child has the wrong parent', + }); + } + const packetDelta = ( + await gitChecked(runner, materialized.targetDir, [ + 'diff', + '--name-only', + materialized.syntheticCommit, + materialized.baseSha, + ]) + ).stdout + .trim() + .split('\n') + .filter(Boolean) + .sort(); + const declaredPacketFiles = materialized.packetFiles.map(({ path }) => path).sort(); + if (!sameStrings(packetDelta, declaredPacketFiles)) { + reasons.push({ + code: 'identity_mismatch', + detail: `historical replay child is not packet-only: ${packetDelta.join(', ')}`, + }); + } + for (const file of materialized.packetFiles) { + const bytes = await readFile(join(materialized.targetDir, file.path)); + const digest = `sha256:${createHash('sha256').update(bytes).digest('hex')}`; + if (digest !== file.sha256) { + reasons.push({ + code: 'identity_mismatch', + detail: `historical replay packet drifted: ${file.path}`, + }); + } + } const remotes = (await gitChecked(runner, materialized.targetDir, ['remote'])).stdout.trim(); if (remotes.length > 0) { reasons.push({ code: 'git_remote_present', detail: remotes }); @@ -497,7 +560,7 @@ async function inspectMaterializedRepository( reasons.push({ code: 'git_ref_present', detail: extraRefs.join(', ') }); } const worktreeChanges = ( - await gitChecked(runner, materialized.targetDir, ['status', '--porcelain']) + await gitChecked(runner, materialized.targetDir, ['status', '--porcelain', '--untracked-files=no']) ).stdout.trim(); if (worktreeChanges.length > 0) { reasons.push({ code: 'git_worktree_changes_present', detail: worktreeChanges }); diff --git a/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts b/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts new file mode 100644 index 000000000..9f3ee4894 --- /dev/null +++ b/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts @@ -0,0 +1,756 @@ +import { createHash } from 'node:crypto'; +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; +import { + assertExecuteProjectionPlanReady, + projectExecuteGraph, +} from '../../../executor/execute-projection.js'; +import { openWorkspaceDb } from '../../../graph/index.js'; +import { queryGraph } from '../../../graph/queries.js'; +import { runClaudeExecutionWorkspace } from '../../end-to-end-comparison/claude-adapter.js'; +import { + createNetworkDeniedCommandRunner, + type NetworkDeniedCommandRunner, +} from '../../end-to-end-comparison/solution-isolation.js'; +import { isPetrinautOptimizationExecutionCaseContract } from '../case-contract.js'; +import { + HistoricalReplayTargetPreparationError, + prepareHistoricalReplayTarget, + type HistoricalReplayTargetDependencies, +} from '../historical-replay-target.js'; +import { prepareExecutionTarget, resolveExecutionCase } from '../operator-cli.js'; + +const roots: string[] = []; +const frozenCasesRoot = fileURLToPath( + new URL('../../../../testing/execution-comparisons/cases/', import.meta.url), +); + +afterEach(async () => { + await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); +}); + +describe('historical replay target preparation', () => { + it('admits the exact two-commit prefix and returns a launchable Brunch target without installing', async () => { + const fixture = await createBrunchFixture(); + const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); + let installCalled = false; + + const ready = await prepareHistoricalReplayTarget( + { + lane: 'brunch', + selectedCase: selected, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, + }, + portableDependencies({ + dependencyInstallRunner: async () => { + installCalled = true; + return { exitCode: 0, stdout: '', stderr: '' }; + }, + }), + ); + if (ready.lane !== 'brunch') throw new Error('expected a Brunch-ready target'); + + expect(ready).toMatchObject({ + status: 'ready', + lane: 'brunch', + caseId: 'brunch-host-landing-v1', + sourceCommit: fixture.sourceCommit, + sourceTree: fixture.sourceTree, + specId: expect.any(Number), + dependencyPreparation: { recipe: 'none', status: 'not_required' }, + isolationPolicy: { executor: 'brunch' }, + launch: { + command: 'npx', + args: expect.arrayContaining([ + '--workspace', + fixture.targetDir, + '--spec-id', + expect.any(String), + '--solution-isolation', + 'v1', + ]), + }, + }); + expect(ready.specId).toBeGreaterThan(0); + expect(ready.baseSha).toBe(await git(fixture.targetDir, ['rev-parse', 'HEAD'])); + expect(await git(fixture.targetDir, ['rev-list', '--count', ready.baseSha])).toBe('2'); + expect(await git(fixture.targetDir, ['rev-parse', `${ready.baseSha}^`])).toBe(ready.materializedCommit); + expect( + (await git(fixture.targetDir, ['diff', '--name-only', ready.materializedCommit, ready.baseSha])).split( + '\n', + ), + ).toEqual(['public-contract.json', 'spec.md']); + expect(await readFile(join(fixture.targetDir, 'spec.md'))).toEqual(fixture.specification); + expect(installCalled).toBe(false); + const graph = queryGraph(await openWorkspaceDb(fixture.targetDir), ready.specId); + expect(graph.nodes.find(({ source }) => source === 'e2e-handoff [exact-spec]')).toMatchObject({ + kind: 'requirement', + body: fixture.specification.toString('utf8'), + }); + const projection = projectExecuteGraph({ + specId: ready.specId, + graphLsn: graph.lsn, + mode: 'brownfield', + nodes: graph.nodes, + edges: graph.edges, + }); + expect(() => assertExecuteProjectionPlanReady(projection)).not.toThrow(); + }, 30_000); + + it('executes the production deep operation through an injected branded verifier factory', async () => { + const fixture = await createBrunchFixture(); + const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); + let factoryCalls = 0; + + const ready = await prepareHistoricalReplayTarget( + { + lane: 'claude_code', + selectedCase: selected, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, + }, + { + createVerifier: (forbiddenReadRoots: readonly string[]) => { + factoryCalls += 1; + return createNetworkDeniedCommandRunner({ + platform: 'darwin', + forbiddenReadRoots, + run: fakeSandboxRunner, + }); + }, + }, + ); + + expect(factoryCalls).toBe(1); + expect(ready).toMatchObject({ + status: 'ready', + lane: 'claude_code', + baseSha: expect.stringMatching(/^[a-f0-9]{40}$/u), + }); + }); + + it('rejects an injected raw verifier that lacks the existing runtime brand', async () => { + const fixture = await createBrunchFixture(); + const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); + + const rejected = prepareHistoricalReplayTarget( + { + lane: 'claude_code', + selectedCase: selected, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, + }, + portableDependencies({ + createVerifier: (forbiddenReadRoots) => + ({ + recipeVersion: 1, + platform: 'darwin', + forbiddenReadRoots, + run: fakeSandboxRunner, + }) as unknown as NetworkDeniedCommandRunner, + }), + ); + + await expect(rejected).rejects.toMatchObject({ + status: 'setup_failed', + phase: 'admission', + reasons: expect.arrayContaining([expect.objectContaining({ code: 'policy_weakened' })]), + }); + }); + + it('rejects readiness when a branded verifier reaches a required network probe', async () => { + const fixture = await createBrunchFixture(); + const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); + let reachedRequiredProbe = false; + const reachableSandboxRunner: CommandRunner = async (command, args) => { + if (command !== 'sandbox-exec') { + throw new Error(`unexpected verifier command: ${command}`); + } + const verifiedCommand = args[2]; + if (verifiedCommand === '/usr/bin/curl' && args[3] === '--version') { + return { exitCode: 0, stdout: 'curl fake\n', stderr: '' }; + } + if (verifiedCommand === '/usr/bin/curl' && args.at(-1) === 'https://github.com') { + reachedRequiredProbe = true; + return { exitCode: 0, stdout: 'reachable\n', stderr: '' }; + } + return { exitCode: 1, stdout: '', stderr: 'denied by fake sandbox\n' }; + }; + + const rejected = prepareHistoricalReplayTarget( + { + lane: 'claude_code', + selectedCase: selected, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, + }, + { + createVerifier: (forbiddenReadRoots) => + createNetworkDeniedCommandRunner({ + platform: 'darwin', + forbiddenReadRoots, + run: reachableSandboxRunner, + }), + }, + ); + + await expect(rejected).rejects.toMatchObject({ + status: 'setup_failed', + phase: 'admission', + reasons: [ + expect.objectContaining({ + code: 'network_probe_reachable', + detail: 'https://github.com', + }), + ], + }); + expect(reachedRequiredProbe).toBe(true); + await expect(readFile(join(fixture.targetDir, 'spec.md'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('dispatches pinned preparation by repository substrate for the Brunch host-landing case', async () => { + const fixture = await createBrunchFixture(); + + const ready = await prepareExecutionTarget( + { + lane: 'claude_code', + caseReference: 'brunch-host-landing', + casesRoot: fixture.casesRoot, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, + }, + portableDependencies({ + dependencyInstallRunner: async () => { + throw new Error('Brunch host landing must not install dependencies'); + }, + }), + ); + + expect(ready).toMatchObject({ + status: 'ready', + preparation: 'historical_replay', + lane: 'claude_code', + caseId: 'brunch-host-landing-v1', + dependencyPreparation: { recipe: 'none', status: 'not_required' }, + isolationPolicy: { + executor: 'claude_code', + strictMcp: true, + webTools: false, + ambientSettings: false, + ambientPlugins: false, + permissionMode: 'dontAsk', + }, + launch: { command: 'claude', cwd: fixture.targetDir }, + }); + expect('specId' in ready).toBe(false); + if (ready.preparation !== 'historical_replay' || ready.lane !== 'claude_code') { + throw new Error('expected a Claude-ready historical replay target'); + } + const run = await runClaudeExecutionWorkspace( + { + prepared: ready, + evidenceDir: join(fixture.controllerDir, 'evidence'), + elapsedMinutes: 1, + }, + async (command, args, options) => { + if (command === 'claude') { + await writeFile(join(options.cwd, 'candidate.ts'), 'export const candidate = true;\n'); + return { exitCode: 0, stdout: '{"type":"result","subtype":"success"}\n', stderr: '' }; + } + return await runCommand(command, args, options); + }, + ); + expect(run.repository?.finalGitRange).toMatch(new RegExp(`^${ready.baseSha}\\.\\.[a-f0-9]{40}$`, 'u')); + }, 30_000); + + it.each([ + ['wrong parent', rewritePrefixWithWrongParent, 'identity_mismatch'], + ['packet drift', driftPacketCommit, 'identity_mismatch'], + ['third commit', appendThirdCommit, 'git_history_present'], + ['remote and later ref', addRemoteAndRef, 'git_remote_present'], + ['escaping symlink', addEscapingSymlink, 'path_boundary_weakened'], + ] as const)( + 'rejects a %s before lane readiness and removes the partial target', + async (_name, mutate, reason) => { + const fixture = await createBrunchFixture(); + const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); + let mutated = false; + const runner: CommandRunner = async (command, args, options) => { + if ( + !mutated && + command === 'git' && + args[0] === 'rev-list' && + args[1] === '--count' && + args[2] === 'HEAD' && + options.cwd === fixture.targetDir + ) { + mutated = true; + await mutate(fixture.targetDir); + } + return await runCommand(command, args, options); + }; + + const rejected = prepareHistoricalReplayTarget( + { + lane: 'claude_code', + selectedCase: selected, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, + }, + portableDependencies({ runner }), + ); + + await expect(rejected).rejects.toMatchObject({ + status: 'setup_failed', + phase: 'admission', + reasons: expect.arrayContaining([expect.objectContaining({ code: reason })]), + }); + await expect(readFile(join(fixture.targetDir, 'spec.md'))).rejects.toMatchObject({ code: 'ENOENT' }); + }, + 30_000, + ); + + it('retains path-boundary setup evidence without returning a partial launch descriptor', async () => { + const fixture = await createBrunchFixture(); + const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); + + const rejected = prepareHistoricalReplayTarget( + { + lane: 'brunch', + selectedCase: selected, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, + forbiddenRoots: [join(fixture.targetDir, 'nested-controller')], + }, + portableDependencies(), + ); + + await expect(rejected).rejects.toBeInstanceOf(HistoricalReplayTargetPreparationError); + await expect(rejected).rejects.toMatchObject({ + status: 'setup_failed', + phase: 'admission', + reasons: expect.arrayContaining([expect.objectContaining({ code: 'path_boundary_weakened' })]), + }); + await expect(readFile(join(fixture.targetDir, 'spec.md'))).rejects.toMatchObject({ code: 'ENOENT' }); + }, 30_000); + + it('rejects a pre-existing target without deleting caller-owned files', async () => { + const fixture = await createBrunchFixture(); + const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); + await mkdir(fixture.targetDir); + await writeFile(join(fixture.targetDir, 'keep.txt'), 'caller owned\n'); + + await expect( + prepareHistoricalReplayTarget( + { + lane: 'brunch', + selectedCase: selected, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, + }, + portableDependencies(), + ), + ).rejects.toMatchObject({ + status: 'setup_failed', + phase: 'source_materialization', + }); + await expect(readFile(join(fixture.targetDir, 'keep.txt'), 'utf8')).resolves.toBe('caller owned\n'); + }); + + it('admits non-ignored untracked Petrinaut dependency artifacts when tracked source stays clean', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-replay-')); + roots.push(root); + const sourceDir = join(root, 'source'); + const controllerDir = join(root, 'controller'); + const targetDir = join(root, 'target'); + await Promise.all([mkdir(sourceDir), mkdir(controllerDir)]); + const selected = await resolveExecutionCase('petrinaut-optimization-v1', frozenCasesRoot); + if (!isPetrinautOptimizationExecutionCaseContract(selected.packet.contract)) { + throw new Error('expected the frozen Petrinaut contract'); + } + const runner = createSyntheticPinnedGitRunner({ + sourceDir, + targetDir, + sourceCommit: selected.packet.contract.case.repository.parentCommit, + sourceTree: selected.packet.contract.case.repository.parentTree, + reportUntrackedArtifact: true, + }); + const installCalls: { command: string; args: readonly string[]; cwd: string }[] = []; + + const ready = await prepareHistoricalReplayTarget( + { + lane: 'claude_code', + selectedCase: selected, + sourceRepositoryDir: sourceDir, + targetDir, + controllerRoot: controllerDir, + }, + portableDependencies({ + runner, + dependencyInstallRunner: async (command, args, options) => { + installCalls.push({ command, args, cwd: options.cwd }); + await writeFile(join(options.cwd, '.pnp.cjs'), '// generated dependency artifact\n'); + return { exitCode: 0, stdout: '', stderr: '' }; + }, + }), + ); + + expect(installCalls).toEqual([ + { + command: 'corepack', + args: ['yarn', 'install', '--immutable', '--mode=skip-build'], + cwd: targetDir, + }, + ]); + await expect(readFile(join(targetDir, '.pnp.cjs'), 'utf8')).resolves.toBe( + '// generated dependency artifact\n', + ); + expect(ready).toMatchObject({ + status: 'ready', + lane: 'claude_code', + dependencyPreparation: { + recipe: 'petrinaut-yarn-immutable-v1', + command: 'corepack', + args: ['yarn', 'install', '--immutable', '--mode=skip-build'], + status: 'passed', + exitCode: 0, + }, + isolationPolicy: { + executor: 'claude_code', + strictMcp: true, + mcpServers: [], + webTools: false, + ambientSettings: false, + ambientPlugins: false, + permissionMode: 'dontAsk', + nativeSandbox: { + enabled: true, + failIfUnavailable: true, + allowedDomains: [], + }, + }, + launch: { command: 'claude', cwd: targetDir }, + }); + expect('specId' in ready).toBe(false); + }, 30_000); + + it('rejects Petrinaut dependency preparation that mutates tracked source', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-mutation-')); + roots.push(root); + const sourceDir = join(root, 'source'); + const controllerDir = join(root, 'controller'); + const targetDir = join(root, 'target'); + await Promise.all([mkdir(sourceDir), mkdir(controllerDir)]); + const selected = await resolveExecutionCase('petrinaut-optimization-v1', frozenCasesRoot); + if (!isPetrinautOptimizationExecutionCaseContract(selected.packet.contract)) { + throw new Error('expected the frozen Petrinaut contract'); + } + + const rejected = prepareHistoricalReplayTarget( + { + lane: 'claude_code', + selectedCase: selected, + sourceRepositoryDir: sourceDir, + targetDir, + controllerRoot: controllerDir, + }, + portableDependencies({ + runner: createSyntheticPinnedGitRunner({ + sourceDir, + targetDir, + sourceCommit: selected.packet.contract.case.repository.parentCommit, + sourceTree: selected.packet.contract.case.repository.parentTree, + detectTrackedMutation: true, + }), + dependencyInstallRunner: async (_command, _args, options) => { + await writeFile(join(options.cwd, 'package.json'), '{"mutated":true}\n'); + return { exitCode: 0, stdout: '', stderr: '' }; + }, + }), + ); + + await expect(rejected).rejects.toMatchObject({ + status: 'setup_failed', + phase: 'dependency_preparation', + }); + await expect(rejected).rejects.toThrow('modified tracked source'); + await expect(readFile(join(targetDir, 'package.json'))).rejects.toMatchObject({ code: 'ENOENT' }); + }, 30_000); +}); + +async function createBrunchFixture(): Promise<{ + readonly casesRoot: string; + readonly controllerDir: string; + readonly sourceDir: string; + readonly sourceCommit: string; + readonly sourceTree: string; + readonly targetDir: string; + readonly specification: Buffer; +}> { + const root = await mkdtemp(join(tmpdir(), 'brunch-historical-replay-')); + roots.push(root); + const sourceDir = join(root, 'source'); + const controllerDir = join(root, 'controller'); + const casesRoot = join(root, 'cases'); + const caseDir = join(casesRoot, 'brunch-host-landing'); + const targetDir = join(root, 'target'); + await Promise.all([mkdir(sourceDir), mkdir(controllerDir), mkdir(caseDir, { recursive: true })]); + await writeFile(join(sourceDir, 'package.json'), '{"name":"historical-source","private":true}\n'); + await writeFile(join(sourceDir, 'source.ts'), 'export const historical = true;\n'); + await git(sourceDir, ['init', '--initial-branch=main']); + await git(sourceDir, ['add', '--all']); + await git(sourceDir, [ + '-c', + 'user.name=Historical Fixture', + '-c', + 'user.email=historical@example.invalid', + 'commit', + '-m', + 'Pinned source', + ]); + const sourceCommit = await git(sourceDir, ['rev-parse', 'HEAD']); + const sourceTree = await git(sourceDir, ['rev-parse', 'HEAD^{tree}']); + const specification = Buffer.from('# Exact approved host landing specification\n\nBytes survive. \n'); + await writeFile(join(caseDir, 'spec.md'), specification); + await writeFile( + join(caseDir, 'public-contract.json'), + `${JSON.stringify( + { + schemaVersion: 1, + case: { + id: 'brunch-host-landing-v1', + specification: 'spec.md', + specificationSha256: createHash('sha256').update(specification).digest('hex'), + provider: 'anthropic', + model: 'claude-opus-4-8', + product: 'brunch', + mode: 'brownfield', + scope: 'single_feature', + surface: 'backend', + repository: { + substrate: 'pinned_git', + parentCommit: sourceCommit, + parentTree: sourceTree, + }, + }, + budgets: { + elapsedMinutes: 90, + mechanicalInterventions: 2, + substantiveHumanInterventions: 0, + }, + delivery: { + runtimeNetwork: 'forbidden', + dependencyInstallNetwork: 'forbidden', + }, + acceptance: { + publicCommand: '/brunch:land', + executionTerminal: 'promotion_prepared', + }, + rules: ['Work only in the target repository.', 'Stop after promotion_prepared without landing.'], + }, + null, + 2, + )}\n`, + ); + return { + casesRoot, + controllerDir, + sourceDir, + sourceCommit, + sourceTree, + targetDir, + specification, + }; +} + +async function git(cwd: string, args: readonly string[]): Promise { + const result = await runCommand('git', args, { cwd }); + if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); + return result.stdout.trim(); +} + +async function appendThirdCommit(targetDir: string): Promise { + await git(targetDir, [ + '-c', + 'user.name=Historical Rival', + '-c', + 'user.email=historical-rival@example.invalid', + 'commit', + '--allow-empty', + '-m', + 'Third commit rival', + ]); +} + +async function driftPacketCommit(targetDir: string): Promise { + await writeFile(join(targetDir, 'spec.md'), '# Drifted packet\n'); + await git(targetDir, ['add', '--', 'spec.md']); + await git(targetDir, [ + '-c', + 'user.name=Historical Rival', + '-c', + 'user.email=historical-rival@example.invalid', + 'commit', + '--amend', + '--no-edit', + ]); +} + +async function rewritePrefixWithWrongParent(targetDir: string): Promise { + const originalRootTree = await git(targetDir, ['rev-parse', 'HEAD^^{tree}']); + const finalTree = await git(targetDir, ['rev-parse', 'HEAD^{tree}']); + const replacementRoot = await git(targetDir, [ + '-c', + 'user.name=Historical Rival', + '-c', + 'user.email=historical-rival@example.invalid', + 'commit-tree', + originalRootTree, + '-m', + 'Replacement root', + ]); + const replacementChild = await git(targetDir, [ + '-c', + 'user.name=Historical Rival', + '-c', + 'user.email=historical-rival@example.invalid', + 'commit-tree', + finalTree, + '-p', + replacementRoot, + '-m', + 'Replacement packet child', + ]); + await git(targetDir, ['update-ref', 'refs/heads/main', replacementChild]); +} + +async function addRemoteAndRef(targetDir: string): Promise { + await git(targetDir, ['remote', 'add', 'origin', 'https://github.com/hashintel/brunch.git']); + await git(targetDir, ['branch', 'later-solution']); +} + +async function addEscapingSymlink(targetDir: string): Promise { + await symlink('../controller', join(targetDir, 'escaped-controller')); +} + +function createSyntheticPinnedGitRunner(input: { + readonly sourceDir: string; + readonly targetDir: string; + readonly sourceCommit: string; + readonly sourceTree: string; + readonly detectTrackedMutation?: boolean; + readonly reportUntrackedArtifact?: boolean; +}): CommandRunner { + const materializedCommit = 'a'.repeat(40); + const baseSha = 'b'.repeat(40); + let commits = 0; + return async (command, args, options) => { + if (command === 'tar') { + await writeFile(join(options.cwd, 'package.json'), '{"private":true}\n'); + return { exitCode: 0, stdout: '', stderr: '' }; + } + if (command !== 'git') { + return await runCommand(command, args, options); + } + if (args[0] === 'archive') { + const output = args.find((arg) => arg.startsWith('--output='))?.slice('--output='.length); + if (output === undefined) throw new Error('synthetic archive call omitted --output'); + await writeFile(output, 'synthetic archive'); + return { exitCode: 0, stdout: '', stderr: '' }; + } + if (args[0] === 'rev-parse' && options.cwd === input.sourceDir) { + const revision = args.at(-1); + if (revision === `${input.sourceCommit}^{commit}`) { + return { exitCode: 0, stdout: `${input.sourceCommit}\n`, stderr: '' }; + } + if (revision === `${input.sourceCommit}^{tree}`) { + return { exitCode: 0, stdout: `${input.sourceTree}\n`, stderr: '' }; + } + } + if (args.includes('commit')) { + commits += 1; + return { exitCode: 0, stdout: '', stderr: '' }; + } + if (args[0] === 'rev-parse') { + const revision = args.at(-1); + if (revision === 'HEAD') { + return { + exitCode: 0, + stdout: `${commits === 1 ? materializedCommit : baseSha}\n`, + stderr: '', + }; + } + if (revision === `${baseSha}^`) { + return { exitCode: 0, stdout: `${materializedCommit}\n`, stderr: '' }; + } + } + if (args[0] === 'rev-list' && args[1] === '--count') { + return { exitCode: 0, stdout: '2\n', stderr: '' }; + } + if (args[0] === 'rev-list' && args[1] === '--max-parents=0') { + return { exitCode: 0, stdout: `${materializedCommit}\n`, stderr: '' }; + } + if (args[0] === 'diff') { + return { exitCode: 0, stdout: 'public-contract.json\nspec.md\n', stderr: '' }; + } + if (args[0] === 'for-each-ref') { + return { exitCode: 0, stdout: 'refs/heads/main\n', stderr: '' }; + } + if (args[0] === 'status') { + if ( + input.detectTrackedMutation === true && + (await readFile(join(input.targetDir, 'package.json'), 'utf8')) !== '{"private":true}\n' + ) { + return { exitCode: 0, stdout: ' M package.json\n', stderr: '' }; + } + if (input.reportUntrackedArtifact === true && !args.includes('--untracked-files=no')) { + return { exitCode: 0, stdout: '?? .pnp.cjs\n', stderr: '' }; + } + return { exitCode: 0, stdout: '', stderr: '' }; + } + if (args[0] === 'remote') { + return { exitCode: 0, stdout: '', stderr: '' }; + } + return { exitCode: 0, stdout: '', stderr: '' }; + }; +} + +function portableDependencies( + overrides: HistoricalReplayTargetDependencies = {}, +): HistoricalReplayTargetDependencies { + return { + createVerifier: (forbiddenReadRoots) => + createNetworkDeniedCommandRunner({ + platform: 'darwin', + forbiddenReadRoots, + run: fakeSandboxRunner, + }), + ...overrides, + }; +} + +const fakeSandboxRunner: CommandRunner = async (command, args) => { + if (command !== 'sandbox-exec') { + throw new Error(`unexpected verifier command: ${command}`); + } + const verifiedCommand = args[2]; + if (verifiedCommand === '/usr/bin/curl' && args[3] === '--version') { + return { exitCode: 0, stdout: 'curl fake\n', stderr: '' }; + } + return { exitCode: 1, stdout: '', stderr: 'denied by fake sandbox\n' }; +}; diff --git a/src/dev/execution-comparison/__tests__/operator-cli.test.ts b/src/dev/execution-comparison/__tests__/operator-cli.test.ts index 2c4bb6936..f24f84b44 100644 --- a/src/dev/execution-comparison/__tests__/operator-cli.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-cli.test.ts @@ -5,24 +5,12 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; -import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; -import { - assertExecuteProjectionPlanReady, - projectExecuteGraph, -} from '../../../executor/execute-projection.js'; -import { openWorkspaceDb } from '../../../graph/index.js'; -import { queryGraph } from '../../../graph/queries.js'; -import { - parseExecutionComparisonArgs, - resolvePinnedBrunchPreflight, -} from '../../execution-comparison-brunch.js'; import { runExecutionComparisonOperatorCli } from '../../execution-comparison-operator.js'; import { listExecutionCases, prepareExecutionTarget, resolveExecutionCase } from '../operator-cli.js'; const casesRoot = fileURLToPath(new URL('../../../../testing/execution-comparisons/cases/', import.meta.url)); const frozenCase = join(casesRoot, 'minimal-petri-net-editor'); const controllerRoot = fileURLToPath(new URL('../../../../', import.meta.url)); -const hashSourceRepository = fileURLToPath(new URL('../../../../../hash/', import.meta.url)); describe('execution comparison operator case selection', () => { it('lists and resolves eligible case ids only inside the cases root', async () => { @@ -151,7 +139,7 @@ describe('execution comparison target preparation', () => { '--target', join(root, 'petri'), '--source-repository', - hashSourceRepository, + controllerRoot, ]), ).rejects.toThrow('valid only for pinned execution cases'); } finally { @@ -167,161 +155,67 @@ describe('execution comparison target preparation', () => { await symlink(sourceRoot, sourceLink); try { await expect( - prepareExecutionTarget({ - lane: 'claude_code', - caseReference: 'petrinaut-optimization-v1', - casesRoot, - targetDir: join(root, 'relative-source-target'), - controllerRoot, - sourceRepositoryDir: 'relative-source', - dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), - }), + prepareExecutionTarget( + { + lane: 'claude_code', + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir: join(root, 'relative-source-target'), + controllerRoot, + sourceRepositoryDir: 'relative-source', + }, + { + dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + }, + ), ).rejects.toThrow('must be an absolute path'); await expect( - prepareExecutionTarget({ - lane: 'claude_code', - caseReference: 'petrinaut-optimization-v1', - casesRoot, - targetDir: join(controllerRoot, '.unsafe-petrinaut-target'), - controllerRoot, - sourceRepositoryDir: sourceRoot, - dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), - }), + prepareExecutionTarget( + { + lane: 'claude_code', + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir: join(controllerRoot, '.unsafe-petrinaut-target'), + controllerRoot, + sourceRepositoryDir: sourceRoot, + }, + { + dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + }, + ), ).rejects.toThrow('controller and target roots must be disjoint'); await expect( - prepareExecutionTarget({ - lane: 'claude_code', - caseReference: 'petrinaut-optimization-v1', - casesRoot, - targetDir: join(root, 'linked-source-target'), - controllerRoot, - sourceRepositoryDir: sourceLink, - dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), - }), + prepareExecutionTarget( + { + lane: 'claude_code', + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir: join(root, 'linked-source-target'), + controllerRoot, + sourceRepositoryDir: sourceLink, + }, + { + dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + }, + ), ).rejects.toThrow('real directory, not a symlink'); await expect( - prepareExecutionTarget({ - lane: 'claude_code', - caseReference: 'petrinaut-optimization-v1', - casesRoot, - targetDir: join(sourceRoot, 'target'), - controllerRoot, - sourceRepositoryDir: sourceRoot, - dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), - }), + prepareExecutionTarget( + { + lane: 'claude_code', + caseReference: 'petrinaut-optimization-v1', + casesRoot, + targetDir: join(sourceRoot, 'target'), + controllerRoot, + sourceRepositoryDir: sourceRoot, + }, + { + dependencyInstallRunner: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + }, + ), ).rejects.toThrow('controller and target roots must be disjoint'); } finally { await rm(root, { recursive: true, force: true }); } }); - - it.each(['brunch', 'claude_code'] as const)( - 'materializes and controller-installs the pinned Petrinaut source for %s', - async (lane) => { - const root = await mkdtemp(join(tmpdir(), `brunch-petrinaut-operator-${lane}-`)); - const targetDir = join(root, 'target'); - const installCalls: { command: string; args: readonly string[]; cwd: string }[] = []; - const installRunner: CommandRunner = async (command, args, options) => { - installCalls.push({ command, args, cwd: options.cwd }); - return { exitCode: 0, stdout: '', stderr: '' }; - }; - try { - const prepared = await prepareExecutionTarget({ - lane, - caseReference: 'petrinaut-optimization-v1', - casesRoot, - targetDir, - controllerRoot, - sourceRepositoryDir: hashSourceRepository, - dependencyInstallRunner: installRunner, - }); - - expect(prepared).toMatchObject({ - lane, - caseId: 'petrinaut-optimization-v1', - sourceCommit: '5c7a2d9db5caa851c38938f4b1bac19005b0e978', - sourceTree: 'a3e08cf75e00cc9016c931f4665341506e03533e', - dependencyPreparation: { - command: 'corepack', - args: ['yarn', 'install', '--immutable', '--mode=skip-build'], - status: 'passed', - exitCode: 0, - }, - }); - expect(installCalls).toEqual([ - { - command: 'corepack', - args: ['yarn', 'install', '--immutable', '--mode=skip-build'], - cwd: targetDir, - }, - ]); - expect(await readFile(join(targetDir, 'spec.md'))).toEqual( - await readFile(join(casesRoot, 'petrinaut-optimization', 'spec.md')), - ); - expect(await readFile(join(targetDir, 'public-contract.json'))).toEqual( - await readFile(join(casesRoot, 'petrinaut-optimization', 'public-contract.json')), - ); - expect(await git(targetDir, ['rev-list', '--count', 'HEAD'])).toBe('2'); - expect(await git(targetDir, ['remote'])).toBe(''); - expect(await git(targetDir, ['for-each-ref', '--format=%(refname)'])).toBe('refs/heads/main'); - expect(await git(targetDir, ['status', '--porcelain', '--untracked-files=no'])).toBe(''); - if (lane === 'brunch') { - expect(prepared).toMatchObject({ lane: 'brunch', specId: expect.any(Number) }); - if (!('specId' in prepared)) throw new Error('pinned Brunch preparation omitted specId'); - expect(prepared.specId).toBeGreaterThan(0); - await expect( - resolvePinnedBrunchPreflight({ - workspaceDir: prepared.targetDir, - specId: prepared.specId, - }), - ).resolves.toEqual({ - action: 'newSession', - specId: prepared.specId, - }); - expect( - parseExecutionComparisonArgs([ - '--workspace', - prepared.targetDir, - '--spec-id', - String(prepared.specId), - '--solution-isolation', - 'v1', - '--forbidden-root', - controllerRoot, - ]), - ).toEqual({ - workspaceDir: prepared.targetDir, - specId: prepared.specId, - forbiddenRoots: [controllerRoot], - solutionIsolation: 'v1', - }); - const db = await openWorkspaceDb(targetDir); - const graph = queryGraph(db, prepared.specId); - expect(graph.nodes.find(({ source }) => source === 'e2e-handoff [exact-spec]')).toMatchObject({ - kind: 'requirement', - body: await readFile(join(targetDir, 'spec.md'), 'utf8'), - }); - const projection = projectExecuteGraph({ - specId: prepared.specId, - graphLsn: graph.lsn, - mode: 'brownfield', - nodes: graph.nodes, - edges: graph.edges, - }); - expect(() => assertExecuteProjectionPlanReady(projection)).not.toThrow(); - } else { - expect('specId' in prepared).toBe(false); - } - } finally { - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); - } - }, - 120_000, - ); }); - -async function git(cwd: string, args: readonly string[]): Promise { - const result = await runCommand('git', args, { cwd }); - if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); - return result.stdout.trim(); -} diff --git a/src/dev/execution-comparison/__tests__/pinned-source-preparation.test.ts b/src/dev/execution-comparison/__tests__/pinned-source-preparation.test.ts deleted file mode 100644 index 3af888ee0..000000000 --- a/src/dev/execution-comparison/__tests__/pinned-source-preparation.test.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -import { afterAll, describe, expect, it } from 'vitest'; - -import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; -import { - PINNED_DEPENDENCY_PREPARATION, - preparePinnedExecutionWorkspace, -} from '../../end-to-end-comparison/pinned-source-preparation.js'; - -const HASH_PARENT_COMMIT = '5c7a2d9db5caa851c38938f4b1bac19005b0e978'; -const HASH_PARENT_TREE = 'a3e08cf75e00cc9016c931f4665341506e03533e'; -const sourceRepositoryDir = fileURLToPath(new URL('../../../../../hash/', import.meta.url)); -const contractTemplatePath = fileURLToPath( - new URL( - '../../../../testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json', - import.meta.url, - ), -); -const roots: string[] = []; - -afterAll(async () => { - await Promise.all( - roots - .splice(0) - .map(async (root) => await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 })), - ); -}, 60_000); - -describe('pinned HASH source preparation', () => { - it('gives Brunch and Claude separate history-free materializations of the same full parent tree', async () => { - const root = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-pinned-source-')); - roots.push(root); - const controllerRoot = join(root, 'controller'); - const handoffRoot = join(root, 'handoff'); - await Promise.all([mkdir(controllerRoot), mkdir(handoffRoot)]); - const specification = Buffer.from('# Exact approved specification\n\nSpacing survives. \n'); - const specificationPath = join(handoffRoot, 'spec.md'); - await writeFile(specificationPath, specification); - const installs: { command: string; args: readonly string[]; cwd: string }[] = []; - const dependencyInstallRunner: CommandRunner = async (command, args, options) => { - installs.push({ command, args, cwd: options.cwd }); - return { exitCode: 0, stdout: '', stderr: '' }; - }; - - const prepared = await Promise.all( - (['brunch', 'claude_code'] as const).map( - async (lane) => - await preparePinnedExecutionWorkspace({ - lane, - sourceRepositoryDir, - sourceCommit: HASH_PARENT_COMMIT, - expectedSourceTree: HASH_PARENT_TREE, - targetDir: join(root, lane), - controllerRoot, - specificationPath, - publicContractTemplatePath: contractTemplatePath, - dependencyInstallRunner, - }), - ), - ); - - for (const target of prepared) { - expect(target).toMatchObject({ - sourceCommit: HASH_PARENT_COMMIT, - sourceTree: HASH_PARENT_TREE, - lane: expect.stringMatching(/^(?:brunch|claude_code)$/u), - }); - expect(await git(target.targetDir, ['rev-list', '--count', 'HEAD'])).toBe('2'); - expect(await git(target.targetDir, ['rev-parse', `${target.materializedCommit}^{tree}`])).not.toBe( - target.sourceTree, - ); - expect(await git(target.targetDir, ['remote'])).toBe(''); - expect(await git(target.targetDir, ['for-each-ref', '--format=%(refname)'])).toBe('refs/heads/main'); - expect(await git(target.targetDir, ['status', '--porcelain'])).toBe(''); - expect( - (await git(target.targetDir, ['diff', '--name-only', target.materializedCommit, target.baseSha])) - .split('\n') - .filter(Boolean) - .sort(), - ).toEqual(['public-contract.json', 'spec.md']); - expect(await readFile(join(target.targetDir, 'spec.md'))).toEqual(specification); - expect(JSON.parse(await readFile(join(target.targetDir, '.comparison-source.json'), 'utf8'))).toEqual({ - recipeVersion: 1, - sourceCommit: HASH_PARENT_COMMIT, - sourceTree: HASH_PARENT_TREE, - }); - expect(target.baseSha).not.toBe(target.materializedCommit); - expect(target.dependencyPreparation).toEqual({ - ...PINNED_DEPENDENCY_PREPARATION, - status: 'passed', - exitCode: 0, - }); - } - expect(prepared.map(({ targetDir }) => targetDir)).toEqual([ - join(root, 'brunch'), - join(root, 'claude_code'), - ]); - expect(installs.sort((left, right) => left.cwd.localeCompare(right.cwd))).toEqual( - prepared - .map(({ targetDir }) => ({ - command: 'corepack', - args: ['yarn', 'install', '--immutable', '--mode=skip-build'], - cwd: targetDir, - })) - .sort((left, right) => left.cwd.localeCompare(right.cwd)), - ); - }, 120_000); - - it.each([ - [ - 'install failure', - async (): Promise => async () => ({ - exitCode: 23, - stdout: '', - stderr: 'immutable install failed', - }), - 'controller dependency preparation failed (23)', - ], - [ - 'tracked mutation', - async (): Promise => async (_command, _args, options) => { - await writeFile(join(options.cwd, 'package.json'), '{"mutated":true}\n'); - return { exitCode: 0, stdout: '', stderr: '' }; - }, - 'modified tracked source', - ], - ])('fails closed and removes the target after %s', async (_name, createRunner, message) => { - const root = await mkdtemp(join(tmpdir(), 'brunch-pinned-install-failure-')); - roots.push(root); - const source = await createTinyPinnedSource(root); - const controllerRoot = join(root, 'controller'); - const handoffRoot = join(root, 'handoff'); - const targetDir = join(root, 'target'); - await Promise.all([mkdir(controllerRoot), mkdir(handoffRoot)]); - const specificationPath = join(handoffRoot, 'spec.md'); - await writeFile(specificationPath, '# Exact approved specification\n'); - - await expect( - preparePinnedExecutionWorkspace({ - lane: 'claude_code', - sourceRepositoryDir: source.repositoryDir, - sourceCommit: source.commit, - expectedSourceTree: source.tree, - targetDir, - controllerRoot, - specificationPath, - publicContractTemplatePath: contractTemplatePath, - dependencyInstallRunner: await createRunner(), - }), - ).rejects.toThrow(message); - await expect(readFile(join(targetDir, 'package.json'))).rejects.toMatchObject({ code: 'ENOENT' }); - }); -}); - -async function git(cwd: string, args: readonly string[]): Promise { - const result = await runCommand('git', args, { cwd }); - if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); - return result.stdout.trim(); -} - -async function createTinyPinnedSource(root: string): Promise<{ - readonly repositoryDir: string; - readonly commit: string; - readonly tree: string; -}> { - const repositoryDir = join(root, 'source'); - await mkdir(repositoryDir); - await writeFile(join(repositoryDir, 'package.json'), '{"private":true}\n'); - await git(repositoryDir, ['init', '--initial-branch=main']); - await git(repositoryDir, ['add', '--all']); - await git(repositoryDir, [ - '-c', - 'user.name=Comparison Test', - '-c', - 'user.email=comparison@example.invalid', - 'commit', - '-m', - 'source', - ]); - return { - repositoryDir, - commit: await git(repositoryDir, ['rev-parse', 'HEAD']), - tree: await git(repositoryDir, ['rev-parse', 'HEAD^{tree}']), - }; -} diff --git a/src/dev/execution-comparison/case-contract.ts b/src/dev/execution-comparison/case-contract.ts index f01cb0c23..145c453c9 100644 --- a/src/dev/execution-comparison/case-contract.ts +++ b/src/dev/execution-comparison/case-contract.ts @@ -128,6 +128,13 @@ export type ExecutionCasePublicContract = | BrunchHostLandingExecutionCasePublicContract | PetrinautOptimizationExecutionCasePublicContract; +export type PinnedExecutionCasePublicContract = Extract< + ExecutionCasePublicContract, + { readonly case: { readonly repository: { readonly substrate: 'pinned_git' } } } +>; + +export type PinnedExecutionCaseId = PinnedExecutionCasePublicContract['case']['id']; + export function isBrowserExecutionCaseContract( value: ExecutionCasePublicContract, ): value is BrowserExecutionCasePublicContract { diff --git a/src/dev/execution-comparison/historical-replay-target.ts b/src/dev/execution-comparison/historical-replay-target.ts new file mode 100644 index 000000000..7d7077bd2 --- /dev/null +++ b/src/dev/execution-comparison/historical-replay-target.ts @@ -0,0 +1,367 @@ +import { copyFile, lstat, mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { runCommand, type CommandRunner } from '../../app/command-runner.js'; +import { + createBrunchExecutionLaunch, + type ExecutionLaunch, +} from '../end-to-end-comparison/brunch-adapter.js'; +import { createClaudeExecutionLaunch } from '../end-to-end-comparison/claude-adapter.js'; +import { materializeExactExecutionPacket } from '../end-to-end-comparison/public-packet.js'; +import { + admitHistoricalReplay, + createBrunchSolutionIsolationPolicy, + createClaudeSolutionIsolationPolicy, + createNetworkDeniedCommandRunner, + materializePinnedSourceTree, + SolutionIsolationAdmissionError, + type BrunchSolutionIsolationPolicy, + type ClaudeSolutionIsolationPolicy, + type IsolationAdmissionReason, + type NetworkDeniedCommandRunner, +} from '../end-to-end-comparison/solution-isolation.js'; +import { assertControllerIsolation } from '../end-to-end-comparison/study-contract.js'; +import { seedBrownfieldBrunchExecutionWorkspace } from './brunch-lane.js'; +import { + isBrowserExecutionCaseContract, + type PinnedExecutionCaseId, + type PublicCasePacket, +} from './case-contract.js'; + +const RECIPE_VERSION = 1 as const; +const COMPARISON_GIT_IDENTITY = [ + '-c', + 'user.name=Brunch Comparison', + '-c', + 'user.email=brunch-comparison@invalid.local', +] as const; +const NO_DEPENDENCY_RECIPE = { recipe: 'none' } as const; +const PETRINAUT_DEPENDENCY_RECIPE = { + recipe: 'petrinaut-yarn-immutable-v1', + command: 'corepack', + args: ['yarn', 'install', '--immutable', '--mode=skip-build'], +} as const; +type HistoricalReplayDependencyRecipe = typeof NO_DEPENDENCY_RECIPE | typeof PETRINAUT_DEPENDENCY_RECIPE; +const DEPENDENCY_RECIPE_BY_PINNED_CASE = { + 'brunch-host-landing-v1': NO_DEPENDENCY_RECIPE, + 'petrinaut-optimization-v1': PETRINAUT_DEPENDENCY_RECIPE, +} as const satisfies Record; + +type HistoricalReplayPreparationPhase = + | 'source_materialization' + | 'packet_freeze' + | 'dependency_preparation' + | 'admission' + | 'lane_finalization'; + +interface HistoricalReplayReadyBase { + readonly status: 'ready'; + readonly recipeVersion: typeof RECIPE_VERSION; + readonly caseId: string; + readonly targetDir: string; + readonly sourceCommit: string; + readonly sourceTree: string; + readonly materializedCommit: string; + readonly baseSha: string; + readonly dependencyPreparation: + | { + readonly recipe: 'none'; + readonly status: 'not_required'; + } + | { + readonly recipe: typeof PETRINAUT_DEPENDENCY_RECIPE.recipe; + readonly command: typeof PETRINAUT_DEPENDENCY_RECIPE.command; + readonly args: typeof PETRINAUT_DEPENDENCY_RECIPE.args; + readonly status: 'passed'; + readonly exitCode: 0; + }; + readonly launch: ExecutionLaunch; +} + +export interface BrunchHistoricalReplayReady extends HistoricalReplayReadyBase { + readonly lane: 'brunch'; + readonly specId: number; + readonly isolationPolicy: BrunchSolutionIsolationPolicy; +} + +export interface ClaudeHistoricalReplayReady extends HistoricalReplayReadyBase { + readonly lane: 'claude_code'; + readonly isolationPolicy: ClaudeSolutionIsolationPolicy; +} + +export type HistoricalReplayReady = BrunchHistoricalReplayReady | ClaudeHistoricalReplayReady; + +export interface HistoricalReplayCaseSelection { + readonly caseDir: string; + readonly packet: PublicCasePacket; +} + +export interface HistoricalReplayTargetDependencies { + readonly runner?: CommandRunner; + readonly dependencyInstallRunner?: CommandRunner; + readonly createVerifier?: (forbiddenReadRoots: readonly string[]) => NetworkDeniedCommandRunner; +} + +export class HistoricalReplayTargetPreparationError extends Error { + readonly status = 'setup_failed' as const; + readonly phase: HistoricalReplayPreparationPhase; + readonly reasons: readonly IsolationAdmissionReason[]; + override readonly cause: unknown; + + constructor(phase: HistoricalReplayPreparationPhase, cause: unknown) { + super(`historical replay target preparation failed during ${phase}: ${errorMessage(cause)}`, { + cause, + }); + this.name = 'HistoricalReplayTargetPreparationError'; + this.phase = phase; + this.cause = cause; + this.reasons = cause instanceof SolutionIsolationAdmissionError ? cause.reasons : []; + } +} + +export async function prepareHistoricalReplayTarget( + input: { + readonly lane: HistoricalReplayReady['lane']; + readonly selectedCase: HistoricalReplayCaseSelection; + readonly sourceRepositoryDir: string; + readonly targetDir: string; + readonly controllerRoot: string; + readonly forbiddenRoots?: readonly string[]; + }, + dependencies: HistoricalReplayTargetDependencies = {}, +): Promise { + const runner = dependencies.runner ?? runCommand; + let phase: HistoricalReplayPreparationPhase = 'source_materialization'; + let targetOwned = false; + try { + await validateInput(input); + const contract = input.selectedCase.packet.contract; + if (isBrowserExecutionCaseContract(contract) || contract.case.repository.substrate !== 'pinned_git') { + throw new Error('historical replay preparation requires a pinned brownfield case'); + } + const forbiddenRoots = uniqueRoots([ + input.sourceRepositoryDir, + input.controllerRoot, + repositoryRoot(), + ...(input.forbiddenRoots ?? []), + ]); + const materialized = await materializePinnedSourceTree({ + sourceRepositoryDir: input.sourceRepositoryDir, + sourceCommit: contract.case.repository.parentCommit, + targetDir: input.targetDir, + runner, + }); + targetOwned = true; + if ( + materialized.sourceCommit !== contract.case.repository.parentCommit || + materialized.sourceTree !== contract.case.repository.parentTree + ) { + throw new Error('pinned source identity does not match the frozen execution contract'); + } + + phase = 'packet_freeze'; + const packetRoot = await mkdtemp(join(tmpdir(), 'brunch-historical-replay-packet-')); + let packetFiles: typeof input.selectedCase.packet.files; + try { + const packet = await materializeExactExecutionPacket({ + specificationPath: join( + input.selectedCase.caseDir, + input.selectedCase.packet.contract.case.specification, + ), + publicContractTemplatePath: join(input.selectedCase.caseDir, 'public-contract.json'), + packetDir: join(packetRoot, 'packet'), + }); + if (packet.packet.packetSha256 !== input.selectedCase.packet.packetSha256) { + throw new Error('exact execution packet drifted from the frozen selected case'); + } + packetFiles = packet.packet.files; + for (const file of packetFiles) { + await copyFile(join(packet.packetDir, file.path), join(input.targetDir, file.path)); + } + } finally { + await rm(packetRoot, { recursive: true, force: true }); + } + await gitChecked(runner, input.targetDir, ['add', '--', 'public-contract.json', 'spec.md']); + await gitChecked(runner, input.targetDir, [ + ...COMPARISON_GIT_IDENTITY, + 'commit', + '-m', + 'Freeze exact comparison handoff', + ]); + const baseSha = (await gitChecked(runner, input.targetDir, ['rev-parse', 'HEAD'])).stdout.trim(); + + phase = 'dependency_preparation'; + const dependencyRecipe = DEPENDENCY_RECIPE_BY_PINNED_CASE[contract.case.id]; + const dependencyPreparation = + dependencyRecipe.recipe === 'none' + ? ({ ...dependencyRecipe, status: 'not_required' } as const) + : await preparePetrinautDependencies({ + targetDir: input.targetDir, + runner, + dependencyInstallRunner: dependencies.dependencyInstallRunner ?? runCommand, + }); + + phase = 'admission'; + const claudePolicy = createClaudeSolutionIsolationPolicy(input.targetDir, forbiddenRoots); + const brunchPolicy = createBrunchSolutionIsolationPolicy(input.targetDir); + await admitHistoricalReplay({ + prefix: { + ...materialized, + baseSha, + packetFiles, + }, + policies: [claudePolicy, brunchPolicy], + forbiddenRoots, + networkProbeUrls: [], + verifier: + dependencies.createVerifier?.(forbiddenRoots) ?? + createNetworkDeniedCommandRunner({ forbiddenReadRoots: forbiddenRoots }), + localChecks: [], + runner, + }); + + phase = 'lane_finalization'; + const common = { + status: 'ready' as const, + recipeVersion: RECIPE_VERSION, + caseId: contract.case.id, + targetDir: input.targetDir, + sourceCommit: materialized.sourceCommit, + sourceTree: materialized.sourceTree, + materializedCommit: materialized.syntheticCommit, + baseSha, + dependencyPreparation, + }; + if (input.lane === 'brunch') { + const seeded = await seedBrownfieldBrunchExecutionWorkspace({ workspaceDir: input.targetDir }); + await assertTrackedSourceClean(runner, input.targetDir, 'Brunch graph preparation'); + return { + ...common, + lane: 'brunch', + specId: seeded.specId, + isolationPolicy: brunchPolicy, + launch: createBrunchExecutionLaunch({ + workspaceDir: input.targetDir, + specId: seeded.specId, + forbiddenRoots, + }), + }; + } + return { + ...common, + lane: 'claude_code', + isolationPolicy: claudePolicy, + launch: createClaudeExecutionLaunch({ + workspaceDir: input.targetDir, + isolationPolicy: claudePolicy, + }), + }; + } catch (error) { + if (targetOwned) { + await rm(input.targetDir, { recursive: true, force: true }); + } + throw new HistoricalReplayTargetPreparationError(phase, error); + } +} + +async function preparePetrinautDependencies(input: { + readonly targetDir: string; + readonly runner: CommandRunner; + readonly dependencyInstallRunner: CommandRunner; +}): Promise<{ + readonly recipe: typeof PETRINAUT_DEPENDENCY_RECIPE.recipe; + readonly command: typeof PETRINAUT_DEPENDENCY_RECIPE.command; + readonly args: typeof PETRINAUT_DEPENDENCY_RECIPE.args; + readonly status: 'passed'; + readonly exitCode: 0; +}> { + const result = await input.dependencyInstallRunner( + PETRINAUT_DEPENDENCY_RECIPE.command, + PETRINAUT_DEPENDENCY_RECIPE.args, + { + cwd: input.targetDir, + timeoutMs: 30 * 60_000, + maxOutputBytes: 256 * 1024, + }, + ); + if (result.exitCode !== 0) { + throw new Error( + `${PETRINAUT_DEPENDENCY_RECIPE.command} ${PETRINAUT_DEPENDENCY_RECIPE.args.join( + ' ', + )} controller dependency preparation failed (${result.exitCode}): ${result.stderr || result.stdout}`, + ); + } + await assertTrackedSourceClean(input.runner, input.targetDir, 'controller dependency preparation'); + return { + ...PETRINAUT_DEPENDENCY_RECIPE, + status: 'passed', + exitCode: 0, + }; +} + +async function assertTrackedSourceClean( + runner: CommandRunner, + targetDir: string, + owner: string, +): Promise { + const trackedStatus = ( + await gitChecked(runner, targetDir, ['status', '--porcelain', '--untracked-files=no']) + ).stdout.trim(); + if (trackedStatus.length > 0) { + throw new Error(`${owner} modified tracked source: ${trackedStatus}`); + } +} + +async function validateInput(input: { + readonly sourceRepositoryDir: string; + readonly targetDir: string; + readonly controllerRoot: string; +}): Promise { + if (!isAbsolute(input.sourceRepositoryDir)) { + throw new Error('pinned source repository must be an absolute path'); + } + if (!isAbsolute(input.targetDir)) { + throw new Error('historical replay target must be an absolute path'); + } + if (!isAbsolute(input.controllerRoot)) { + throw new Error('historical replay preparation requires an absolute controller root'); + } + const source = await lstat(input.sourceRepositoryDir); + if (!source.isDirectory() || source.isSymbolicLink()) { + throw new Error('pinned source repository must be a real directory, not a symlink'); + } + assertControllerIsolation({ + controllerRoot: input.controllerRoot, + targetRoots: [input.targetDir], + }); + assertControllerIsolation({ + controllerRoot: input.sourceRepositoryDir, + targetRoots: [input.targetDir], + }); +} + +async function gitChecked( + runner: CommandRunner, + cwd: string, + args: readonly string[], +): ReturnType { + const result = await runner('git', args, { cwd }); + if (result.exitCode !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${result.stderr || result.stdout}`); + } + return result; +} + +function uniqueRoots(roots: readonly string[]): string[] { + return [...new Set(roots.map((root) => resolve(root)))]; +} + +function repositoryRoot(): string { + return fileURLToPath(new URL('../../../', import.meta.url)); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/dev/execution-comparison/operator-cli.ts b/src/dev/execution-comparison/operator-cli.ts index f3255f257..c721e9afe 100644 --- a/src/dev/execution-comparison/operator-cli.ts +++ b/src/dev/execution-comparison/operator-cli.ts @@ -1,19 +1,19 @@ import { execFile } from 'node:child_process'; -import { lstat, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; import { isAbsolute, join } from 'node:path'; import { promisify } from 'node:util'; -import type { CommandRunner } from '../../app/command-runner.js'; +import { prepareBrunchExecutionWorkspace } from './brunch-lane.js'; import { - preparePinnedExecutionWorkspace, - type PreparedPinnedExecutionWorkspace, -} from '../end-to-end-comparison/pinned-source-preparation.js'; -import { prepareBrunchExecutionWorkspace, seedBrownfieldBrunchExecutionWorkspace } from './brunch-lane.js'; -import { - isPetrinautOptimizationExecutionCaseContract, + isBrowserExecutionCaseContract, loadPublicCasePacket, type PublicCasePacket, } from './case-contract.js'; +import { + prepareHistoricalReplayTarget, + type HistoricalReplayReady, + type HistoricalReplayTargetDependencies, +} from './historical-replay-target.js'; const execFileAsync = promisify(execFile); const SAFE_CASE_ID = /^[a-z0-9][a-z0-9-]*$/u; @@ -48,15 +48,8 @@ export type PreparedExecutionTarget = readonly baseSha: string; }) | (ResolvedExecutionCase & - Omit & { - readonly preparation: 'pinned_git'; - readonly lane: 'brunch'; - readonly specId: number; - }) - | (ResolvedExecutionCase & - Omit & { - readonly preparation: 'pinned_git'; - readonly lane: 'claude_code'; + HistoricalReplayReady & { + readonly preparation: 'historical_replay'; }); export async function listExecutionCases(casesRoot: string): Promise { @@ -99,75 +92,48 @@ export async function resolveExecutionCase( }; } -export async function prepareExecutionTarget(input: { - readonly lane: 'brunch' | 'claude_code'; - readonly caseReference: string; - readonly casesRoot: string; - readonly targetDir: string; - readonly controllerRoot?: string; - readonly sourceRepositoryDir?: string; - readonly dependencyInstallRunner?: CommandRunner; -}): Promise { +export async function prepareExecutionTarget( + input: { + readonly lane: 'brunch' | 'claude_code'; + readonly caseReference: string; + readonly casesRoot: string; + readonly targetDir: string; + readonly controllerRoot?: string; + readonly sourceRepositoryDir?: string; + }, + dependencies: HistoricalReplayTargetDependencies = {}, +): Promise { const selected = await resolveExecutionCase(input.caseReference, input.casesRoot); - if (isPetrinautOptimizationExecutionCaseContract(selected.packet.contract)) { + const contract = selected.packet.contract; + if (!isBrowserExecutionCaseContract(contract) && contract.case.repository.substrate === 'pinned_git') { if (input.sourceRepositoryDir === undefined) { throw new Error('pinned execution case requires --source-repository'); } - if (!isAbsolute(input.sourceRepositoryDir)) { - throw new Error('pinned source repository must be an absolute path'); - } - const sourceRepository = await lstat(input.sourceRepositoryDir); - if (!sourceRepository.isDirectory() || sourceRepository.isSymbolicLink()) { - throw new Error('pinned source repository must be a real directory, not a symlink'); - } - if (input.controllerRoot === undefined || !isAbsolute(input.controllerRoot)) { + if (input.controllerRoot === undefined) { throw new Error('pinned execution case requires an absolute controller root'); } - const prepared = await preparePinnedExecutionWorkspace({ - lane: input.lane, - sourceRepositoryDir: input.sourceRepositoryDir, - sourceCommit: selected.packet.contract.case.repository.parentCommit, - expectedSourceTree: selected.packet.contract.case.repository.parentTree, - targetDir: input.targetDir, - controllerRoot: input.controllerRoot, - specificationPath: join(selected.caseDir, selected.packet.contract.case.specification), - publicContractTemplatePath: join(selected.caseDir, 'public-contract.json'), - ...(input.dependencyInstallRunner === undefined - ? {} - : { dependencyInstallRunner: input.dependencyInstallRunner }), - }); - if (prepared.lane === 'brunch') { - try { - const seeded = await seedBrownfieldBrunchExecutionWorkspace({ - workspaceDir: prepared.targetDir, - }); - const trackedStatus = await gitOutput( - ['status', '--porcelain', '--untracked-files=no'], - prepared.targetDir, - ); - if (trackedStatus.length > 0) { - throw new Error(`Brunch graph preparation modified tracked source: ${trackedStatus}`); - } - return { - ...selected, - ...prepared, - preparation: 'pinned_git', - lane: 'brunch', - specId: seeded.specId, - }; - } catch (error) { - await rm(prepared.targetDir, { recursive: true, force: true }); - throw error; - } - } + const prepared = await prepareHistoricalReplayTarget( + { + lane: input.lane, + selectedCase: selected, + sourceRepositoryDir: input.sourceRepositoryDir, + targetDir: input.targetDir, + controllerRoot: input.controllerRoot, + }, + dependencies, + ); return { ...selected, ...prepared, - preparation: 'pinned_git', - lane: 'claude_code', + preparation: 'historical_replay', }; } - if (input.sourceRepositoryDir !== undefined || input.dependencyInstallRunner !== undefined) { + if ( + input.sourceRepositoryDir !== undefined || + dependencies.runner !== undefined || + dependencies.dependencyInstallRunner !== undefined || + dependencies.createVerifier !== undefined + ) { throw new Error('--source-repository is valid only for pinned execution cases'); } if (input.lane === 'brunch') { From 665bee510cde00a004c165abbf0c7f558039db01 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 13:05:41 +0200 Subject: [PATCH 03/15] FE-1241: Make isolation tests host-independent Exercise the branded sandbox contract with a portable test runner so Linux CI does not invoke the macOS-only production verifier. Co-authored-by: Cursor --- .../__tests__/solution-isolation.test.ts | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts b/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts index 58b4e3413..eb7dcaf62 100644 --- a/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts @@ -41,6 +41,32 @@ async function git(cwd: string, args: readonly string[]): Promise { return result.stdout.trim(); } +function createPortableTestVerifier(forbiddenReadRoots: readonly string[]): NetworkDeniedCommandRunner { + return createNetworkDeniedCommandRunner({ + platform: 'darwin', + forbiddenReadRoots, + run: async (command, args, options) => { + if (command !== 'sandbox-exec') { + throw new Error(`unexpected verifier command: ${command}`); + } + const verifiedCommand = args[2]; + const verifiedArgs = args.slice(3); + if (verifiedCommand === '/usr/bin/curl') { + return args[3] === '--version' + ? { exitCode: 0, stdout: 'curl fake\n', stderr: '' } + : { exitCode: 1, stdout: '', stderr: 'network denied by fake sandbox\n' }; + } + if (verifiedCommand === '/bin/ls') { + return { exitCode: 1, stdout: '', stderr: 'path denied by fake sandbox\n' }; + } + if (verifiedCommand === undefined) { + throw new Error('test verifier received no target command'); + } + return await runCommand(verifiedCommand, verifiedArgs, options); + }, + }); +} + async function sourceRepository(root: string): Promise<{ readonly repositoryDir: string; readonly sourceCommit: string; @@ -149,7 +175,7 @@ describe('brownfield historical-solution isolation', () => { policies: strictPolicies(targetDir, forbiddenRoots), forbiddenRoots, networkProbeUrls: [probeUrl, 'https://github.com', 'https://linear.app', 'https://www.notion.so'], - verifier: createNetworkDeniedCommandRunner({ forbiddenReadRoots: forbiddenRoots }), + verifier: createPortableTestVerifier(forbiddenRoots), localChecks: [{ command: process.execPath, args: ['verify.mjs'] }], }); @@ -181,7 +207,7 @@ describe('brownfield historical-solution isolation', () => { prefix, forbiddenRoots, networkProbeUrls: [probeUrl, 'https://github.com', 'https://linear.app', 'https://www.notion.so'], - verifier: createNetworkDeniedCommandRunner({ forbiddenReadRoots: forbiddenRoots }), + verifier: createPortableTestVerifier(forbiddenRoots), localChecks: [{ command: process.execPath, args: ['verify.mjs'] }], } as const; From 3f751f4eb3d6fa6108c704d9292c3eded1bcf35e Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 20:58:24 +0200 Subject: [PATCH 04/15] FE-1241: Calibrate Petrinaut historical replay Align the frozen mission with merged reference behavior while preserving greenfield access, isolated runtime bounds, and portable CI verification. Co-authored-by: Cursor --- .github/workflows/test.yml | 5 + docs/praxis/comparison-runs.md | 33 + memory/PLAN.md | 4 +- memory/SPEC.md | 3 +- ...son-cases--petrinaut-provider-preflight.md | 224 ++++++ ...son-cases--petrinaut-reference-fidelity.md | 203 +++++ .../__tests__/agent-runtime-runtime.test.ts | 53 +- .../extensions/agent-runtime/runtime/index.ts | 16 +- src/dev/TOPOLOGY.md | 14 +- .../__tests__/execution-adapters.test.ts | 60 +- .../end-to-end-comparison/claude-adapter.ts | 44 +- src/dev/execution-comparison-operator.ts | 66 +- .../__tests__/case-contract.test.ts | 42 +- .../execution-comparison-brunch.test.ts | 4 +- .../historical-replay-target.test.ts | 18 +- .../__tests__/operator-cli.test.ts | 143 +++- .../petrinaut-historical-preflight.test.ts | 691 +++++++++++++++++ ...petrinaut-optimization-oracle.slow.test.ts | 74 +- .../petrinaut-optimization-oracle.test.ts | 67 +- src/dev/execution-comparison/case-contract.ts | 116 ++- .../historical-replay-target.ts | 122 ++- .../petrinaut-historical-preflight.ts | 707 ++++++++++++++++++ .../evidence.ts | 255 +++++++ .../petrinaut-optimization-oracle.ts | 1 + .../petrinaut-optimization-oracle/browser.ts | 369 ++++++--- .../calibration-seed.json | 165 ++++ .../petrinaut-optimization-oracle/claims.ts | 6 +- .../fake-optimizer.ts | 55 +- .../petrinaut-optimization-oracle/fixture.ts | 284 +++++-- .../petrinaut-optimization-oracle/runner.ts | 17 +- .../controller/requirement-registry.json | 6 +- .../study-contract.json | 4 +- .../public-contract.json | 95 ++- .../cases/petrinaut-optimization/spec.md | 7 +- 34 files changed, 3587 insertions(+), 386 deletions(-) create mode 100644 memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md create mode 100644 memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md create mode 100644 src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts create mode 100644 src/dev/execution-comparison/petrinaut-historical-preflight.ts create mode 100644 src/dev/execution-comparison/petrinaut-historical-preflight/evidence.ts create mode 100644 src/dev/execution-comparison/petrinaut-optimization-oracle/calibration-seed.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9abe38661..977889c54 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,11 @@ jobs: git config --global user.name "Brunch CI" git config --global init.defaultBranch main + - name: Install PTY test dependency + run: | + sudo apt-get update + sudo apt-get install --yes expect + - name: Install run: npm ci diff --git a/docs/praxis/comparison-runs.md b/docs/praxis/comparison-runs.md index f19fc31f8..5dcf5f48a 100644 --- a/docs/praxis/comparison-runs.md +++ b/docs/praxis/comparison-runs.md @@ -88,6 +88,39 @@ This is developer/evaluation tooling under `.pi/prompts/`, not a shipped Brunch Brunch stops at `promotion_prepared`; the operator never invokes `/brunch:land`, scores a lane, chooses a winner, or treats Brunch-only run/Petri/debug evidence as common comparison evidence. +### Petrinaut historical setup preflight + +Before spending a provider turn on the frozen Petrinaut case, run the controller-only preflight with +an explicit real HASH source checkout and three disjoint scratch roots: + +```sh +npx tsx src/dev/execution-comparison-operator.ts petrinaut-preflight \ + --source-repository /absolute/path/to/hash \ + --parent-target /absolute/disposable-work/parent \ + --reference-target /absolute/disposable-work/reference \ + --output-root /absolute/retained-evidence +``` + +The command accepts no provider, reference commit, dependency command, manifest command, or oracle +plugin. It prepares the frozen parent through D137-L, calibrates only the code-owned merged PR #9051 +reference with the same compiled immutable install and unchanged +`petrinaut-optimization-oracles-v1`, writes one path-redacted receipt plus bounded write-once +`parent-dependency.json`, `reference-dependency.json`, and `oracle-summary.json` files for each reached +phase, and removes both owned workspaces. Receipt evidence entries contain only fixed relative +filenames, SHA-256, byte count, and truncation state. The receipt is evidence, not an admission token. +`setup_failed` or `assertion_failed` blocks historical provider work; never bypass the failed phase or +replace an invalid receipt. + +The 2026-07-22 real-source run is currently blocked on historical browser semantics: both immutable +installs, tracked-source checks, and the six focused builds (including the directly declared +Refractive build immediately before Petrinaut UI) pass. Browser navigation uses bounded +`domcontentloaded` readiness independently from the unchanged 5-second semantic action budget, and +the real run retains no failed request or navigation timeout. The pinned historical shell does not +expose the fixture-authored exact `Optimizations` heading / `Create optimization` control contract; +its optimization view labels the action `Create`. Re-entry must respecify real public semantics before +provider work rather than weakening locators in place. Exact retained evidence is recorded in the +[active scope card](../../memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md). + ## End-to-end comparison tracer The end-to-end tracer composes the rigorous elicitation recipe with the execution-comparison contracts diff --git a/memory/PLAN.md b/memory/PLAN.md index 260de00a7..2b8fb9f02 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -467,7 +467,7 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Linear / branch:** [FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases), child of FE-1211; `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240. - **Kind:** structural evaluation expansion — parameterized case/substrate/oracle composition plus two repository-backed end-to-end cases; one frontier with several scoped slices, not one frontier per case. - **Certainty:** proving. -- **Status:** active 2026-07-22; D137-L is materialized as the sole pinned preparation path and its admission contracts are hardened. The production operation validates the declared root plus packet-only child, chooses from an exhaustive compile-time recipe map with no default, admits the complete prefix under the strict asymmetric Claude/Brunch policy pair, and returns lane-ready launch metadata; Brunch additionally returns an Execute-plan-ready brownfield `specId`. D136-L's compiled immutable install may leave non-ignored untracked dependency artifacts, while tracked source or packet mutation still rejects. Self-contained rivals retain wrong-parent, packet-drift, third-commit, remote/ref, path-escape, raw-verifier, and reachable-network rejection before launch. Both frozen case/oracle slices remain built; A50-L remains invalidated; D136-L's standalone `/optimization` hard gate and non-gating `/processes/draft` outer evidence remain unchanged. Historical provider runs remain unexecuted. +- **Status:** blocked 2026-07-22 after the diagnostic real no-provider Petrinaut preflight. D137-L is materialized as the sole pinned preparation path and its admission contracts are hardened. The controller preflight operation and CLI now compose the real parent preparation, disjoint controller-only merged-reference materialization, the same compiled immutable recipe, unchanged focused oracle, write-once path-redacted receipt plus bounded retained phase evidence, and owned cleanup without a provider turn. The real frozen parent and merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0` both passed immutable install and tracked-source cleanliness. The focused oracle includes the directly declared Refractive workspace build before Petrinaut UI; all six preparation steps pass without broad graph preparation. Browser navigation now separates a 30-second `domcontentloaded` budget from 5-second semantic actions, and real evidence contains no aborted request or navigation timeout. Calibration still returns `assertion_failed`: the exact `Optimizations` heading is absent and every downstream check cannot find an exact `Create optimization` button. Pinned historical source proves that the real view labels its action `Create` and that `/optimization` mounts the ordinary local-storage Petrinaut shell, so the synthetic fixture's semantic contract requires respecification before provider work. Receipt SHA-256 `0ddbd4bffd4e57ce7426bc254bd8780e6d399c1eaf98f27499c261ac174a8046` retains the invalid assertion result and all three bounded evidence files, and both workspaces were removed. - **Objective:** prove that FE-1239's exact elicitation → immutable handoff → crossed Brunch/Claude execution → requirement-ledger path works against real pinned brownfield repositories without weakening controller isolation or turning the Petri tracer into generic campaign machinery. - **Lights up:** pinned repository + private feature mission + shared public baseline → two codebase-aware elicitation lanes → two exact handoffs applied to the same base tree → four isolated execution cells → repository-specific controller oracle → validity-first requirement ledger, once for Brunch and once for Petrinaut. - **Stabilizes:** one data-driven case profile across `greenfield` and `brownfield`; immutable repository/base-tree identity beside immutable spec identity; clean same-base materialization for every lane; an exhaustive code-owned dependency recipe per pinned case, including the closed controller-only immutable install before runtime denial; pinned Brunch graph seeding and launch identity without tracked-source mutation; compile-time oracle dispatch; and diff/evidence collection that excludes harness seed files without widening FE-1230's `ExecutionAttempt`. @@ -482,7 +482,7 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Cross-cutting obligations:** public baseline controls only addressability needed by the common oracle and is never credited as elicitation gain; existing repository knowledge may be inspected equally by both elicitation lanes; private mission/reveal/oracle files never enter target workspaces; failed and invalid attempts remain retained; Brunch stops at `promotion_prepared` and never lands into either source repository; provider/model/budget/intervention rules remain frozen per study. - **Explicitly out:** Clay/Pi Campaign Machine; any second greenfield case; whole-repository rewrites; evolving-plan studies; reliability repetitions; automatic scoring or cross-case winner; live production deployment; Cursor/Codex lanes; arbitrary oracle plugins; production `/compare-end-to-end`; landing provider outputs into either source repository; and FE-1230 `ExecutionAttempt` schema widening. - **Traceability:** D70-L fixture taxonomy; D134-L/I67-L controller/mission isolation; D40-L/D120-L/I62-L execution and no-landing boundaries; FE-1210/FE-1230/FE-1232/FE-1239/FE-1240; [`docs/praxis/comparison-runs.md`](../docs/praxis/comparison-runs.md); Notion `brunch Test Plan` axes. -- **Current execution pointer:** none; the admission-contract hardening card is consumed. Next scope the fresh historical provider preflight for actual HASH commit/tree resolution, real immutable install, and focused-route setup validity; separately named host-integration evidence remains non-gating. +- **Current execution pointer:** [`memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md`](cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md) is active. It owns D138-L's one pre-provider source-fidelity audit and rebaseline, typed mechanical-address contract/hash regeneration, synthetic sensitivity rivals, and unchanged real merged-reference pass. The prior [`petrinaut-provider-preflight`](cards/brownfield-comparison-cases--petrinaut-provider-preflight.md) remains the retained blocked receipt/evidence source; no provider matrix may start until the new card closes. ### comparison-reporting-skills diff --git a/memory/SPEC.md b/memory/SPEC.md index d1665a027..43b0df6e9 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -306,8 +306,9 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D132-L | Standalone interactive web uses one cwd-scoped combined Brunch host with a target-addressed inventory of sealed, in-process Pi `AgentSession`s (2026-07-14). The same process serves React assets/WebSocket Brunch RPC and owns coordinator/graph authority; it does not construct `InteractiveMode`, expose raw Pi RPC, spawn one Pi child per session, host multiple projects, or promise in-flight survival across host restart. One durable session target has one driver/many observers and cannot be opened as duplicate writable runtimes; write leases wait for real same-session contention. Hosted-session mutations return the complete `LiveSessionHostResult` discriminated `{status}` union as JSON-RPC success payloads, including domain refusals; only malformed boundary input and thrown host failures use JSON-RPC errors. FE-1200 materialized the one-target path and validated simultaneous target isolation (A43-L). Depends on: D5-L, D10-L, D33-L, D39-L, D84-L; req 4, req 31. Supersedes: D10-L/D72-L read-only-sidecar posture and D84-L singleton/TUI-owned target topology. | [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md), [`src/session/TOPOLOGY.md`](../src/session/TOPOLOGY.md), [`src/rpc/TOPOLOGY.md`](../src/rpc/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — target-addressed host and concurrent-session isolation materialized 2026-07-14 | | D133-L | Web and TUI share transport-neutral presentation semantics, not platform components or separately-authored domain decoders (2026-07-14). Validated Brunch `toolResult.details` project to a shared semantic presentation model; LLM-context, TUI, and React adapters render that meaning for their audiences. Web hydrates from a named JSONL-derived product projection and overlays target-addressed live events, then refetches canonical truth at settlement/reconnect. The host emits neither ANSI/TUI strings nor ready-made HTML, and no chat mirror/event store is introduced. FE-1200 materialized the full required persisted family inventory: ordinary text; free-text, choice, choices, and bounded-questionnaire terminal read-back; candidate, review-set, and digest offers/continuations; and receipt-bearing review settlement. Live React controls answer free text and listed single/multi choices; bounded questionnaires remain answerable headlessly through D38-L's schema-tagged string/JSON envelope, without a dedicated React questionnaire form. Depends on: A44-L, D17-L, D19-L, D104-L; req 12, req 17, req 32. | [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — full required-family coverage materialized 2026-07-15 | | D134-L | The approachable `/compare-specs` control topology is one top-level project Pi session acting as the simulated user and driving exactly one comparison-harness interactive subshell at a time. It does not spawn a Pi actor that then opens a nested interactive shell. The top-level agent alone receives the private mission; each harness receives only its approved minimal framing and the user's natural messages. Shared top-level context and lane order are acceptable and disclosed for this exploratory workflow; isolation-sensitive studies continue to use FE-1210's separate rigorous campaign recipe. Ordinary text interaction is the stock-Pi baseline for choices and approvals; a custom structured-question tool may enhance presentation but is never required. Setup checks are bounded to actual selected-harness prerequisites—no throwaway Pi/Claude provider turns or synthetic actor launches on every run. Supersedes: FE-1215's unmaterialized fresh-nested-actor design note. | [`.pi/prompts/compare-specs.md`](../.pi/prompts/compare-specs.md), [`testing/comparisons/missions/README.md`](../testing/comparisons/missions/README.md); PLAN `operator-comparison-workflow` | active — remediation materialized and focused Brunch smoke witnessed 2026-07-17; full comparison witness deferred | -| D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller preparation boundary. Before the network-denied candidate lane, the controller materializes the declared parent tree and exact handoff, then runs only `corepack yarn install --immutable --mode=skip-build`; this closed immutable install is the sole package-registry-network allowance and must leave tracked source clean. After the lane terminates at `promotion_prepared`, the controller runs the closed focused package builds and launches the standalone website `/optimization` route. A deterministic loopback fake optimizer grades capability-present UI, scenario-first configuration, fixed/optimized flat bindings, one saved/custom metric plus direction, request construction, progressive trials/best/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and accessibility. The real HASH `/processes/draft` host/iframe integration is not a common hard gate: its global Next shell requires unrelated GraphQL/codegen and broad 61-package platform/toolchain preparation, so one fully provisioned host/iframe smoke plus masked code/visual review remain explicit non-gating outer evidence and report `not_assessable` when unavailable. Depends on: A50-L, D134-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases`; Petrinaut scope card | active — materialized 2026-07-22 | +| D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller preparation boundary. Before the network-denied candidate lane, the controller materializes the declared parent tree and exact handoff, then runs only `corepack yarn install --immutable --mode=skip-build`; this closed immutable install is the sole package-registry-network allowance and must leave tracked source clean. After the lane terminates at `promotion_prepared`, the controller runs the closed focused package builds and launches the standalone website `/optimization` route. A deterministic loopback fake optimizer grades capability-present UI, scenario-first configuration, fixed/optimized flat bindings, one saved/custom metric plus direction, request construction, progressive trials/best/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and the source task's observable accessibility semantics. D138-L's merged-reference calibration is the authority for whether this historical replay's mechanical contract is setup-valid; synthetic fixtures prove sensitivity but cannot invent exact labels, roles, or interaction shapes absent from the source task/reference. The real HASH `/processes/draft` host/iframe integration is not a common hard gate: its global Next shell requires unrelated GraphQL/codegen and broad 61-package platform/toolchain preparation, so one fully provisioned host/iframe smoke plus masked code/visual review remain explicit non-gating outer evidence and report `not_assessable` when unavailable. Depends on: A50-L, D134-L, D138-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases`; Petrinaut scope card | active — materialized 2026-07-22; reference-fidelity correction active | | D137-L | Historical replay preparation has one deep public operation that owns the security-sensitive order from pinned-source materialization through exact handoff, closed case-owned dependency preparation, strict admission, and lane finalization (2026-07-22). Its final Git authority is the declared two-commit synthetic prefix clarified under A49-L; admission validates the root source identity, packet-only child, strict Claude/Brunch policy pair, network/path probes, and absence of any third history, remote, ref, residue, or escaping symlink. Public callers receive only a lane-discriminated ready descriptor (`Brunch` includes the seeded brownfield `specId`; Claude does not); private stage types may enforce internal ordering, but no public phase machine, arbitrary command/plugin surface, or serializable security token is introduced. A compile-time case registry selects either no dependency preparation for Brunch host landing or D136-L's sole immutable Petrinaut recipe. Depends on: A49-L retirement, D136-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases` | active — materialized 2026-07-22 | +| D138-L | A historical replay profile must pass a no-provider-turn behavioral calibration against its declared merged reference before its first provider attempt (2026-07-22). The source issue/PR and merged behavior—not a synthetic fixture—own the replay task's observable semantics; the merged patch is calibration evidence, never a golden diff or architecture oracle. Before any attempt exists, a contradicted public packet/oracle may be rebaselined to those source semantics with all hashes and rivals regenerated. Synthetic fixtures remain sensitivity rivals and may not author exact labels, roles, control shapes, or stronger requirements absent from the replay source. Any reference mismatch blocks provider work rather than being relabeled setup-valid; after the first provider attempt, frozen packet/oracle bytes may not be rewritten. Depends on: D70-L, D136-L, D137-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases`; Petrinaut provider-preflight card | active — Petrinaut reference-fidelity rebaseline selected 2026-07-22 | | D60-L | Agent context splits into pull / projection / render / surface, distinguishes graph-truth from active-context reads, and keeps `workspace.state` separate. Agent context (what the agent reasons over) spans `cwd` (filesystem kickoff heuristic — `.brunch?`, session count/length, README/markdown sizes, file counts), `graph` (overview/list/query), and `node` (variable-hop neighborhood). ... | [`src/graph/TOPOLOGY.md`](../src/graph/TOPOLOGY.md), [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md) | active | | D83-L | Context-render house style: a markdown frame (md-pen) with TOON for uniform data and a fenced ASCII tree for hierarchy, wrapped in `
` tags; agent context clusters into `` / `` / `` scopes. Refines D60-L's RENDER stage. LLM-facing agent-context renders adopt one consistent dialect instead of ad-hoc `[bracket]` + bullet lists: - **Audience scope.** `src/agents/contexts/` owns the LLM agent-context dialect (the `
` scope-clustering plus TOON data blocks and t ... | See archive snapshot for full rationale. | active | | D85-L | Suspended prompt-resource axis model: strategy/lens/method are no longer runtime state. A 2026-06-18 grill consolidation of the `agents/skills/` topology and the D58-L manifest axes, implemented across FE-893, FE-861, and FE-898, produced useful prompt-resource content and path topology. ... | [`src/agents/TOPOLOGY.md`](../src/agents/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md) | retired/suspended | diff --git a/memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md b/memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md new file mode 100644 index 000000000..b1f50e74a --- /dev/null +++ b/memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md @@ -0,0 +1,224 @@ +# Prove the real Petrinaut provider preflight + +Frontier: brownfield-comparison-cases +Status: blocked +Mode: single +Created: 2026-07-22 + +Posture: proving (inherited from `brownfield-comparison-cases`). + +## Orientation + +- **Containing seam:** D136-L's real HASH preparation and standalone `/optimization` mechanical hard gate, behind D137-L's admitted lane-ready target. +- **Frontier:** FE-1241 `brownfield-comparison-cases`; this remains on `ka/fe-1241-brownfield-comparison-cases`. +- **Volatile state:** synthetic Petrinaut preparation/oracle proofs are green; no retained controller receipt proves the frozen identities, real immutable install, and real merged-reference route together. +- **Main risk:** interpreting a provider candidate failure when the real HASH setup or oracle calibration is itself invalid. + +## Target Behavior + +The controller emits a setup-valid Petrinaut preflight receipt only after the frozen real HASH parent and a disjoint controller-only merged reference pass their distinct preparation and calibration gates. + +## Cold-start reads + +- `memory/SPEC.md` — A49-L retirement clarification; D136-L; D137-L +- `memory/PLAN.md` — frontier `brownfield-comparison-cases` +- `src/dev/TOPOLOGY.md` — Historical Replay Isolation and Brownfield Comparison Oracles +- `memory/cards/brownfield-comparison-cases--admission-contract-hardening.md` — remaining provider-preflight debt +- `src/dev/execution-comparison/historical-replay-target.ts` — real parent preparation/admission +- `src/dev/execution-comparison-operator.ts` and `operator-cli.ts` — controller command surface +- `src/dev/execution-comparison/petrinaut-optimization-oracle.ts` and private subtree — focused builds/browser calibration +- `testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json` — frozen parent identity +- `testing/execution-comparisons/cases/petrinaut-optimization/controller/oracle-manifest.json` — hard-gate identity + +## Boundary Crossings + +```text +→ explicit safe real HASH source checkout +→ frozen parent commit/tree resolution +→ D137 lane-ready parent preparation with the real immutable install +→ disjoint controller-only merged-reference materialization +→ the same code-owned immutable install +→ closed Petrinaut focused-build/browser oracle +→ redacted write-once receipt + bounded hashed logs +→ cleanup of both owned workspaces +``` + +The parent workspace is the only future provider target. The merged-reference workspace calibrates controller setup and oracle sensitivity; it is always a forbidden root and is never returned as lane-ready input. + +## Risks and Assumptions + +- **RISK:** reference calibration leaks historical solution material into the provider target. + - **MITIGATION:** use disjoint roots, include the reference root in admission denial, retain no absolute path in the receipt, and scan the target/receipt for reference commit and path leakage. +- **RISK:** the heavy real install mutates tracked source or depends on ambient tooling. + - **MITIGATION:** reuse the one code-owned immutable recipe, retain bounded stdout/stderr digests, and fail before calibration/provider work on nonzero exit or tracked mutation. +- **RISK:** the current synthetic oracle does not match the real merged UI. + - **MITIGATION:** run the unchanged compile-time oracle against the controller-only merged reference; any setup or claim failure invalidates the preflight instead of weakening the oracle. +- **ASSUMPTION:** the real merged PR #9051 tree can satisfy D136-L's focused builds and standalone `/optimization` route under the immutable install. + - **IMPACT IF FALSE:** historical provider runs remain blocked and the Petrinaut oracle/profile must be respecified or corrected. + - **VALIDATE:** this slice's real no-provider-turn preflight is the falsifier. + +## Posture check + +This tracer scores on all proving axes: + +- **Proof of life:** the production parent preparation and real merged-reference browser oracle run through one controller command. +- **Invariant:** the historical reference remains controller-only while the parent target receives strict admission. +- **Uncertainty:** setup-validity is settled before provider budget or candidate interpretation. + +No separate spike is cheaper: the real preflight command is itself the minimum experiment and leaves a structured receipt. + +## Acceptance Criteria + +```text +Petrinaut real-source preflight +├── controller composition +│ ├── ✓ petrinaut-historical-preflight.test.ts — the closed case resolves the frozen parent and code-owned merged-reference commit +│ ├── ✓ operator-cli.test.ts — preflight requires absolute disjoint source/work/output roots and dispatches no arbitrary command +│ └── ✓ petrinaut-historical-preflight.test.ts — command trace contains no Claude or Brunch provider-lane launch +├── parent gate +│ ├── ✓ petrinaut-historical-preflight.test.ts — D137 returns a lane-ready parent with exact commit/tree, packet child, admission, and real-recipe result +│ └── ✓ petrinaut-historical-preflight.test.ts — parent setup failure yields setup_failed and no calibration +├── controller-only calibration +│ ├── ✓ petrinaut-historical-preflight.test.ts — reference materialization is disjoint and never returned as lane-ready +│ ├── ✓ petrinaut-historical-preflight.test.ts — install/build/oracle failure yields setup_failed rather than candidate assertion evidence +│ └── ✓ real preflight command — unchanged petrinaut-optimization-oracles-v1 passes against merged PR #9051 +├── retained evidence +│ ├── ✓ petrinaut-historical-preflight.test.ts — receipt is write-once, schema-validated, path-redacted, and hashes bounded install/oracle logs +│ └── ✓ petrinaut-historical-preflight.test.ts — receipt binds case, parent commit/tree, reference commit/tree, dependency recipe, oracle id/pack hash, and final setup status +└── isolation and cleanup + ├── ✓ petrinaut-historical-preflight.test.ts — target and receipt contain no reference commit/path or controller-private packet + └── ✓ petrinaut-historical-preflight.test.ts — pass and failure remove only owned parent/reference workspaces while retaining the receipt/evidence +``` + +## Invariants preserved + +- Historical reference material never becomes provider-target-visible — guarded by: receipt/target negative scans and D137 forbidden-root admission. +- Public packets continue to name only the parent source identity; merged-reference identity remains controller-owned — guarded by: existing case-profile redaction tests. +- Dependency and oracle selection remain compile-time closed — guarded by: TypeScript build and existing registry tests. +- Preflight evidence is not a serializable admission/security token and cannot authorize a lane — guarded by: receipt type and operator dispatch tests. +- Brunch stops at `promotion_prepared`; no source repository receives candidate output — guarded by: existing no-landing suites. +- Minimal Petri remains the sole greenfield case — guarded by: existing case-profile tests. + +## Verification Approach + +- **Inner:** self-contained temporary Git parent/reference fixtures with injected install/oracle runners prove ordering, receipt shape, redaction, no-provider trace, and cleanup. +- **Middle:** existing D137 admission and Petrinaut oracle suites remain unchanged; the preflight invokes those public operations rather than duplicating their assertions. +- **Outer:** one explicit real preflight command against the local `hashintel/hash` checkout. It must retain a setup-valid receipt before historical provider work can be scoped. +- **Checkpoint:** focused tests, `npm run verify:full`, `npm run check`, `git diff --check`, and skipped-test delta versus `origin/next`. + +## Cross-cutting obligations + +- The source checkout, controller root, parent target, reference calibration root, and receipt output root are explicit and pairwise safe. +- No env-gated skipped test may stand in for the real preflight witness. +- Install output is bounded and retained by digest; secrets, absolute paths, and full dependency logs do not enter the receipt. +- Setup invalidity remains separate from candidate assertion failure. +- The full `/processes/draft` host/iframe journey remains separately owned non-gating evidence. + +## Explicitly Out + +- Any Claude or Brunch provider turn, elicitation lane, 2×2 matrix, candidate interpretation, or promoted comparison report. +- The fully provisioned HASH `/processes/draft` host/iframe smoke. +- Arbitrary reference commits, dependency commands, oracle plugins, or manifest-selected shell behavior. +- Making the preflight receipt an admission capability or extending `ExecutionAttempt`. + +## Completion evidence + +The controller operation and CLI are built, but the scope is **not done**. The direct declared +`@hashintel/refractive` build closes the focused setup defect without generic graph preparation: the +diagnostic real no-provider run now passes both immutable installs, tracked-source checks, and all six +focused builds. Navigation readiness is no longer coupled to network idleness, and the unchanged +5-second semantic checks now expose a fixture/public-UI contract mismatch: the exact heading is absent +and the pinned historical action is `Create`, not `Create optimization`. No provider run may proceed +until the browser contract is respecified from the real public UI. + +| Leaf | Outcome | Evidence | +| ---- | ------- | -------- | +| Closed parent/reference identities | met | `petrinaut-historical-preflight.test.ts` proves fixed case selection and code-owned merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0`; the real receipt binds parent commit/tree | +| Absolute disjoint CLI roots; no arbitrary command/reference/plugin | met | `operator-cli.test.ts` — `Petrinaut preflight operator command` | +| No Claude/Brunch provider turn | met | `petrinaut-historical-preflight.test.ts` command trace contains controller preparation/calibration only | +| D137 lane-ready parent, exact identities, packet child, admission, recipe | met | focused preflight + existing `historical-replay-target.test.ts`; real parent install passed | +| Parent setup failure stops calibration | met | `retains setup_failed evidence and never calibrates after parent preparation fails` | +| Reference is disjoint/controller-only and never returned lane-ready | met | core preflight test checks forbidden-root admission, parent/reference separation, redacted receipt, and cleanup | +| Reference install/build/oracle setup failure remains setup_failed | met | install-stage, tracked-cleanliness, and oracle-preparation rivals retain distinct bounded evidence | +| Oracle assertion failure remains distinct from setup failure | met | `preserves the calibration assertion distinction from the unchanged oracle` | +| Unchanged real `petrinaut-optimization-oracles-v1` passes merged PR #9051 | **blocked** | all six focused steps pass; semantic navigation has no timeout or failed request, but the exact heading is absent and all downstream checks fail on the fixture-only `Create optimization` label | +| Write-once schema-validated redacted receipt with digests | met | receipt/evidence collision rivals plus failed-result digest round-trip; fixed relative filenames bind bounded parent/reference dependency and oracle-summary files | +| Receipt binds parent/reference identities, recipe, oracle pack/report, status | met | real invalid receipt binds both resolved identities, both passed recipes, oracle pack/report digests, setup status, evidence metadata, and cleanup | +| Parent/receipt leakage and controller-private packet exclusion | met-with-divergence | parent negative scans and receipt path-redaction pass; the receipt intentionally contains the reference commit/tree because the controlling requirement explicitly requires that binding, while containing no reference absolute path | +| Pass/failure clean only owned workspaces and retain receipt | met | parent is removed after reference-leak validation and before the reference install to bound scratch usage; real receipt records both workspaces removed | + +Diagnostic red: `npm test -- src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts -t "reference installation fails"` +failed because the receipt had no structured dependency stage or retained file. The oracle diagnostic +rival then failed because preparation command output was discarded. Green: parent/reference failures +cross D137 as structured observations; bounded path/secret-redacted files are retained after cleanup +with `wx`, and the oracle exposes only a narrow capture callback for its fixed preparation results. +Direct-fix red: `petrinaut-optimization-oracle.test.ts -t "Refractive workspace build"` failed because +the compile-time sequence omitted the declared workspace dependency. Green: the exact public sequence +contains `refractive-build` directly before UI, while the known-good slow fixture passes that sequence +and rejects omission and reordering. +Navigation red: the background-readiness slow rival failed all seven checks because `page.goto` +waited for `networkidle` under the 5-second action timeout. Green: the same DOM-ready candidate passes +all claims while a background request remains active for six seconds; `failedRequests` stays empty. + +Real evidence: + +- command: `npx tsx src/dev/execution-comparison-operator.ts petrinaut-preflight` with the real + `/Users/kostandin/Projects/hashdev/hash` checkout and explicit disjoint scratch roots +- receipt: + `/private/tmp/brunch-petrinaut-preflight-20260722T1558/evidence/petrinaut-historical-preflight-receipt.json` +- receipt SHA-256: `0ddbd4bffd4e57ce7426bc254bd8780e6d399c1eaf98f27499c261ac174a8046` +- final status: `assertion_failed`; setup status: `invalid`; exact stage: + `oracle_calibration` → historical semantic contract +- parent dependency evidence: `parent-dependency.json`, SHA-256 + `a9252a4448a87b9589efb4e4f1a7474d2540fdc87f80c28645ea3848804b1773`, 14,724 + bytes, not truncated +- reference dependency evidence: `reference-dependency.json`, SHA-256 + `813ac025c81b6b29c35bd296d41f2e98b4fb2dec13824998360713acd2d976b7`, 14,724 + bytes, not truncated +- oracle evidence: `oracle-summary.json`, SHA-256 + `39106c60d47d88699df2a92920ed46348ecb1e3efdbb61b05a7a80efe51468ae`, 22,011 + bytes, not truncated +- parent/reference: exact commit/tree matched; `corepack yarn install --immutable --mode=skip-build` + and tracked-source cleanliness passed for both +- oracle preparation: all six fixed steps passed; `refractive-build` exited 0 before + `petrinaut-ui-build` exited 0 +- browser calibration: `route-and-accessibility` reports `view: expected 1, received 0`; all six + downstream checks time out under the unchanged 5-second semantic budget waiting for the exact + `Create optimization` button +- real navigation contributes no timeout or `net::ERR_ABORTED`; `failedRequests` and `consoleErrors` + are empty +- pinned-source check: `OptimizationsView` renders `title="Optimizations"` but its button child is + `Create`; `/optimization` mounts `LocalStorageDemoApp` without a route-specific direct-view override +- no setup failure, provider turn, or browser claim pass +- cleanup: parent `removed`; reference `removed` + +Re-entry requires respecifying the browser oracle from pinned historical public semantics, including +the route's initial mode/view transition and actual accessible labels. Do not silently rename the +historical control, weaken semantic assertions, or replace this invalid receipt. + +Final verification after the semantic-contract witness: focused preflight/invariant run — 7 files / +54 tests passed; focused known-good/sensitivity/background-readiness run — 1 slow file / 4 tests +passed; `npm run verify:full` — 329 default files / 2,587 tests passed and 1 file / 2 tests skipped, +plus 9 slow files / 71 tests passed, then build passed; `npm run check`, `git diff --check`, and +touched-file lints passed. Skipped-test declarations are unchanged versus `origin/next` (1 → 1; +delta 0). + +## Expected touched paths (tentative) + +```text +memory/ +├── PLAN.md ~ +└── cards/brownfield-comparison-cases--petrinaut-provider-preflight.md +docs/praxis/comparison-runs.md ~ +src/dev/ +├── TOPOLOGY.md ~ +├── execution-comparison-operator.ts ~ +└── execution-comparison/ + ├── historical-replay-target.ts ? + ├── petrinaut-historical-preflight.ts + + ├── petrinaut-historical-preflight/ ? + ├── pinned-dependency-preparation.ts ? + └── __tests__/ + ├── operator-cli.test.ts ~ + └── petrinaut-historical-preflight.test.ts + +``` diff --git a/memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md b/memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md new file mode 100644 index 000000000..e4b483c70 --- /dev/null +++ b/memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md @@ -0,0 +1,203 @@ +# Rebaseline Petrinaut reference fidelity + +Frontier: brownfield-comparison-cases +Status: active +Mode: single +Created: 2026-07-22 + +Posture: proving (inherited from `brownfield-comparison-cases`). + +## Orientation + +- **Containing seam:** D136-L's Petrinaut public packet, typed mechanical-address contract, controller-owned browser oracle, synthetic sensitivity fixture, and D137-L/D138-L real historical preflight. +- **Frontier:** FE-1241 `brownfield-comparison-cases`; this is the next slice on `ka/fe-1241-brownfield-comparison-cases`, not a new issue or branch. +- **Volatile state:** both real immutable installs, tracked-clean checks, all six focused builds, semantic route navigation, retained evidence, and cleanup pass without a provider turn; calibration stops because fixture-authored exact heading/action claims contradict merged PR #9051 behavior. +- **Main risk:** treating synthetic accessibility labels or implementation shape as source truth would either reject the known-good reference or weaken genuine scenario/request/progress/error/cancel/secrecy requirements. + +## Target Behavior + +The frozen Petrinaut comparison profile passes its no-provider merged-reference calibration under a source-faithful typed mechanical contract. + +## Cold-start reads + +- `memory/SPEC.md` — D70-L, D136-L, D137-L, D138-L; A50-L; relevant verification and historical-replay invariants +- `memory/PLAN.md` — frontier `brownfield-comparison-cases`, especially selected Petrinaut replay, acceptance, verification, cross-cutting obligations, and current execution pointer +- `memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md` — retained invalid receipt, exact stop condition, completed setup evidence, and re-entry requirement +- `memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md` — original oracle contract, sensitivity intent, and inherited invariants +- `src/dev/TOPOLOGY.md` — Historical Replay Isolation and Brownfield Comparison Oracles +- `docs/praxis/comparison-runs.md` — controller isolation, validity, evidence retention, redaction, and no-landing rules +- `docs/praxis/manual-testing.md` — browser/accessibility observation, bounded evidence, findings ownership, and teardown discipline +- Linear `FE-1162` task body and [hashintel/hash PR #9051](https://github.com/hashintel/hash/pull/9051) title/body — source task semantics and UI-slice claims +- merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0` over parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978` — observable behavior only; never a golden diff, architecture oracle, or provider-visible source +- `testing/execution-comparisons/cases/petrinaut-optimization/{spec.md,public-contract.json,controller/oracle-manifest.json}` — current public packet and claim map +- `testing/end-to-end-comparisons/cases/petrinaut-optimization/{shared-baseline.md,study-contract.json,controller/requirement-registry.json}` — public baseline, hash chain, and requirement provenance +- `src/dev/execution-comparison/{case-contract.ts,oracle-pack.ts,petrinaut-optimization-oracle.ts}` and `petrinaut-optimization-oracle/{browser,claims,fixture,runner,types}.ts` — parser/type mismatch, compiled oracle, addresses, rivals, and evidence +- `src/dev/execution-comparison/{petrinaut-historical-preflight.ts,petrinaut-historical-preflight/evidence.ts}` — controller-only reference composition, retained receipt, redaction, and cleanup + +## Boundary Crossings + +```text +→ FE-1162 task body + PR #9051 body +→ controller-only disposable checkout of merged commit 276e17d7 +→ unchanged immutable install + closed focused builds +→ standalone /optimization route in a fresh browser +→ bounded DOM, accessibility-tree, request, stream, abort, and rendered-state evidence +→ source-fidelity map for route/view/navigation/drawer/control/progress/error/cancel semantics +→ corrected public spec, shared baseline, requirement provenance, and closed mechanical addresses +→ regenerated specification/public-packet/study/oracle content hashes +→ compiled browser oracle consuming one typed address per source-backed interaction +→ rewritten synthetic known-good plus focused omission/reordering/behavioral rivals +→ unchanged real no-provider preflight against the merged reference +→ write-once redacted receipt/evidence and cleanup of every owned workspace/process +``` + +The merged-reference workspace is controller-only and forbidden to every future lane. Audit evidence may describe observable behavior and source provenance, but no reference path, implementation file, diff, selector, or solution material may enter the public packet, synthetic fixture, provider target, or address contract. + +## Risks and Assumptions + +- **RISK:** exact labels, roles, or drawer/navigation steps are inferred from the synthetic fixture rather than the source task/reference. + - **MITIGATION:** capture a controller-only behavioral map from the task/PR body plus browser accessibility evidence first; delete unsupported stronger claims and permit exactly one source-backed role/name address per required interaction. +- **RISK:** rebaselining removes real behavior merely because the current merged UI is awkward. + - **MITIGATION:** preserve scenario-first configuration, flat fixed/optimized typed parameters, one saved/custom metric plus direction, same-origin request construction, progressive trial/best/completion, distinct error/cancel states, browser abort, and private-origin secrecy; only unsupported addressability/accessibility strength may change. +- **RISK:** the historical patch becomes a golden implementation or leaks into a provider lane. + - **MITIGATION:** inspect source only inside a disposable controller root; grade browser/API outcomes, never file names, component structure, diff shape, or architecture; retain negative leakage scans. +- **RISK:** a permissive locator ladder makes both the reference and defective rivals pass. + - **MITIGATION:** use a closed discriminated mechanical-address type with one exact role/name contract per source-backed action; prohibit locator alternatives, arbitrary CSS/XPath selectors, text fallbacks, and runtime-selected addresses. +- **RISK:** edited packet bytes leave stale transitive hashes or a receipt bound to the prior oracle pack. + - **MITIGATION:** regenerate every affected specification, public-contract, baseline, registry, study, manifest, packet, and implementation-pack digest through the existing loaders; tests must reject one-byte drift. +- **ASSUMPTION:** no Petrinaut provider attempt exists, so D138-L still permits rebaselining the frozen packet/oracle. + - **IMPACT IF FALSE:** packet and oracle bytes are immutable; this slice must stop without editing them and report the attempt evidence. + - **VALIDATE:** pre-edit attempt/evidence inventory plus preflight command trace proving zero Claude/Brunch provider launches. +- **ASSUMPTION:** merged PR #9051 satisfies its source task's observable UI/API behavior after unsupported stronger labels are removed. + - **IMPACT IF FALSE:** FE-1241 cannot use this merged commit as a known-good reference; provider work remains blocked and D136-L/the selected replay requires explicit respecification. + - **VALIDATE:** run the rebaselined compiled oracle unchanged against the merged reference; any deeper scenario, request, progress, error, cancel/abort, or secrecy mismatch is stop-the-line evidence, not permission to weaken the requirement. + +## D138-L posture check + +This slice exercises D138-L's one pre-attempt rebaseline window and scores on all proving axes: + +- **Proof of life:** the selected merged reference passes the same compiled public-behavior oracle intended for future candidates without consuming a provider turn. +- **Invariant:** source task/reference semantics become the sole authority for mechanical addresses while the synthetic fixture returns to its proper role as a sensitivity rival. +- **Uncertainty:** the slice determines whether PR #9051 is actually a valid known-good behavioral reference before FE-1241 spends provider budget. + +The rebaseline is permitted only because no attempt exists. It ends when the regenerated packet/oracle passes the merged reference and sensitivity rivals; all later provider attempts freeze those bytes again. If the reference misses a genuine source requirement, stop and report instead of converting the mismatch into a weaker contract. + +## Acceptance Criteria + +```text +Petrinaut reference-fidelity acceptance +├── source-fidelity audit +│ ├── ✓ retained source-fidelity evidence — FE-1162 and PR #9051 body claims map to observable route/view/navigation/drawer/control/progress/error/cancel/API semantics +│ ├── ✓ controller-only browser/AX capture — merged commit 276e17d7 is inspected from a disposable forbidden root with no provider launch or target-visible reference material +│ ├── ✓ source-fidelity review — every retained exact role/name and interaction step is witnessed by source behavior; unsupported fixture-authored strength is absent +│ └── ✓ stop-the-line review — any mismatch in scenario, bindings, metric/direction, request, progress/best/completion, error, cancel/abort, or private-origin behavior blocks the slice +├── typed contract and hash regeneration +│ ├── ✓ case-contract.test.ts — Petrinaut's interface and parser expose the same closed typed mechanical-address shape +│ ├── ✓ case-contract.test.ts — each required address has one exact source-backed role/name; alternatives, arbitrary selectors, extra keys, and missing keys fail closed +│ ├── ✓ case-profile.test.ts + loadPublicCasePacket — corrected spec/public baseline/registry/contract bytes retain the behavioral requirements and reject stale specification or packet hashes +│ └── ✓ study-contract/profile hash checks + loadControllerOraclePack — every changed transitive content hash and oracle-pack identity is regenerated; one-byte drift fails +├── oracle sensitivity +│ ├── ✓ petrinaut-optimization-oracle.slow.test.ts — the rewritten synthetic candidate implements the rebaselined public semantics and passes every compiled claim +│ ├── ✓ petrinaut-optimization-oracle.test.ts — source-backed mechanical addresses are consumed from the typed contract rather than duplicated as fixture-owned truth +│ ├── ✓ petrinaut-optimization-oracle.test.ts — focused omission rivals fail scenario, saved/custom metric, progress/best, error, cancel/abort, and private-origin claims +│ └── ✓ petrinaut-optimization-oracle.slow.test.ts — focused ordering/behavioral rivals fail when scenario gating/reset, preparation order, progressive state order, abort, or same-origin behavior is wrong +├── real-reference pass +│ ├── ✓ real petrinaut-preflight command — unchanged regenerated compiled oracle passes merged commit 276e17d7 after the fixed immutable install and six focused builds +│ ├── ✓ real oracle summary — all declared checks pass with no setup failure, assertion failure, console error, or unexpected failed request +│ └── ✓ petrinaut-historical-preflight.test.ts — any genuine source-behavior mismatch remains assertion_failed and cannot be relabeled setup-valid +├── cleanup and redaction +│ ├── ✓ petrinaut-historical-preflight.test.ts + real receipt — source, parent, reference, browser, server, and fake-optimizer roots/processes are clean or removed on pass and failure +│ ├── ✓ leakage scans — target, public packet, synthetic fixture, receipt, and retained evidence contain no reference path, source diff, implementation selector, secret, or controller-private packet +│ └── ✓ receipt/evidence contract — final write-once redacted receipt binds regenerated packet/oracle identities and bounded evidence hashes while retaining the successful reference verdict +└── no-provider proof + ├── ✓ petrinaut-historical-preflight.test.ts — command trace contains zero Claude/Brunch provider, elicitation, execution-lane, or matrix launches + ├── ✓ attempt inventory check — no Petrinaut ExecutionAttempt exists before or after calibration + └── ✓ repository gate — focused suites, npm run verify:full, npm run check, git diff --check, touched-file lints, and skipped-test delta pass with every owned root clean +``` + +## Invariants preserved + +- The merged reference is behavioral calibration evidence, never a golden diff or architecture oracle — guarded by: source-dependency scan, browser/API-only assertions, and retained source-fidelity review. +- No provider attempt may start before the merged reference passes the frozen no-provider calibration — guarded by: preflight status/receipt and command-trace tests. +- After the first provider attempt, packet/oracle bytes cannot be rebaselined — guarded by: pre-edit attempt inventory and D138-L stop condition. +- Public requirements still cover scenario-first/reset behavior, flat typed fixed/optimized bindings, one saved/custom metric with direction, same-origin request construction, progressive trials/best/completion, distinct error/cancel, upstream abort, and private-origin secrecy — guarded by: requirement registry, browser checks, captured request/abort evidence, and focused rivals. +- Mechanical addresses remain exact and typed without alternatives or arbitrary selectors — guarded by: contract parser/type tests and oracle source checks. +- Synthetic fixtures prove sensitivity only and contain no historical implementation knowledge or invented ground truth — guarded by: fixture/oracle source scan and contrastive rivals. +- Historical reference roots, controller packets, private optimizer origins, and retained raw evidence never become provider-visible — guarded by: D137-L forbidden-root admission, redaction tests, and negative leakage scans. +- Setup invalidity remains distinct from assertion failure; a deeper real-reference mismatch is a respec signal — guarded by: preflight stage/result tests. +- Minimal Petri remains the sole greenfield case; Brunch/Petri packets, oracles, attempts, and promoted witnesses remain unchanged — guarded by: predecessor profile/oracle regression suites. +- Neither source repository receives candidate output or a landed change — guarded by: existing `promotion_prepared` and no-landing suites. + +## Verification Approach + +- **Inner:** parser/type parity, exact-key mechanical-address validation, content/hash-chain regeneration, oracle source isolation, claim mapping, preflight failure classification, receipt/redaction, no-provider trace, and focused omission rivals. +- **Middle:** rewritten synthetic candidate plus focused ordering/behavioral rivals exercise the complete compiled browser oracle with deterministic same-origin optimizer responses; unchanged Petri/Brunch and D137-L admission suites remain green. +- **Outer:** a controller-only disposable merged-reference inspection captures bounded browser/accessibility/request evidence, then the real no-provider `petrinaut-preflight` command must pass the unchanged regenerated oracle and retain its final receipt/evidence. D138-L owns this outer gate in this card; no later provider matrix or host/iframe smoke substitutes for it. +- **Checkpoint:** focused tests while iterating; then `npm run verify:full`, `npm run check`, `git diff --check`, touched-file lints, skipped-test delta versus `origin/next`, process/workspace cleanup inspection, and source/target Git cleanliness. + +## Cross-cutting obligations + +- Preserve FE-1239 exact-byte handoff and FE-1230 immutable-attempt contracts without schema widening. +- Preserve D137-L's two-commit target admission, explicit forbidden roots, compiled dependency recipe, runtime network denial, and absence of arbitrary commands/plugins/selectors. +- Keep source issue/PR/reference material controller-only; public artifacts may state behavior but never historical identity, expected hidden values, implementation paths, or solution wording. +- Public baseline controls only source-backed common addressability and is never credited as elicitation gain. +- Regenerate hashes only for bytes changed by source contradiction; do not opportunistically rewrite unrelated frozen content. +- Retain invalid and successful preflight evidence separately; never replace the earlier assertion-failed receipt. +- Bound and redact browser/AX/request evidence; retain hashes and safe summaries rather than absolute paths, secrets, or full dependency logs. +- Reconcile D136-L, `src/dev/TOPOLOGY.md`, `memory/PLAN.md`, and this card's completion evidence to actual final truth. +- Leave the working repository, source checkout, disposable targets, browser/server processes, and all owned scratch roots clean. + +## Explicitly Out + +- Any Claude or Brunch provider turn, elicitation run, execution lane, 2×2 matrix, comparison report, scoring, winner claim, or reliability repetition. +- Weakening a genuine source behavior so the merged reference passes. +- Golden-diff, source-layout, component-name, module-boundary, CSS/XPath, screenshot-pixel, or architecture matching. +- Permissive locator alternatives, fallback label ladders, arbitrary selectors, runtime-selected addresses, or fixture-authored accessibility requirements. +- Changes to merged HASH/Petrinaut implementation, optimizer backend/API behavior, Brunch product runtime, or `ExecutionAttempt`. +- The fully provisioned HASH `/processes/draft` host/iframe smoke, masked visual/code review, optional real-optimizer smoke, or provider-campaign evidence. +- Replacing or mutating the retained failed preflight receipt. +- New cases, a second greenfield case, arbitrary oracle plugins, broad HASH build preparation, source landing, or PR publication. + +## Expected touched paths (tentative) + +```text +testing/ +├── execution-comparisons/cases/petrinaut-optimization/ +│ ├── spec.md ~ +│ ├── public-contract.json ~ +│ └── controller/oracle-manifest.json ? +└── end-to-end-comparisons/cases/petrinaut-optimization/ + ├── shared-baseline.md ~ + ├── study-contract.json ~ + └── controller/requirement-registry.json ~ +src/dev/ +├── TOPOLOGY.md ~ +├── end-to-end-comparison/__tests__/case-profile.test.ts ~ +└── execution-comparison/ + ├── case-contract.ts ~ + ├── oracle-pack.ts ? + ├── petrinaut-optimization-oracle.ts ? + ├── petrinaut-optimization-oracle/ + │ ├── browser.ts ~ + │ ├── claims.ts ~ + │ ├── fixture.ts ~ + │ ├── runner.ts ? + │ └── types.ts ? + ├── petrinaut-historical-preflight.ts ? + ├── petrinaut-historical-preflight/evidence.ts ? + └── __tests__/ + ├── case-contract.test.ts ~ + ├── operator-oracle-dispatch.test.ts ? + ├── petrinaut-optimization-oracle.test.ts ~ + ├── petrinaut-optimization-oracle.slow.test.ts ~ + └── petrinaut-historical-preflight.test.ts ~ +memory/ +├── PLAN.md ~ +├── SPEC.md ~ +└── cards/ + ├── brownfield-comparison-cases--petrinaut-reference-fidelity.md ~ + └── brownfield-comparison-cases--petrinaut-provider-preflight.md ~ +docs/praxis/comparison-runs.md ? +``` + +Controller-only disposable workspaces and final preflight evidence live under explicit external scratch/output roots. They are runtime evidence, not additional repository planning files or target-visible fixtures. diff --git a/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts b/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts index 291aff149..2b35a1097 100644 --- a/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts +++ b/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -202,6 +202,57 @@ describe('Brunch agent runtime-state projection', () => { }); }); + it('keeps normal skill reads available while comparison roots remain bounded', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-read-policy-')); + const workspace = join(root, 'workspace'); + const packageRoot = join(root, 'package'); + const skillPath = join(packageRoot, 'SKILL.md'); + await mkdir(workspace); + await mkdir(packageRoot); + await writeFile(skillPath, '# Skill\n'); + + type RegisteredRead = { + execute( + toolCallId: string, + params: { readonly path: string }, + signal: AbortSignal | undefined, + onUpdate: undefined, + ctx: { readonly cwd: string }, + ): Promise; + }; + const registerRead = (filesystemRoot?: string): RegisteredRead => { + let registered: RegisteredRead | undefined; + registerBrunchOperationalModePolicy( + { + registerTool: (tool: { readonly name: string }) => { + if (tool.name === 'read') registered = tool as unknown as RegisteredRead; + }, + getAllTools: () => ['read', 'grep', 'find', 'ls'].map((name) => ({ name })), + setActiveTools: (_tools: string[]) => {}, + on: (_event: string, _handler: (event: never, ctx?: never) => unknown) => {}, + } as never, + filesystemRoot === undefined ? {} : { filesystemRoot }, + ); + if (registered === undefined) throw new Error('read tool was not registered'); + return registered; + }; + + try { + await expect( + registerRead().execute('normal-read', { path: skillPath }, undefined, undefined, { + cwd: workspace, + }), + ).resolves.toBeDefined(); + await expect( + registerRead(workspace).execute('bounded-read', { path: skillPath }, undefined, undefined, { + cwd: workspace, + }), + ).rejects.toThrow('read-only tool path escapes target root'); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it('activates an explicit non-empty execute-mode tool set from registered tools', () => { const executeState: BrunchAgentState = { schemaVersion: 1, diff --git a/src/.pi/extensions/agent-runtime/runtime/index.ts b/src/.pi/extensions/agent-runtime/runtime/index.ts index 3ee8c7e97..21ae16bad 100644 --- a/src/.pi/extensions/agent-runtime/runtime/index.ts +++ b/src/.pi/extensions/agent-runtime/runtime/index.ts @@ -167,7 +167,9 @@ export function registerBrunchOperationalModePolicy( label: 'read', async execute(toolCallId, params, signal, onUpdate, ctx) { const root = options.filesystemRoot ?? ctx.cwd; - await assertBoundedReadPath(root, params.path); + if (options.filesystemRoot !== undefined) { + await assertBoundedReadPath(root, params.path); + } return getReadOnlyTools(root).read.execute(toolCallId, params, signal, onUpdate); }, renderCall(args, theme) { @@ -197,7 +199,9 @@ export function registerBrunchOperationalModePolicy( label: 'grep', async execute(toolCallId, params, signal, onUpdate, ctx) { const root = options.filesystemRoot ?? ctx.cwd; - await assertBoundedReadPath(root, params.path ?? '.'); + if (options.filesystemRoot !== undefined) { + await assertBoundedReadPath(root, params.path ?? '.'); + } return getReadOnlyTools(root).grep.execute(toolCallId, params, signal, onUpdate); }, renderCall(args, theme) { @@ -224,7 +228,9 @@ export function registerBrunchOperationalModePolicy( label: 'find', async execute(toolCallId, params, signal, onUpdate, ctx) { const root = options.filesystemRoot ?? ctx.cwd; - await assertBoundedReadPath(root, params.path ?? '.'); + if (options.filesystemRoot !== undefined) { + await assertBoundedReadPath(root, params.path ?? '.'); + } return getReadOnlyTools(root).find.execute(toolCallId, params, signal, onUpdate); }, renderCall(args, theme) { @@ -250,7 +256,9 @@ export function registerBrunchOperationalModePolicy( label: 'ls', async execute(toolCallId, params, signal, onUpdate, ctx) { const root = options.filesystemRoot ?? ctx.cwd; - await assertBoundedReadPath(root, params.path ?? '.'); + if (options.filesystemRoot !== undefined) { + await assertBoundedReadPath(root, params.path ?? '.'); + } return getReadOnlyTools(root).ls.execute(toolCallId, params, signal, onUpdate); }, renderCall(args, theme) { diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index 74fd2fdd5..deaf70d95 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -29,7 +29,19 @@ It does not own published CLI behavior, public RPC contracts, database imports f `execution-comparison/host-landing-oracle.ts` is the public Brunch controller entry point; its same-named private subtree owns disposable Git fixtures, settled-session public-TUI actuation, the independent Git outcome model, and report types. The oracle resumes only controller-supplied settled sessions under `PI_OFFLINE=1`, invokes `/brunch:land` through the built candidate, and distinguishes setup invalidity from claim failure. -Petrinaut preparation selected by D136-L is split across the historical-replay operation and oracle. `execution-comparison/operator-cli.ts` resolves every `pinned_git` contract into the deep operation; that operation materializes the frozen full HASH tree and exact handoff per lane, then runs the one compiled immutable install (`corepack yarn install --immutable --mode=skip-build`) before the network-denied lane. Install failure or tracked-source mutation removes only the newly owned target. The pinned Brunch branch then seeds the exact specification as one opaque settled requirement, establishes brownfield posture, and returns the `specId` consumed by `execution-comparison-brunch.ts`; Claude receives the same source and handoff without Brunch graph state. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against the standalone `/optimization` route. The browser sees only the candidate origin and same-origin proxy, while controller fixtures and the private optimizer origin remain outside target-visible roots. The fully provisioned `/processes/draft` host/iframe journey is non-gating outer evidence and does not alter this mechanical hard gate. +Petrinaut preparation selected by D136-L is split across the historical-replay operation and oracle. `execution-comparison/operator-cli.ts` resolves every `pinned_git` contract into the deep operation; that operation materializes the frozen full HASH tree and exact handoff per lane, then runs the one compiled immutable install (`corepack yarn install --immutable --mode=skip-build`) before the network-denied lane. Install failure or tracked-source mutation removes only the newly owned target. The pinned Brunch branch then seeds the exact specification as one opaque settled requirement, establishes brownfield posture, and returns the `specId` consumed by `execution-comparison-brunch.ts`; Claude receives the same source and handoff without Brunch graph state. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against the standalone `/optimization` route. The compiled sequence includes the directly declared `@hashintel/refractive` workspace build after design-system/core/optimizer prerequisites and immediately before Petrinaut UI; the synthetic candidate rejects omission or reordering rather than discovering a generic workspace graph. Controller HTTP reachability remains a separate setup check. Fresh browser pages use bounded `domcontentloaded` navigation rather than network-idle coupling, then retain the 5-second semantic locator/action budget; a synthetic route with more than five seconds of background traffic proves readiness without spurious aborted-request evidence. The real merged reference now passes every focused build and reaches browser semantics, where the fixture-authored heading/control contract diverges from the pinned historical public UI. The browser sees only the candidate origin and same-origin proxy, while controller fixtures and the private optimizer origin remain outside target-visible roots. The fully provisioned `/processes/draft` host/iframe journey is non-gating outer evidence and does not alter this mechanical hard gate. + +`execution-comparison/petrinaut-historical-preflight.ts` is the controller-only, no-provider-turn +calibration composition for that seam. It prepares and admits a disposable parent target through +D137-L, materializes the one code-owned merged reference in a disjoint forbidden root, reuses the same +compiled immutable recipe, and invokes the unchanged compiled Petrinaut oracle. Structured fixed-recipe +and fixed-oracle observations remain controller-only until owned-workspace cleanup, then become bounded, +path/secret-redacted, write-once parent dependency, reference dependency, and oracle-summary files. +The schema-validated receipt names only those safe relative files plus their hashes, sizes, and +truncation state. The admitted parent is removed after reference-leak validation and before reference +installation so two large dependency trees do not coexist. The receipt is not lane admission authority +and no reference workspace or launch descriptor is returned. A failed real calibration remains setup +evidence rather than permission to weaken the recipe or oracle. `execution-comparison-operator.ts` dispatches both brownfield oracles and the retained Petri browser oracle through a closed compile-time registry and includes every implementation module in the immutable oracle-pack hash; manifests cannot select paths, commands, or plugins at runtime. diff --git a/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts b/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts index 2374afa89..95302d5ea 100644 --- a/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts @@ -1,7 +1,6 @@ -import { realpathSync } from 'node:fs'; import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join, resolve } from 'node:path'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; @@ -97,62 +96,13 @@ describe('end-to-end execution adapters', () => { '--model', 'claude-opus-4-8', '--permission-mode', - 'dontAsk', + 'bypassPermissions', '--no-session-persistence', - '--strict-mcp-config', - '--mcp-config', - '{"mcpServers":{}}', - '--setting-sources', - '', - '--tools', - 'Bash,Edit,Glob,Grep,Read,Write', - '--allowedTools', - 'Bash', - 'Edit', - 'Glob', - 'Grep', - 'Read', - 'Write', - '--disallowedTools', - 'WebFetch', - 'WebSearch', ]), ); - const settingsIndex = prepared.launch.args.indexOf('--settings'); - expect(settingsIndex).toBeGreaterThan(-1); - expect(JSON.parse(prepared.launch.args[settingsIndex + 1]!)).toEqual({ - enabledPlugins: {}, - permissions: { - allow: ['Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write'], - deny: ['WebFetch', 'WebSearch'], - }, - sandbox: { - enabled: true, - failIfUnavailable: true, - autoAllowBashIfSandboxed: true, - allowUnsandboxedCommands: false, - filesystem: { - denyRead: [ - join(realpathSync(root), 'controller'), - realpathSync(resolve(fileURLToPath(new URL('../../../../', import.meta.url)))), - ], - }, - network: { - allowedDomains: [], - deniedDomains: [ - 'github.com', - '*.github.com', - '*.githubusercontent.com', - 'linear.app', - '*.linear.app', - 'notion.so', - '*.notion.so', - 'notion.site', - '*.notion.site', - ], - }, - }, - }); + expect(prepared.launch.args).not.toEqual( + expect.arrayContaining(['--strict-mcp-config', '--settings', '--tools']), + ); expect(prepared.launch.args.at(-1)).not.toContain(controllerRoot); await writeFile(join(workspaceDir, 'package.json'), '{"scripts":{"test":"true","build":"true"}}\n'); diff --git a/src/dev/end-to-end-comparison/claude-adapter.ts b/src/dev/end-to-end-comparison/claude-adapter.ts index ba1b11b1f..3a11e61a3 100644 --- a/src/dev/end-to-end-comparison/claude-adapter.ts +++ b/src/dev/end-to-end-comparison/claude-adapter.ts @@ -1,14 +1,10 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; import { runCommand, type CommandResult, type CommandRunner } from '../../app/command-runner.js'; import type { ExecutionLaunch } from './brunch-adapter.js'; import { materializeExactExecutionPacket } from './public-packet.js'; -import { - createClaudeSolutionIsolationPolicy, - type ClaudeSolutionIsolationPolicy, -} from './solution-isolation.js'; +import type { ClaudeSolutionIsolationPolicy } from './solution-isolation.js'; import { assertControllerIsolation } from './study-contract.js'; import { containedPath } from './validation.js'; @@ -109,6 +105,29 @@ export function createClaudeExecutionLaunch(input: { }; } +function createGreenfieldClaudeExecutionLaunch(workspaceDir: string): ExecutionLaunch { + return { + command: 'claude', + args: [ + '--print', + '--verbose', + '--output-format', + 'stream-json', + '--model', + 'claude-opus-4-8', + '--effort', + 'max', + '--permission-mode', + 'bypassPermissions', + '--no-session-persistence', + '--disable-slash-commands', + '--no-chrome', + implementationPrompt(), + ], + cwd: workspaceDir, + }; +} + export async function prepareClaudeExecutionWorkspace( input: { readonly workspaceDir: string; @@ -139,26 +158,13 @@ export async function prepareClaudeExecutionWorkspace( 'Freeze comparison input', ]); const baseSha = (await gitChecked(runner, input.workspaceDir, ['rev-parse', 'HEAD'])).stdout.trim(); - const isolationPolicy = createClaudeSolutionIsolationPolicy( - input.workspaceDir, - [input.controllerRoot, repositoryRoot()].filter( - (root) => !containedPath(root, input.workspaceDir) && !containedPath(input.workspaceDir, root), - ), - ); return { workspaceDir: input.workspaceDir, baseSha, - launch: createClaudeExecutionLaunch({ - workspaceDir: input.workspaceDir, - isolationPolicy, - }), + launch: createGreenfieldClaudeExecutionLaunch(input.workspaceDir), }; } -function repositoryRoot(): string { - return fileURLToPath(new URL('../../../', import.meta.url)); -} - export async function runClaudeExecutionWorkspace( input: { readonly prepared: PreparedClaudeExecutionWorkspace | ClaudeLaneReadyExecutionTarget; diff --git a/src/dev/execution-comparison-operator.ts b/src/dev/execution-comparison-operator.ts index cb033b7ca..3d68f9826 100644 --- a/src/dev/execution-comparison-operator.ts +++ b/src/dev/execution-comparison-operator.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; import process from 'node:process'; import { fileURLToPath } from 'node:url'; @@ -24,6 +24,10 @@ import { loadControllerOracleManifest, loadControllerOraclePack, } from './execution-comparison/oracle-pack.js'; +import { + runPetrinautHistoricalPreflight, + type PetrinautHistoricalPreflightReceipt, +} from './execution-comparison/petrinaut-historical-preflight.js'; import { runPetrinautOptimizationOracle, type PetrinautOptimizationOracleReport, @@ -134,7 +138,22 @@ const DEFAULT_CASES_ROOT = fileURLToPath( ); const DEFAULT_CONTROLLER_ROOT = fileURLToPath(new URL('../../', import.meta.url)); -export async function runExecutionComparisonOperatorCli(args: readonly string[]): Promise { +interface ExecutionComparisonOperatorDependencies { + readonly runPetrinautPreflight?: (input: { + readonly sourceRepositoryDir: string; + readonly parentTargetDir: string; + readonly referenceTargetDir: string; + readonly outputRoot: string; + }) => Promise<{ + readonly receiptFile: string; + readonly receipt: PetrinautHistoricalPreflightReceipt; + }>; +} + +export async function runExecutionComparisonOperatorCli( + args: readonly string[], + dependencies: ExecutionComparisonOperatorDependencies = {}, +): Promise { const [command, ...rest] = args; const options = parseOptions(rest); const casesRoot = DEFAULT_CASES_ROOT; @@ -206,6 +225,39 @@ export async function runExecutionComparisonOperatorCli(args: readonly string[]) process.stdout.write(`${JSON.stringify(retained)}\n`); return; } + case 'petrinaut-preflight': { + assertOnlyOptions(options, ['source-repository', 'parent-target', 'reference-target', 'output-root']); + const input = { + sourceRepositoryDir: requiredAbsolute(options, 'source-repository'), + parentTargetDir: requiredAbsolute(options, 'parent-target'), + referenceTargetDir: requiredAbsolute(options, 'reference-target'), + outputRoot: requiredAbsolute(options, 'output-root'), + }; + const result = await ( + dependencies.runPetrinautPreflight ?? + (async (closedInput) => + await runPetrinautHistoricalPreflight({ + sourceRepositoryDir: closedInput.sourceRepositoryDir, + parentTargetDir: closedInput.parentTargetDir, + referenceTargetDir: closedInput.referenceTargetDir, + controllerRoot: DEFAULT_CONTROLLER_ROOT, + receiptFile: join(closedInput.outputRoot, 'petrinaut-historical-preflight-receipt.json'), + })) + )(input); + process.stdout.write( + `${JSON.stringify({ + receiptFile: result.receiptFile, + status: result.receipt.status, + setupStatus: result.receipt.setupStatus, + })}\n`, + ); + if (result.receipt.status !== 'passed') { + throw new Error( + `Petrinaut preflight ${result.receipt.status}; retained receipt at ${result.receiptFile}`, + ); + } + return; + } case 'retain-attempt': { assertOnlyOptions(options, ['attempt-file', 'attempts-root']); const value = JSON.parse(await readFile(resolve(required(options, 'attempt-file')), 'utf8')) as unknown; @@ -218,7 +270,7 @@ export async function runExecutionComparisonOperatorCli(args: readonly string[]) } default: throw new Error( - 'Usage: execution-comparison-operator [options]', + 'Usage: execution-comparison-operator [options]', ); } } @@ -250,6 +302,14 @@ function required(options: ReadonlyMap, name: string): string { return value; } +function requiredAbsolute(options: ReadonlyMap, name: string): string { + const value = required(options, name); + if (!isAbsolute(value)) { + throw new Error(`--${name} must be an absolute path`); + } + return resolve(value); +} + if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { runExecutionComparisonOperatorCli(process.argv.slice(2)).catch((error: unknown) => { process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); diff --git a/src/dev/execution-comparison/__tests__/case-contract.test.ts b/src/dev/execution-comparison/__tests__/case-contract.test.ts index 73e977563..c54a2d552 100644 --- a/src/dev/execution-comparison/__tests__/case-contract.test.ts +++ b/src/dev/execution-comparison/__tests__/case-contract.test.ts @@ -82,17 +82,24 @@ describe('execution comparison public case contract', () => { sameOriginApi: '/api/petrinaut-opt/optimize/all', executionTerminal: 'promotion_prepared', }, - accessibility: { - view: { role: 'heading', name: 'Optimizations' }, - tab: { role: 'tab', name: 'Optimizations' }, - create: { role: 'button', name: 'Create optimization' }, - scenario: { role: 'combobox', name: 'Scenario' }, - metric: { role: 'combobox', name: 'Objective metric' }, - direction: { role: 'combobox', name: 'Objective direction' }, - run: { role: 'button', name: 'Run optimization' }, - cancel: { role: 'button', name: 'Cancel optimization' }, - status: { role: 'status', name: 'Optimization status' }, - results: { role: 'region', name: 'Optimization results' }, + mechanicalAddresses: { + skipTour: { kind: 'roleName', role: 'button', name: 'Skip tour' }, + dismissAssistant: { kind: 'roleName', role: 'button', name: 'Dismiss' }, + simulateMode: { kind: 'roleName', role: 'radio', name: 'Simulate' }, + optimizationsNav: { kind: 'roleValue', role: 'radio', value: 'optimizations' }, + viewTitle: { kind: 'exactText', text: 'Optimizations' }, + create: { kind: 'roleName', role: 'button', name: 'Create' }, + createDrawer: { kind: 'roleName', role: 'dialog', name: 'Create an optimization' }, + scenario: { kind: 'roleContents', role: 'combobox', contents: 'Select a scenario' }, + metric: { kind: 'roleContents', role: 'combobox', contents: 'Select a metric' }, + metricCode: { kind: 'roleName', role: 'textbox', name: 'Editor content' }, + directionMaximize: { kind: 'roleName', role: 'radio', name: 'Maximize' }, + directionMinimize: { kind: 'roleName', role: 'radio', name: 'Minimize' }, + run: { kind: 'roleName', role: 'button', name: 'Run' }, + cancel: { kind: 'roleName', role: 'button', name: 'Cancel' }, + statusComplete: { kind: 'exactText', text: 'Complete' }, + statusError: { kind: 'exactText', text: 'Error' }, + statusCancelled: { kind: 'exactText', text: 'Cancelled' }, }, }; expect(parsePublicCaseContract(petrinautContract)).toEqual(petrinautContract); @@ -106,6 +113,19 @@ describe('execution comparison public case contract', () => { }, }, { acceptance: { ...petrinautContract.acceptance, publicRoute: '/processes/draft' } }, + { + mechanicalAddresses: { + ...petrinautContract.mechanicalAddresses, + create: { kind: 'roleName', role: 'button', name: 'Create optimization' }, + }, + }, + { + mechanicalAddresses: { + ...petrinautContract.mechanicalAddresses, + create: { kind: 'roleName', role: 'button', name: 'Create' }, + extra: { kind: 'roleName', role: 'button', name: 'Extra' }, + }, + }, ]) { expect(() => parsePublicCaseContract({ ...petrinautContract, ...mutation })).toThrow( 'invalid fixed public execution contract', diff --git a/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts b/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts index fb9b8c17c..ae867ed39 100644 --- a/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts +++ b/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts @@ -90,8 +90,8 @@ describe('execution comparison Brunch CLI arguments', () => { await mkdir(targetRoot); await mkdir(outsideRoot); await writeFile(join(targetRoot, 'inside.txt'), 'inside\n'); - await writeFile(join(outsideRoot, 'secret.txt'), 'historical solution\n'); - await symlink(join(outsideRoot, 'secret.txt'), join(targetRoot, 'escaped-link.txt')); + await writeFile(join(outsideRoot, 'excluded.txt'), 'outside comparison target\n'); + await symlink(join(outsideRoot, 'excluded.txt'), join(targetRoot, 'escaped-link.txt')); await symlink(outsideRoot, join(targetRoot, 'escaped-directory')); const tools: Array<{ diff --git a/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts b/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts index 9f3ee4894..6afd30d58 100644 --- a/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts +++ b/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts @@ -272,7 +272,10 @@ describe('historical replay target preparation', () => { return await runCommand(command, args, options); }, ); - expect(run.repository?.finalGitRange).toMatch(new RegExp(`^${ready.baseSha}\\.\\.[a-f0-9]{40}$`, 'u')); + const [rangeBase, rangeReview, rangeRemainder] = run.repository?.finalGitRange.split('..') ?? []; + expect(rangeBase).toBe(ready.baseSha); + expect(rangeReview).toMatch(/^[a-f0-9]{40}$/u); + expect(rangeRemainder).toBeUndefined(); }, 30_000); it.each([ @@ -487,6 +490,19 @@ describe('historical replay target preparation', () => { await expect(rejected).rejects.toMatchObject({ status: 'setup_failed', phase: 'dependency_preparation', + cause: { + outcome: { + status: 'failed', + exitCode: 0, + failureStage: 'tracked_source_cleanliness', + }, + observation: { + commandResult: { + exitCode: 0, + }, + trackedSourceStatus: 'M package.json', + }, + }, }); await expect(rejected).rejects.toThrow('modified tracked source'); await expect(readFile(join(targetDir, 'package.json'))).rejects.toMatchObject({ code: 'ENOENT' }); diff --git a/src/dev/execution-comparison/__tests__/operator-cli.test.ts b/src/dev/execution-comparison/__tests__/operator-cli.test.ts index f24f84b44..c401dfc05 100644 --- a/src/dev/execution-comparison/__tests__/operator-cli.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-cli.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { runExecutionComparisonOperatorCli } from '../../execution-comparison-operator.js'; import { listExecutionCases, prepareExecutionTarget, resolveExecutionCase } from '../operator-cli.js'; @@ -219,3 +219,144 @@ describe('execution comparison target preparation', () => { } }); }); + +describe('Petrinaut preflight operator command', () => { + it('requires absolute closed roots and dispatches no provider or arbitrary command input', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-preflight-cli-')); + const sourceRepositoryDir = join(root, 'source'); + const workRoot = join(root, 'work'); + const outputRoot = join(root, 'evidence'); + await Promise.all([mkdir(sourceRepositoryDir), mkdir(workRoot), mkdir(outputRoot)]); + const parentTargetDir = join(workRoot, 'parent'); + const referenceTargetDir = join(workRoot, 'reference'); + const calls: unknown[] = []; + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + await runExecutionComparisonOperatorCli( + [ + 'petrinaut-preflight', + '--source-repository', + sourceRepositoryDir, + '--parent-target', + parentTargetDir, + '--reference-target', + referenceTargetDir, + '--output-root', + outputRoot, + ], + { + runPetrinautPreflight: async (input) => { + calls.push(input); + return { + receiptFile: join(outputRoot, 'petrinaut-historical-preflight-receipt.json'), + receipt: { + schemaVersion: 1, + caseId: 'petrinaut-optimization-v1', + status: 'passed', + setupStatus: 'valid', + commandTrace: [ + 'prepare_parent_target', + 'materialize_reference', + 'prepare_reference_dependencies', + 'run_compiled_oracle', + 'cleanup_workspaces', + ], + cleanup: { + parentWorkspace: 'removed', + referenceWorkspace: 'removed', + }, + evidence: {}, + }, + }; + }, + }, + ); + + expect(calls).toEqual([ + { + sourceRepositoryDir, + parentTargetDir, + referenceTargetDir, + outputRoot, + }, + ]); + expect(JSON.stringify(calls)).not.toMatch(/claude|brunch.*provider|command|referenceCommit/iu); + await expect( + runExecutionComparisonOperatorCli( + [ + 'petrinaut-preflight', + '--source-repository', + sourceRepositoryDir, + '--parent-target', + parentTargetDir, + '--reference-target', + referenceTargetDir, + '--output-root', + outputRoot, + ], + { + runPetrinautPreflight: async () => ({ + receiptFile: join(outputRoot, 'invalid-receipt.json'), + receipt: { + schemaVersion: 1, + caseId: 'petrinaut-optimization-v1', + status: 'setup_failed', + setupStatus: 'invalid', + commandTrace: [ + 'prepare_parent_target', + 'materialize_reference', + 'prepare_reference_dependencies', + 'run_compiled_oracle', + 'cleanup_workspaces', + ], + cleanup: { + parentWorkspace: 'not_created', + referenceWorkspace: 'not_created', + }, + evidence: {}, + failure: { + phase: 'parent_preparation', + messageSha256: `sha256:${'f'.repeat(64)}`, + }, + }, + }), + }, + ), + ).rejects.toThrow('setup_failed'); + await expect( + runExecutionComparisonOperatorCli( + [ + 'petrinaut-preflight', + '--source-repository', + 'relative-source', + '--parent-target', + parentTargetDir, + '--reference-target', + referenceTargetDir, + '--output-root', + outputRoot, + ], + { runPetrinautPreflight: async () => Promise.reject(new Error('must not dispatch')) }, + ), + ).rejects.toThrow('absolute'); + await expect( + runExecutionComparisonOperatorCli([ + 'petrinaut-preflight', + '--source-repository', + sourceRepositoryDir, + '--parent-target', + parentTargetDir, + '--reference-target', + referenceTargetDir, + '--output-root', + outputRoot, + '--command', + 'yarn anything', + ]), + ).rejects.toThrow('unknown execution comparison operator option: --command'); + } finally { + stdout.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts new file mode 100644 index 000000000..0688bee74 --- /dev/null +++ b/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts @@ -0,0 +1,691 @@ +import { createHash } from 'node:crypto'; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; +import { createNetworkDeniedCommandRunner } from '../../end-to-end-comparison/solution-isolation.js'; +import { resolveExecutionCase, type ResolvedExecutionCase } from '../operator-cli.js'; +import { + PETRINAUT_HISTORICAL_REFERENCE_COMMIT, + parsePetrinautHistoricalPreflightReceipt, + runPetrinautHistoricalPreflight, + type PetrinautHistoricalPreflightDependencies, +} from '../petrinaut-historical-preflight.js'; + +const roots: string[] = []; +const frozenCasesRoot = fileURLToPath( + new URL('../../../../testing/execution-comparisons/cases/', import.meta.url), +); +const frozenParentCommit = '5c7a2d9db5caa851c38938f4b1bac19005b0e978'; +const frozenParentTree = 'a3e08cf75e00cc9016c931f4665341506e03533e'; + +afterEach(async () => { + await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); +}); + +describe('Petrinaut historical provider preflight', () => { + it('prepares the parent, calibrates a disjoint reference, retains redacted evidence, and cleans up', async () => { + const fixture = await createFixture(); + const selectedCase = await selectedCaseForFixture(fixture); + const order: string[] = []; + const forbiddenRootCalls: string[][] = []; + let parentRemovedBeforeReferenceInstall = false; + + const result = await runPetrinautHistoricalPreflight( + { + sourceRepositoryDir: fixture.sourceDir, + parentTargetDir: fixture.parentTargetDir, + referenceTargetDir: fixture.referenceTargetDir, + controllerRoot: fixture.controllerDir, + receiptFile: fixture.receiptFile, + }, + { + runner: fixtureRunner(fixture), + selectedCase, + referenceCommit: fixture.referenceCommit, + dependencyInstallRunner: async (command, args, options) => { + order.push(options.cwd === fixture.parentTargetDir ? 'parent-install' : 'reference-install'); + expect({ command, args }).toEqual({ + command: 'corepack', + args: ['yarn', 'install', '--immutable', '--mode=skip-build'], + }); + if (options.cwd === fixture.referenceTargetDir) { + try { + await readFile(fixture.parentTargetDir); + } catch (error) { + parentRemovedBeforeReferenceInstall = + typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT'; + } + } + await writeFile(join(options.cwd, '.pnp.cjs'), '// immutable install artifact\n'); + return { + exitCode: 0, + stdout: `installed ${options.cwd}\n`, + stderr: 'bounded warning\n', + }; + }, + createVerifier: (forbiddenRoots) => { + forbiddenRootCalls.push([...forbiddenRoots]); + return createNetworkDeniedCommandRunner({ + platform: 'darwin', + forbiddenReadRoots: forbiddenRoots, + run: fakeSandboxRunner, + }); + }, + loadOraclePack: async () => ({ + oracleId: 'petrinaut-optimization-oracles-v1', + packSha256: `sha256:${'c'.repeat(64)}`, + }), + runOracle: async ({ candidateRoot }) => { + order.push('oracle'); + expect(candidateRoot).toBe(fixture.referenceTargetDir); + return passingOracleReport(); + }, + }, + ); + + expect(PETRINAUT_HISTORICAL_REFERENCE_COMMIT).toBe('276e17d7b0f80c8a80d5abe01849bbb67c6169d0'); + expect(order).toEqual(['parent-install', 'reference-install', 'oracle']); + expect(forbiddenRootCalls).toHaveLength(1); + expect(forbiddenRootCalls[0]).toContain(fixture.referenceTargetDir); + expect(parentRemovedBeforeReferenceInstall).toBe(true); + expect(result.receipt).toMatchObject({ + schemaVersion: 1, + caseId: 'petrinaut-optimization-v1', + status: 'passed', + setupStatus: 'valid', + parent: { + sourceCommit: frozenParentCommit, + sourceTree: frozenParentTree, + dependencyPreparation: { + recipe: 'petrinaut-yarn-immutable-v1', + status: 'passed', + exitCode: 0, + }, + }, + reference: { + sourceCommit: fixture.referenceCommit, + sourceTree: fixture.referenceTree, + dependencyPreparation: { + recipe: 'petrinaut-yarn-immutable-v1', + status: 'passed', + exitCode: 0, + }, + }, + oracle: { + id: 'petrinaut-optimization-oracles-v1', + packSha256: `sha256:${'c'.repeat(64)}`, + reportStatus: 'passed', + }, + cleanup: { + parentWorkspace: 'removed', + referenceWorkspace: 'removed', + }, + }); + expect(result.receipt.commandTrace).toEqual([ + 'prepare_parent_target', + 'materialize_reference', + 'prepare_reference_dependencies', + 'run_compiled_oracle', + 'cleanup_workspaces', + ]); + expect(result.receipt.commandTrace.join('\n')).not.toMatch(/claude|brunch.*provider|lane/iu); + await expect(readFile(fixture.parentTargetDir)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(readFile(fixture.referenceTargetDir)).rejects.toMatchObject({ code: 'ENOENT' }); + + const rawReceipt = await readFile(fixture.receiptFile, 'utf8'); + expect(parsePetrinautHistoricalPreflightReceipt(JSON.parse(rawReceipt))).toEqual(result.receipt); + expect(rawReceipt).not.toContain(fixture.sourceDir); + expect(rawReceipt).not.toContain(fixture.parentTargetDir); + expect(rawReceipt).not.toContain(fixture.referenceTargetDir); + expect(rawReceipt).not.toContain(fixture.controllerDir); + expect(rawReceipt).not.toContain('.comparison-source.json'); + if ( + result.receipt.parent === undefined || + result.receipt.reference === undefined || + result.receipt.oracle === undefined + ) { + throw new Error('passing receipt omitted required evidence'); + } + expect(result.receipt.oracle.reportSha256).toBe(sha256(`${JSON.stringify(passingOracleReport())}\n`)); + for (const [key, filename] of [ + ['parentDependency', 'parent-dependency.json'], + ['referenceDependency', 'reference-dependency.json'], + ['oracleSummary', 'oracle-summary.json'], + ] as const) { + const metadata = result.receipt.evidence[key]; + expect(metadata?.file).toBe(filename); + if (metadata === undefined) throw new Error(`${key} evidence missing`); + const retained = await readFile(join(dirname(fixture.receiptFile), metadata.file), 'utf8'); + expect(sha256(retained)).toBe(metadata.sha256); + expect(Buffer.byteLength(retained, 'utf8')).toBe(metadata.bytes); + expect(retained).not.toContain(fixture.sourceDir); + expect(retained).not.toContain(fixture.parentTargetDir); + expect(retained).not.toContain(fixture.referenceTargetDir); + expect(retained).not.toContain(fixture.controllerDir); + } + expect(rawReceipt).toContain('"file": "parent-dependency.json"'); + expect(rawReceipt).toContain('"file": "reference-dependency.json"'); + expect(rawReceipt).toContain('"file": "oracle-summary.json"'); + const { parent: _parent, ...missingParent } = result.receipt; + expect(() => parsePetrinautHistoricalPreflightReceipt(missingParent)).toThrow(); + expect(() => + parsePetrinautHistoricalPreflightReceipt({ + ...result.receipt, + launch: { command: 'claude', cwd: fixture.parentTargetDir }, + }), + ).toThrow(); + }, 30_000); + + it('retains setup_failed evidence and never calibrates after parent preparation fails', async () => { + const fixture = await createFixture(); + let oracleCalled = false; + + const result = await runFixturePreflight(fixture, { + dependencyInstallRunner: async () => ({ + exitCode: 9, + stdout: '', + stderr: `parent install failed in ${fixture.parentTargetDir}\n`, + }), + runOracle: async () => { + oracleCalled = true; + return passingOracleReport(); + }, + }); + + expect(result.receipt).toMatchObject({ + status: 'setup_failed', + setupStatus: 'invalid', + failure: { + phase: 'parent_preparation', + dependencyStage: 'install', + }, + cleanup: { + parentWorkspace: 'not_created', + referenceWorkspace: 'not_created', + }, + }); + expect(result.receipt.parent).toBeUndefined(); + expect(result.receipt.reference?.dependencyPreparation).toBeUndefined(); + expect(result.receipt.oracle).toBeUndefined(); + expect(oracleCalled).toBe(false); + await expect(readFile(fixture.receiptFile, 'utf8')).resolves.toContain('"setup_failed"'); + const evidence = result.receipt.evidence.parentDependency; + expect(evidence?.file).toBe('parent-dependency.json'); + if (evidence === undefined) throw new Error('parent dependency evidence missing'); + const evidenceBytes = await readFile(join(dirname(fixture.receiptFile), evidence.file), 'utf8'); + expect(sha256(evidenceBytes)).toBe(evidence.sha256); + expect(evidenceBytes).toContain('"failureStage": "install"'); + expect(evidenceBytes).toContain('"exitCode": 9'); + expect(evidenceBytes).not.toContain(fixture.parentTargetDir); + }, 30_000); + + it('cleans both workspaces and retains the exact phase when reference installation fails', async () => { + const fixture = await createFixture(); + let installs = 0; + let oracleCalled = false; + const secret = 'token=reference-secret-value'; + + const result = await runFixturePreflight(fixture, { + dependencyInstallRunner: async () => { + installs += 1; + return installs === 1 + ? { exitCode: 0, stdout: 'parent ok\n', stderr: '' } + : { + exitCode: 7, + stdout: 'x'.repeat(200_000), + stderr: `reference install failed in ${fixture.referenceTargetDir}; ${secret}\n`, + }; + }, + runOracle: async () => { + oracleCalled = true; + return passingOracleReport(); + }, + }); + + expect(result.receipt).toMatchObject({ + status: 'setup_failed', + failure: { + phase: 'reference_dependency_preparation', + dependencyStage: 'install', + }, + cleanup: { + parentWorkspace: 'removed', + referenceWorkspace: 'removed', + }, + }); + expect(result.receipt.parent).toBeDefined(); + expect(result.receipt.reference?.dependencyPreparation).toMatchObject({ + status: 'failed', + failureStage: 'install', + exitCode: 7, + }); + expect(result.receipt.oracle).toBeUndefined(); + expect(oracleCalled).toBe(false); + const evidence = result.receipt.evidence.referenceDependency; + expect(evidence).toMatchObject({ + file: 'reference-dependency.json', + truncated: true, + bytes: expect.any(Number), + sha256: expect.stringMatching(/^sha256:[a-f0-9]{64}$/u), + }); + if (evidence === undefined) { + throw new Error('reference dependency evidence missing'); + } + expect(evidence.bytes).toBeLessThan(140_000); + const evidenceBytes = await readFile(join(dirname(fixture.receiptFile), evidence.file), 'utf8'); + expect(sha256(evidenceBytes)).toBe(evidence.sha256); + expect(evidenceBytes).toContain('"failureStage": "install"'); + expect(evidenceBytes).toContain('"exitCode": 7'); + expect(evidenceBytes).toContain('reference install failed'); + expect(evidenceBytes).not.toContain(fixture.referenceTargetDir); + expect(evidenceBytes).not.toContain('reference-secret-value'); + expect(JSON.stringify(result.receipt)).not.toContain(fixture.referenceTargetDir); + }, 30_000); + + it('retains a successful install result separately from tracked-source cleanliness rejection', async () => { + const fixture = await createFixture(); + let installs = 0; + + const result = await runFixturePreflight(fixture, { + dependencyInstallRunner: async (_command, _args, options) => { + installs += 1; + if (installs === 2) { + await writeFile(join(options.cwd, 'package.json'), '{"name":"mutated-reference"}\n'); + } + return { + exitCode: 0, + stdout: `immutable install passed in ${options.cwd}\n`, + stderr: '', + }; + }, + }); + + expect(result.receipt).toMatchObject({ + status: 'setup_failed', + failure: { + phase: 'reference_dependency_preparation', + dependencyStage: 'tracked_source_cleanliness', + }, + reference: { + dependencyPreparation: { + status: 'failed', + exitCode: 0, + failureStage: 'tracked_source_cleanliness', + }, + }, + }); + const evidence = result.receipt.evidence.referenceDependency; + if (evidence === undefined) throw new Error('reference dependency evidence missing'); + const retained = await readFile(join(dirname(fixture.receiptFile), evidence.file), 'utf8'); + expect(sha256(retained)).toBe(evidence.sha256); + expect(retained).toContain('"exitCode": 0'); + expect(retained).toContain('"failureStage": "tracked_source_cleanliness"'); + expect(retained).toContain('"trackedSourceStatus": "M package.json"'); + expect(retained).not.toContain(fixture.referenceTargetDir); + }, 30_000); + + it('retains sanitized command output for an oracle preparation failure', async () => { + const fixture = await createFixture(); + const report = setupFailedOracleReport(); + + const result = await runFixturePreflight(fixture, { + dependencyInstallRunner: passingInstall, + runOracle: async ({ onPreparationResult }) => { + await onPreparationResult?.({ + id: 'petrinaut-ui-build', + commandResult: { + exitCode: 1, + stdout: 'x'.repeat(50_000), + stderr: `build failed in ${fixture.referenceTargetDir}; token=oracle-secret\n`, + }, + }); + return report; + }, + }); + + const evidence = result.receipt.evidence.oracleSummary; + if (evidence === undefined) throw new Error('oracle summary evidence missing'); + expect(evidence.truncated).toBe(true); + expect(evidence.bytes).toBeLessThan(100_000); + const retained = await readFile(join(dirname(fixture.receiptFile), evidence.file), 'utf8'); + expect(sha256(retained)).toBe(evidence.sha256); + expect(retained).toContain('"id": "petrinaut-ui-build"'); + expect(retained).toContain('"exitCode": 1'); + expect(retained).toContain('build failed'); + expect(retained).not.toContain(fixture.referenceTargetDir); + expect(retained).not.toContain('oracle-secret'); + }, 30_000); + + it('rejects historical identity or path leakage in the parent target before calibration', async () => { + const fixture = await createFixture(); + let installs = 0; + let oracleCalled = false; + + const result = await runFixturePreflight(fixture, { + dependencyInstallRunner: async (_command, _args, options) => { + installs += 1; + if (installs === 1) { + await writeFile( + join(options.cwd, 'historical-reference.txt'), + `${fixture.referenceCommit}\n${fixture.referenceTargetDir}\n`, + ); + } + return { exitCode: 0, stdout: '', stderr: '' }; + }, + runOracle: async () => { + oracleCalled = true; + return passingOracleReport(); + }, + }); + + expect(result.receipt).toMatchObject({ + status: 'setup_failed', + failure: { phase: 'reference_materialization' }, + cleanup: { + parentWorkspace: 'removed', + referenceWorkspace: 'removed', + }, + }); + expect(result.receipt.reference?.dependencyPreparation).toBeUndefined(); + expect(result.receipt.oracle).toBeUndefined(); + expect(oracleCalled).toBe(false); + const rawReceipt = await readFile(fixture.receiptFile, 'utf8'); + expect(rawReceipt).not.toContain(fixture.referenceTargetDir); + }, 30_000); + + it.each([ + ['setup failure', setupFailedOracleReport(), 'setup_failed'], + ['calibration assertion', assertionFailedOracleReport(), 'assertion_failed'], + ] as const)( + 'preserves the %s distinction from the unchanged oracle', + async (_name, report, status) => { + const fixture = await createFixture(); + const result = await runFixturePreflight(fixture, { + dependencyInstallRunner: passingInstall, + runOracle: async () => report, + }); + + expect(result.receipt.status).toBe(status); + expect(result.receipt.setupStatus).toBe('invalid'); + expect(result.receipt.oracle).toMatchObject({ + reportStatus: status, + id: 'petrinaut-optimization-oracles-v1', + }); + expect(result.receipt.failure).toBeUndefined(); + expect(result.receipt.cleanup).toEqual({ + parentWorkspace: 'removed', + referenceWorkspace: 'removed', + }); + }, + 30_000, + ); + + it('rejects output collisions before creating either workspace', async () => { + const fixture = await createFixture(); + await writeFile(fixture.receiptFile, 'caller-owned evidence\n'); + + await expect( + runFixturePreflight(fixture, { + dependencyInstallRunner: passingInstall, + }), + ).rejects.toMatchObject({ code: 'EEXIST' }); + await expect(readFile(fixture.receiptFile, 'utf8')).resolves.toBe('caller-owned evidence\n'); + await expect(readFile(fixture.parentTargetDir)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(readFile(fixture.referenceTargetDir)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('rejects retained evidence collisions before running dependency preparation', async () => { + const fixture = await createFixture(); + const evidenceFile = join(dirname(fixture.receiptFile), 'parent-dependency.json'); + await writeFile(evidenceFile, 'caller-owned evidence\n'); + let installCalled = false; + + await expect( + runFixturePreflight(fixture, { + dependencyInstallRunner: async () => { + installCalled = true; + return { exitCode: 0, stdout: '', stderr: '' }; + }, + }), + ).rejects.toThrow('retained evidence parent-dependency.json'); + expect(installCalled).toBe(false); + await expect(readFile(evidenceFile, 'utf8')).resolves.toBe('caller-owned evidence\n'); + await expect(readFile(fixture.receiptFile)).rejects.toMatchObject({ + code: 'ENOENT', + }); + }); + + it.each([ + ['parent/reference', (fixture: Awaited>) => fixture.parentTargetDir], + [ + 'source/parent', + (fixture: Awaited>) => join(fixture.sourceDir, 'target'), + ], + [ + 'output/reference', + (fixture: Awaited>) => join(dirname(fixture.receiptFile), 'reference'), + ], + ])('rejects %s root overlap before materialization', async (_name, conflictingReference) => { + const fixture = await createFixture(); + const referenceTargetDir = conflictingReference(fixture); + + await expect( + runPetrinautHistoricalPreflight( + { + sourceRepositoryDir: fixture.sourceDir, + parentTargetDir: fixture.parentTargetDir, + referenceTargetDir, + controllerRoot: fixture.controllerDir, + receiptFile: fixture.receiptFile, + }, + { + selectedCase: await selectedCaseForFixture(fixture), + referenceCommit: fixture.referenceCommit, + runner: fixtureRunner(fixture), + dependencyInstallRunner: passingInstall, + createVerifier: portableVerifier, + loadOraclePack: fixedOraclePack, + runOracle: async () => passingOracleReport(), + }, + ), + ).rejects.toThrow('disjoint'); + await expect(readFile(fixture.receiptFile)).rejects.toMatchObject({ code: 'ENOENT' }); + }); +}); + +async function createFixture(): Promise<{ + readonly root: string; + readonly sourceDir: string; + readonly controllerDir: string; + readonly parentTargetDir: string; + readonly referenceTargetDir: string; + readonly receiptFile: string; + readonly parentCommit: string; + readonly referenceCommit: string; + readonly referenceTree: string; +}> { + const root = await realpath(await mkdtemp(join(tmpdir(), 'brunch-petrinaut-preflight-'))); + roots.push(root); + const sourceDir = join(root, 'source'); + const controllerDir = join(root, 'controller'); + const workRoot = join(root, 'work'); + const evidenceRoot = join(root, 'evidence'); + await Promise.all([mkdir(sourceDir), mkdir(controllerDir), mkdir(workRoot), mkdir(evidenceRoot)]); + + await writeFile(join(sourceDir, 'package.json'), '{"name":"petrinaut-fixture","private":true}\n'); + await writeFile(join(sourceDir, 'parent.ts'), 'export const parent = true;\n'); + await git(sourceDir, ['init', '--initial-branch=main']); + await git(sourceDir, ['add', '--all']); + await commit(sourceDir, 'Pinned parent'); + const parentCommit = await git(sourceDir, ['rev-parse', 'HEAD']); + await writeFile(join(sourceDir, 'reference.ts'), 'export const mergedReference = true;\n'); + await git(sourceDir, ['add', '--all']); + await commit(sourceDir, 'Merged reference'); + const referenceCommit = await git(sourceDir, ['rev-parse', 'HEAD']); + const referenceTree = await git(sourceDir, ['rev-parse', 'HEAD^{tree}']); + + return { + root, + sourceDir, + controllerDir, + parentTargetDir: join(workRoot, 'parent'), + referenceTargetDir: join(workRoot, 'reference'), + receiptFile: join(evidenceRoot, 'receipt.json'), + parentCommit, + referenceCommit, + referenceTree, + }; +} + +async function selectedCaseForFixture( + _fixture: Awaited>, +): Promise { + return await resolveExecutionCase('petrinaut-optimization', frozenCasesRoot); +} + +async function runFixturePreflight( + fixture: Awaited>, + dependencies: PetrinautHistoricalPreflightDependencies, +) { + return await runPetrinautHistoricalPreflight( + { + sourceRepositoryDir: fixture.sourceDir, + parentTargetDir: fixture.parentTargetDir, + referenceTargetDir: fixture.referenceTargetDir, + controllerRoot: fixture.controllerDir, + receiptFile: fixture.receiptFile, + }, + { + selectedCase: await selectedCaseForFixture(fixture), + referenceCommit: fixture.referenceCommit, + runner: fixtureRunner(fixture), + createVerifier: portableVerifier, + loadOraclePack: fixedOraclePack, + ...dependencies, + }, + ); +} + +function fixtureRunner(fixture: Awaited>): CommandRunner { + return async (command, args, options) => { + if (command === 'git' && options.cwd === fixture.sourceDir) { + if (args.at(-1) === `${frozenParentCommit}^{commit}`) { + return { exitCode: 0, stdout: `${frozenParentCommit}\n`, stderr: '' }; + } + if (args.at(-1) === `${frozenParentCommit}^{tree}`) { + return { exitCode: 0, stdout: `${frozenParentTree}\n`, stderr: '' }; + } + if (args[0] === 'archive' && args.at(-1) === frozenParentCommit) { + return await runCommand(command, [...args.slice(0, -1), fixture.parentCommit], options); + } + } + return await runCommand(command, args, options); + }; +} + +async function git(cwd: string, args: readonly string[]): Promise { + const result = await runCommand('git', args, { cwd }); + if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); + return result.stdout.trim(); +} + +async function commit(cwd: string, message: string): Promise { + await git(cwd, [ + '-c', + 'user.name=Petrinaut Preflight Fixture', + '-c', + 'user.email=petrinaut-preflight@example.invalid', + 'commit', + '-m', + message, + ]); +} + +const fakeSandboxRunner: CommandRunner = async (command, args) => { + if (command !== 'sandbox-exec') throw new Error(`unexpected verifier command: ${command}`); + if (args[2] === '/usr/bin/curl' && args[3] === '--version') { + return { exitCode: 0, stdout: 'curl fake\n', stderr: '' }; + } + return { exitCode: 1, stdout: '', stderr: 'denied by fake sandbox\n' }; +}; + +const portableVerifier: NonNullable = ( + forbiddenReadRoots, +) => + createNetworkDeniedCommandRunner({ + platform: 'darwin', + forbiddenReadRoots, + run: fakeSandboxRunner, + }); + +const fixedOraclePack: NonNullable< + PetrinautHistoricalPreflightDependencies['loadOraclePack'] +> = async () => ({ + oracleId: 'petrinaut-optimization-oracles-v1', + packSha256: `sha256:${'c'.repeat(64)}`, +}); + +const passingInstall: CommandRunner = async () => ({ + exitCode: 0, + stdout: 'immutable install passed\n', + stderr: '', +}); + +function passingOracleReport() { + return { + schemaVersion: 1 as const, + caseId: 'petrinaut-optimization-v1' as const, + oracleId: 'petrinaut-optimization-oracles-v1' as const, + status: 'passed' as const, + preparation: [ + { + id: 'petrinaut-ui-build', + status: 'passed' as const, + exitCode: 0, + }, + ], + checks: [ + { + id: 'route-and-accessibility' as const, + claims: ['AC1'], + status: 'passed' as const, + evidence: ['route accessible'], + }, + ], + consoleErrors: [], + failedRequests: [], + }; +} + +function setupFailedOracleReport() { + return { + ...passingOracleReport(), + status: 'setup_failed' as const, + checks: [], + setupFailure: 'browser server did not become ready', + }; +} + +function assertionFailedOracleReport() { + return { + ...passingOracleReport(), + status: 'assertion_failed' as const, + checks: [ + { + id: 'route-and-accessibility' as const, + claims: ['AC1'], + status: 'failed' as const, + evidence: ['focused route missing'], + }, + ], + }; +} + +function sha256(value: string): string { + return `sha256:${createHash('sha256').update(value).digest('hex')}`; +} diff --git a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts index 00aa564d2..cf8d7035e 100644 --- a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts +++ b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts @@ -5,11 +5,15 @@ import { fileURLToPath } from 'node:url'; import { afterAll, describe, expect, it } from 'vitest'; +import { runCommand } from '../../../app/command-runner.js'; import { PETRINAUT_FOCUSED_PREPARATION, runPetrinautOptimizationOracle, } from '../petrinaut-optimization-oracle.js'; -import { createKnownGoodPetrinautCandidate } from '../petrinaut-optimization-oracle/fixture.js'; +import { + createInventedLabelsPetrinautCandidate, + createKnownGoodPetrinautCandidate, +} from '../petrinaut-optimization-oracle/fixture.js'; const caseDir = fileURLToPath( new URL('../../../../testing/execution-comparisons/cases/petrinaut-optimization/', import.meta.url), @@ -60,4 +64,72 @@ describe('standalone Petrinaut optimization browser oracle', () => { ); expect(report.consoleErrors).toEqual([]); }, 120_000); + + it('uses semantic DOM readiness while background traffic remains non-idle beyond five seconds', async () => { + const candidateRoot = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-background-readiness-')); + roots.push(candidateRoot); + await createKnownGoodPetrinautCandidate(candidateRoot, { + backgroundRequestDurationMs: 6_000, + }); + + const report = await runPetrinautOptimizationOracle({ + candidateRoot, + caseDir, + }); + + expect(report.status, report.setupFailure ?? JSON.stringify(report, null, 2)).toBe('passed'); + expect(report.failedRequests).toEqual([]); + expect(report.checks.every(({ status }) => status === 'passed')).toBe(true); + }, 120_000); + + it('fails an invented-labels rival that would satisfy the pre-D138 accessibility fiction', async () => { + const candidateRoot = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-invented-labels-')); + roots.push(candidateRoot); + await createInventedLabelsPetrinautCandidate(candidateRoot); + + const report = await runPetrinautOptimizationOracle({ candidateRoot, caseDir }); + + expect(report.status).toBe('assertion_failed'); + expect(report.checks.find(({ id }) => id === 'route-and-accessibility')?.status).toBe('failed'); + }, 120_000); + + it.each([ + [ + 'omission', + (steps: typeof PETRINAUT_FOCUSED_PREPARATION) => steps.filter(({ id }) => id !== 'refractive-build'), + ], + [ + 'reordering', + (steps: typeof PETRINAUT_FOCUSED_PREPARATION) => { + const reordered = [...steps]; + const optimizerIndex = reordered.findIndex(({ id }) => id === 'optimizer-client-build'); + const refractiveIndex = reordered.findIndex(({ id }) => id === 'refractive-build'); + [reordered[optimizerIndex], reordered[refractiveIndex]] = [ + reordered[refractiveIndex]!, + reordered[optimizerIndex]!, + ]; + return reordered; + }, + ], + ] as const)( + 'rejects focused Refractive %s before browser setup', + async (_name, mutate) => { + const candidateRoot = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-sensitive-')); + roots.push(candidateRoot); + await createKnownGoodPetrinautCandidate(candidateRoot); + const results = []; + for (const step of mutate(PETRINAUT_FOCUSED_PREPARATION)) { + const result = await runCommand(step.command, step.args, { + cwd: candidateRoot, + timeoutMs: 30_000, + maxOutputBytes: 16 * 1024, + }); + results.push(result); + if (result.exitCode !== 0) break; + } + expect(results.at(-1)).toMatchObject({ exitCode: 1 }); + expect(results.at(-1)?.stderr).toContain('focused preparation ran out of order'); + }, + 30_000, + ); }); diff --git a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts index 88f99bbbc..404254513 100644 --- a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts +++ b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts @@ -4,12 +4,68 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; +import { PETRINAUT_FOCUSED_PREPARATION } from '../petrinaut-optimization-oracle.js'; import { assessPetrinautFocusedObservation } from '../petrinaut-optimization-oracle/claims.js'; +import { startDeterministicFakeOptimizer } from '../petrinaut-optimization-oracle/fake-optimizer.js'; const oracleRoot = fileURLToPath(new URL('../petrinaut-optimization-oracle/', import.meta.url)); const oracleEntry = fileURLToPath(new URL('../petrinaut-optimization-oracle.ts', import.meta.url)); describe('controller-owned Petrinaut optimization oracle boundary', () => { + it('frames deterministic optimizer events as the upstream Optuna SSE contract', async () => { + const fake = await startDeterministicFakeOptimizer(); + try { + const response = await fetch(`${fake.origin}/optimize/all`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'source fidelity' }), + }); + + expect(response.headers.get('content-type')).toBe('text/event-stream'); + const stream = await response.text(); + expect(stream).toContain('data: {"step":0,"params":{"rate":4},"metric":12,"state":"COMPLETE"}'); + expect(stream).toContain('data: {"step":1,"params":{"rate":6},"metric":10,"state":"COMPLETE"}'); + expect(stream).toContain('event: done\ndata: {}'); + } finally { + await fake.close(); + } + }); + + it('keeps the declared Refractive workspace build directly before Petrinaut UI', () => { + expect(PETRINAUT_FOCUSED_PREPARATION).toEqual([ + { + id: 'design-system-codegen', + command: 'yarn', + args: ['workspace', '@hashintel/ds-components', 'codegen'], + }, + { + id: 'design-system-build', + command: 'yarn', + args: ['workspace', '@hashintel/ds-components', 'build'], + }, + { + id: 'petrinaut-core-build', + command: 'yarn', + args: ['workspace', '@hashintel/petrinaut-core', 'build'], + }, + { + id: 'optimizer-client-build', + command: 'yarn', + args: ['workspace', '@local/petrinaut-optimizer-client', 'build'], + }, + { + id: 'refractive-build', + command: 'yarn', + args: ['workspace', '@hashintel/refractive', 'build'], + }, + { + id: 'petrinaut-ui-build', + command: 'yarn', + args: ['workspace', '@hashintel/petrinaut', 'build'], + }, + ]); + }); + it('contains no historical, candidate-internal, or runtime-selected implementation dependency', async () => { const sources = await sourceFiles(oracleRoot); const source = ( @@ -30,6 +86,16 @@ describe('controller-owned Petrinaut optimization oracle boundary', () => { expect(source).not.toMatch(/implementationPath|oraclePath|pluginPath|manifest\.(?:command|path)/u); }); + it('consumes source-backed mechanical addresses from the typed public contract', async () => { + const browserSource = await readFile(join(oracleRoot, 'browser.ts'), 'utf8'); + expect(browserSource).toMatch(/mechanicalAddresses/u); + expect(browserSource).toMatch(/locate\(page, addresses\.create\)/u); + expect(browserSource).toMatch(/locate\(page, addresses\.metricCode\)/u); + expect(browserSource).not.toMatch(/Create optimization/u); + expect(browserSource).not.toMatch(/getByRole\('heading', \{ name: 'Optimizations'/u); + expect(browserSource).not.toMatch(/getByRole\('tab', \{ name: 'Optimizations'/u); + }); + it.each([ [ 'missing-route', @@ -37,7 +103,6 @@ describe('controller-owned Petrinaut optimization oracle boundary', () => { check: 'route-and-accessibility', pathname: '/processes/draft', expectedPathname: '/optimization', - controlsReachable: true, } as const, 'focused route missing', ], diff --git a/src/dev/execution-comparison/case-contract.ts b/src/dev/execution-comparison/case-contract.ts index 145c453c9..7fbe81175 100644 --- a/src/dev/execution-comparison/case-contract.ts +++ b/src/dev/execution-comparison/case-contract.ts @@ -88,6 +88,31 @@ export interface BrunchHostLandingExecutionCasePublicContract { readonly rules: readonly string[]; } +export type PetrinautMechanicalAddress = + | { readonly kind: 'roleName'; readonly role: string; readonly name: string } + | { readonly kind: 'roleValue'; readonly role: string; readonly value: string } + | { readonly kind: 'roleContents'; readonly role: string; readonly contents: string } + | { readonly kind: 'exactText'; readonly text: string }; + +export type PetrinautMechanicalAddressKey = + | 'skipTour' + | 'dismissAssistant' + | 'simulateMode' + | 'optimizationsNav' + | 'viewTitle' + | 'create' + | 'createDrawer' + | 'scenario' + | 'metric' + | 'metricCode' + | 'directionMaximize' + | 'directionMinimize' + | 'run' + | 'cancel' + | 'statusComplete' + | 'statusError' + | 'statusCancelled'; + export interface PetrinautOptimizationExecutionCasePublicContract { readonly schemaVersion: 1; readonly case: { @@ -120,6 +145,7 @@ export interface PetrinautOptimizationExecutionCasePublicContract { readonly sameOriginApi: '/api/petrinaut-opt/optimize/all'; readonly executionTerminal: 'promotion_prepared'; }; + readonly mechanicalAddresses: Readonly>; readonly rules: readonly string[]; } @@ -334,6 +360,46 @@ function parseBrunchHostLandingContract( return value as unknown as BrunchHostLandingExecutionCasePublicContract; } +const PETRINAUT_MECHANICAL_ADDRESS_KEYS = [ + 'skipTour', + 'dismissAssistant', + 'simulateMode', + 'optimizationsNav', + 'viewTitle', + 'create', + 'createDrawer', + 'scenario', + 'metric', + 'metricCode', + 'directionMaximize', + 'directionMinimize', + 'run', + 'cancel', + 'statusComplete', + 'statusError', + 'statusCancelled', +] as const satisfies readonly PetrinautMechanicalAddressKey[]; + +const PETRINAUT_MECHANICAL_ADDRESSES = { + skipTour: { kind: 'roleName', role: 'button', name: 'Skip tour' }, + dismissAssistant: { kind: 'roleName', role: 'button', name: 'Dismiss' }, + simulateMode: { kind: 'roleName', role: 'radio', name: 'Simulate' }, + optimizationsNav: { kind: 'roleValue', role: 'radio', value: 'optimizations' }, + viewTitle: { kind: 'exactText', text: 'Optimizations' }, + create: { kind: 'roleName', role: 'button', name: 'Create' }, + createDrawer: { kind: 'roleName', role: 'dialog', name: 'Create an optimization' }, + scenario: { kind: 'roleContents', role: 'combobox', contents: 'Select a scenario' }, + metric: { kind: 'roleContents', role: 'combobox', contents: 'Select a metric' }, + metricCode: { kind: 'roleName', role: 'textbox', name: 'Editor content' }, + directionMaximize: { kind: 'roleName', role: 'radio', name: 'Maximize' }, + directionMinimize: { kind: 'roleName', role: 'radio', name: 'Minimize' }, + run: { kind: 'roleName', role: 'button', name: 'Run' }, + cancel: { kind: 'roleName', role: 'button', name: 'Cancel' }, + statusComplete: { kind: 'exactText', text: 'Complete' }, + statusError: { kind: 'exactText', text: 'Error' }, + statusCancelled: { kind: 'exactText', text: 'Cancelled' }, +} as const satisfies Record; + function parsePetrinautOptimizationContract( value: Record, ): PetrinautOptimizationExecutionCasePublicContract { @@ -342,7 +408,7 @@ function parsePetrinautOptimizationContract( const budgets = requiredRecord(value, 'budgets'); const delivery = requiredRecord(value, 'delivery'); const acceptance = requiredRecord(value, 'acceptance'); - const accessibility = requiredRecord(value, 'accessibility'); + const mechanicalAddresses = requiredRecord(value, 'mechanicalAddresses'); if ( !exactKeys(value, [ 'schemaVersion', @@ -350,7 +416,7 @@ function parsePetrinautOptimizationContract( 'budgets', 'delivery', 'acceptance', - 'accessibility', + 'mechanicalAddresses', 'rules', ]) || !exactKeys(caseValue, [ @@ -369,18 +435,7 @@ function parsePetrinautOptimizationContract( !exactKeys(budgets, ['elapsedMinutes', 'mechanicalInterventions', 'substantiveHumanInterventions']) || !exactKeys(delivery, ['runtimeNetwork', 'dependencyInstallNetwork']) || !exactKeys(acceptance, ['publicRoute', 'sameOriginApi', 'executionTerminal']) || - !exactKeys(accessibility, [ - 'view', - 'tab', - 'create', - 'scenario', - 'metric', - 'direction', - 'run', - 'cancel', - 'status', - 'results', - ]) || + !exactKeys(mechanicalAddresses, PETRINAUT_MECHANICAL_ADDRESS_KEYS) || value['schemaVersion'] !== 1 || caseValue['id'] !== 'petrinaut-optimization-v1' || caseValue['specification'] !== 'spec.md' || @@ -402,16 +457,7 @@ function parsePetrinautOptimizationContract( acceptance['publicRoute'] !== '/optimization' || acceptance['sameOriginApi'] !== '/api/petrinaut-opt/optimize/all' || acceptance['executionTerminal'] !== 'promotion_prepared' || - !accessibleName(accessibility['view'], 'heading', 'Optimizations') || - !accessibleName(accessibility['tab'], 'tab', 'Optimizations') || - !accessibleName(accessibility['create'], 'button', 'Create optimization') || - !accessibleName(accessibility['scenario'], 'combobox', 'Scenario') || - !accessibleName(accessibility['metric'], 'combobox', 'Objective metric') || - !accessibleName(accessibility['direction'], 'combobox', 'Objective direction') || - !accessibleName(accessibility['run'], 'button', 'Run optimization') || - !accessibleName(accessibility['cancel'], 'button', 'Cancel optimization') || - !accessibleName(accessibility['status'], 'status', 'Optimization status') || - !accessibleName(accessibility['results'], 'region', 'Optimization results') || + !petrinautMechanicalAddresses(mechanicalAddresses) || !nonemptyStrings(value['rules']) ) { invalid(); @@ -419,6 +465,28 @@ function parsePetrinautOptimizationContract( return value as unknown as PetrinautOptimizationExecutionCasePublicContract; } +function petrinautMechanicalAddresses( + value: Record, +): value is Record { + return PETRINAUT_MECHANICAL_ADDRESS_KEYS.every((key) => + sameMechanicalAddress(value[key], PETRINAUT_MECHANICAL_ADDRESSES[key]), + ); +} + +function sameMechanicalAddress(value: unknown, expected: PetrinautMechanicalAddress): boolean { + if (!record(value) || value['kind'] !== expected.kind) return false; + switch (expected.kind) { + case 'roleName': + return value['role'] === expected.role && value['name'] === expected.name; + case 'roleValue': + return value['role'] === expected.role && value['value'] === expected.value; + case 'roleContents': + return value['role'] === expected.role && value['contents'] === expected.contents; + case 'exactText': + return value['text'] === expected.text; + } +} + function parseJson(raw: string): unknown { try { return JSON.parse(raw) as unknown; diff --git a/src/dev/execution-comparison/historical-replay-target.ts b/src/dev/execution-comparison/historical-replay-target.ts index 7d7077bd2..f15edbbf7 100644 --- a/src/dev/execution-comparison/historical-replay-target.ts +++ b/src/dev/execution-comparison/historical-replay-target.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { isAbsolute, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { runCommand, type CommandRunner } from '../../app/command-runner.js'; +import { runCommand, type CommandResult, type CommandRunner } from '../../app/command-runner.js'; import { createBrunchExecutionLaunch, type ExecutionLaunch, @@ -56,6 +56,52 @@ type HistoricalReplayPreparationPhase = | 'admission' | 'lane_finalization'; +interface PetrinautDependencyPreparationBase { + readonly recipe: typeof PETRINAUT_DEPENDENCY_RECIPE.recipe; + readonly command: typeof PETRINAUT_DEPENDENCY_RECIPE.command; + readonly args: typeof PETRINAUT_DEPENDENCY_RECIPE.args; + readonly exitCode: number; +} + +export interface PetrinautDependencyPreparationResult extends PetrinautDependencyPreparationBase { + readonly status: 'passed'; + readonly exitCode: 0; +} + +export interface PetrinautDependencyPreparationFailure extends PetrinautDependencyPreparationBase { + readonly status: 'failed'; + readonly failureStage: 'install' | 'tracked_source_cleanliness'; +} + +export type PetrinautDependencyPreparationOutcome = + | PetrinautDependencyPreparationResult + | PetrinautDependencyPreparationFailure; + +export interface PetrinautDependencyPreparationObservation { + readonly outcome: PetrinautDependencyPreparationOutcome; + readonly commandResult: CommandResult; + readonly trackedSourceStatus?: string; +} + +export class PetrinautDependencyPreparationError extends Error { + readonly outcome: PetrinautDependencyPreparationFailure; + readonly observation: PetrinautDependencyPreparationObservation; + + constructor( + outcome: PetrinautDependencyPreparationFailure, + observation: PetrinautDependencyPreparationObservation, + ) { + super( + outcome.failureStage === 'install' + ? 'compiled Petrinaut dependency install failed' + : 'compiled Petrinaut dependency install modified tracked source', + ); + this.name = 'PetrinautDependencyPreparationError'; + this.outcome = outcome; + this.observation = observation; + } +} + interface HistoricalReplayReadyBase { readonly status: 'ready'; readonly recipeVersion: typeof RECIPE_VERSION; @@ -70,13 +116,7 @@ interface HistoricalReplayReadyBase { readonly recipe: 'none'; readonly status: 'not_required'; } - | { - readonly recipe: typeof PETRINAUT_DEPENDENCY_RECIPE.recipe; - readonly command: typeof PETRINAUT_DEPENDENCY_RECIPE.command; - readonly args: typeof PETRINAUT_DEPENDENCY_RECIPE.args; - readonly status: 'passed'; - readonly exitCode: 0; - }; + | PetrinautDependencyPreparationResult; readonly launch: ExecutionLaunch; } @@ -102,6 +142,9 @@ export interface HistoricalReplayTargetDependencies { readonly runner?: CommandRunner; readonly dependencyInstallRunner?: CommandRunner; readonly createVerifier?: (forbiddenReadRoots: readonly string[]) => NetworkDeniedCommandRunner; + readonly onPetrinautDependencyPreparation?: ( + observation: PetrinautDependencyPreparationObservation, + ) => Promise | void; } export class HistoricalReplayTargetPreparationError extends Error { @@ -197,10 +240,15 @@ export async function prepareHistoricalReplayTarget( const dependencyPreparation = dependencyRecipe.recipe === 'none' ? ({ ...dependencyRecipe, status: 'not_required' } as const) - : await preparePetrinautDependencies({ + : await preparePetrinautHistoricalReplayDependencies({ targetDir: input.targetDir, runner, dependencyInstallRunner: dependencies.dependencyInstallRunner ?? runCommand, + ...(dependencies.onPetrinautDependencyPreparation === undefined + ? {} + : { + onObservation: dependencies.onPetrinautDependencyPreparation, + }), }); phase = 'admission'; @@ -266,17 +314,12 @@ export async function prepareHistoricalReplayTarget( } } -async function preparePetrinautDependencies(input: { +export async function preparePetrinautHistoricalReplayDependencies(input: { readonly targetDir: string; readonly runner: CommandRunner; readonly dependencyInstallRunner: CommandRunner; -}): Promise<{ - readonly recipe: typeof PETRINAUT_DEPENDENCY_RECIPE.recipe; - readonly command: typeof PETRINAUT_DEPENDENCY_RECIPE.command; - readonly args: typeof PETRINAUT_DEPENDENCY_RECIPE.args; - readonly status: 'passed'; - readonly exitCode: 0; -}> { + readonly onObservation?: (observation: PetrinautDependencyPreparationObservation) => Promise | void; +}): Promise { const result = await input.dependencyInstallRunner( PETRINAUT_DEPENDENCY_RECIPE.command, PETRINAUT_DEPENDENCY_RECIPE.args, @@ -287,18 +330,39 @@ async function preparePetrinautDependencies(input: { }, ); if (result.exitCode !== 0) { - throw new Error( - `${PETRINAUT_DEPENDENCY_RECIPE.command} ${PETRINAUT_DEPENDENCY_RECIPE.args.join( - ' ', - )} controller dependency preparation failed (${result.exitCode}): ${result.stderr || result.stdout}`, - ); + const outcome: PetrinautDependencyPreparationFailure = { + ...PETRINAUT_DEPENDENCY_RECIPE, + status: 'failed', + exitCode: result.exitCode, + failureStage: 'install', + }; + const observation = { outcome, commandResult: result }; + await input.onObservation?.(observation); + throw new PetrinautDependencyPreparationError(outcome, observation); } - await assertTrackedSourceClean(input.runner, input.targetDir, 'controller dependency preparation'); - return { + const trackedSourceStatus = await readTrackedSourceStatus(input.runner, input.targetDir); + if (trackedSourceStatus.length > 0) { + const outcome: PetrinautDependencyPreparationFailure = { + ...PETRINAUT_DEPENDENCY_RECIPE, + status: 'failed', + exitCode: result.exitCode, + failureStage: 'tracked_source_cleanliness', + }; + const observation = { + outcome, + commandResult: result, + trackedSourceStatus, + }; + await input.onObservation?.(observation); + throw new PetrinautDependencyPreparationError(outcome, observation); + } + const outcome: PetrinautDependencyPreparationResult = { ...PETRINAUT_DEPENDENCY_RECIPE, status: 'passed', exitCode: 0, }; + await input.onObservation?.({ outcome, commandResult: result }); + return outcome; } async function assertTrackedSourceClean( @@ -306,14 +370,18 @@ async function assertTrackedSourceClean( targetDir: string, owner: string, ): Promise { - const trackedStatus = ( - await gitChecked(runner, targetDir, ['status', '--porcelain', '--untracked-files=no']) - ).stdout.trim(); + const trackedStatus = await readTrackedSourceStatus(runner, targetDir); if (trackedStatus.length > 0) { throw new Error(`${owner} modified tracked source: ${trackedStatus}`); } } +async function readTrackedSourceStatus(runner: CommandRunner, targetDir: string): Promise { + return ( + await gitChecked(runner, targetDir, ['status', '--porcelain', '--untracked-files=no']) + ).stdout.trim(); +} + async function validateInput(input: { readonly sourceRepositoryDir: string; readonly targetDir: string; diff --git a/src/dev/execution-comparison/petrinaut-historical-preflight.ts b/src/dev/execution-comparison/petrinaut-historical-preflight.ts new file mode 100644 index 000000000..a3ef96b40 --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-historical-preflight.ts @@ -0,0 +1,707 @@ +import { createHash } from 'node:crypto'; +import { lstat, open, readdir, realpath, rm } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { z } from 'zod'; + +import { runCommand, type CommandRunner } from '../../app/command-runner.js'; +import { materializePinnedSourceTree } from '../end-to-end-comparison/solution-isolation.js'; +import { assertControllerIsolation } from '../end-to-end-comparison/study-contract.js'; +import { + HistoricalReplayTargetPreparationError, + PetrinautDependencyPreparationError, + prepareHistoricalReplayTarget, + preparePetrinautHistoricalReplayDependencies, + type HistoricalReplayTargetDependencies, + type PetrinautDependencyPreparationObservation, + type PetrinautDependencyPreparationOutcome, +} from './historical-replay-target.js'; +import { resolveExecutionCase, type ResolvedExecutionCase } from './operator-cli.js'; +import { + createPreflightEvidenceWriter, + PREFLIGHT_EVIDENCE_FILES, + retainedPreflightEvidenceSchema, + type RetainedPreflightEvidence, +} from './petrinaut-historical-preflight/evidence.js'; +import { + runPetrinautOptimizationOracle, + type PetrinautOraclePreparationObservation, +} from './petrinaut-optimization-oracle.js'; +import type { PetrinautOptimizationOracleReport } from './petrinaut-optimization-oracle/types.js'; + +export const PETRINAUT_HISTORICAL_REFERENCE_COMMIT = '276e17d7b0f80c8a80d5abe01849bbb67c6169d0' as const; + +const CASE_REFERENCE = 'petrinaut-optimization-v1'; +const ORACLE_ID = 'petrinaut-optimization-oracles-v1'; +const DEFAULT_CASES_ROOT = fileURLToPath( + new URL('../../../testing/execution-comparisons/cases/', import.meta.url), +); +const HASH = /^sha256:[a-f0-9]{64}$/u; +const GIT_OBJECT = /^[a-f0-9]{40}$/u; +const COMMAND_TRACE = [ + 'prepare_parent_target', + 'materialize_reference', + 'prepare_reference_dependencies', + 'run_compiled_oracle', + 'cleanup_workspaces', +] as const; + +const dependencyPreparationBase = z.object({ + recipe: z.literal('petrinaut-yarn-immutable-v1'), + command: z.literal('corepack'), + args: z.tuple([ + z.literal('yarn'), + z.literal('install'), + z.literal('--immutable'), + z.literal('--mode=skip-build'), + ]), + exitCode: z.number().int(), +}); +const dependencyPreparationSchema = z.discriminatedUnion('status', [ + dependencyPreparationBase + .extend({ + status: z.literal('passed'), + exitCode: z.literal(0), + }) + .strict(), + dependencyPreparationBase + .extend({ + status: z.literal('failed'), + failureStage: z.enum(['install', 'tracked_source_cleanliness']), + }) + .strict(), +]); + +const retainedEvidenceSetSchema = z + .object({ + parentDependency: retainedPreflightEvidenceSchema.optional(), + referenceDependency: retainedPreflightEvidenceSchema.optional(), + oracleSummary: retainedPreflightEvidenceSchema.optional(), + }) + .strict(); + +const cleanupStatusSchema = z.enum(['removed', 'not_created', 'failed']); +const receiptStatusSchema = z.enum(['passed', 'setup_failed', 'assertion_failed']); +const preflightPhaseSchema = z.enum([ + 'parent_preparation', + 'reference_materialization', + 'reference_dependency_preparation', + 'oracle_pack', + 'oracle_calibration', + 'evidence_retention', + 'cleanup', +]); + +const historicalIdentitySchema = z + .object({ + sourceCommit: z.string().regex(GIT_OBJECT), + sourceTree: z.string().regex(GIT_OBJECT), + }) + .strict(); + +export const petrinautHistoricalPreflightReceiptSchema = z + .object({ + schemaVersion: z.literal(1), + caseId: z.literal('petrinaut-optimization-v1'), + status: receiptStatusSchema, + setupStatus: z.enum(['valid', 'invalid']), + parent: historicalIdentitySchema + .extend({ + materializedCommit: z.string().regex(GIT_OBJECT), + packetCommit: z.string().regex(GIT_OBJECT), + dependencyPreparation: dependencyPreparationSchema, + }) + .strict() + .optional(), + reference: historicalIdentitySchema + .extend({ + dependencyPreparation: dependencyPreparationSchema.optional(), + }) + .strict() + .optional(), + oracle: z + .object({ + id: z.literal(ORACLE_ID), + packSha256: z.string().regex(HASH), + reportSha256: z.string().regex(HASH), + reportStatus: z.enum(['passed', 'setup_failed', 'assertion_failed']), + }) + .strict() + .optional(), + evidence: retainedEvidenceSetSchema, + commandTrace: z.tuple([ + z.literal('prepare_parent_target'), + z.literal('materialize_reference'), + z.literal('prepare_reference_dependencies'), + z.literal('run_compiled_oracle'), + z.literal('cleanup_workspaces'), + ]), + cleanup: z + .object({ + parentWorkspace: cleanupStatusSchema, + referenceWorkspace: cleanupStatusSchema, + }) + .strict(), + failure: z + .object({ + phase: preflightPhaseSchema, + dependencyStage: z.enum(['install', 'tracked_source_cleanliness']).optional(), + messageSha256: z.string().regex(HASH), + }) + .strict() + .optional(), + }) + .strict() + .superRefine((receipt, context) => { + if (receipt.status === 'passed' && receipt.setupStatus !== 'valid') { + context.addIssue({ code: 'custom', message: 'passed receipt must be setup-valid' }); + } + if (receipt.status !== 'passed' && receipt.setupStatus !== 'invalid') { + context.addIssue({ code: 'custom', message: 'failed receipt must be setup-invalid' }); + } + if ( + (receipt.status === 'passed' || receipt.status === 'assertion_failed') && + (receipt.parent === undefined || + receipt.reference?.dependencyPreparation?.status !== 'passed' || + receipt.oracle === undefined || + receipt.evidence.parentDependency === undefined || + receipt.evidence.referenceDependency === undefined || + receipt.evidence.oracleSummary === undefined) + ) { + context.addIssue({ + code: 'custom', + message: 'completed calibration receipt requires parent, reference, oracle, and retained evidence', + }); + } + if (receipt.oracle !== undefined && receipt.oracle.reportStatus !== receipt.status) { + context.addIssue({ + code: 'custom', + message: 'receipt status must match the retained oracle report status', + }); + } + if ( + receipt.status !== 'setup_failed' && + (receipt.cleanup.parentWorkspace !== 'removed' || receipt.cleanup.referenceWorkspace !== 'removed') + ) { + context.addIssue({ + code: 'custom', + message: 'completed calibration receipt requires owned workspace cleanup', + }); + } + for (const [key, expectedFile] of Object.entries(PREFLIGHT_EVIDENCE_FILES) as [ + keyof typeof PREFLIGHT_EVIDENCE_FILES, + (typeof PREFLIGHT_EVIDENCE_FILES)[keyof typeof PREFLIGHT_EVIDENCE_FILES], + ][]) { + const retained = receipt.evidence[key]; + if (retained !== undefined && retained.file !== expectedFile) { + context.addIssue({ + code: 'custom', + message: `${key} evidence uses the wrong fixed filename`, + }); + } + } + if ( + (receipt.parent?.dependencyPreparation !== undefined || + (receipt.failure?.phase === 'parent_preparation' && receipt.failure.dependencyStage !== undefined)) && + receipt.evidence.parentDependency === undefined && + receipt.failure?.phase !== 'evidence_retention' + ) { + context.addIssue({ + code: 'custom', + message: 'parent dependency result requires retained evidence', + }); + } + if ( + (receipt.reference?.dependencyPreparation !== undefined || + (receipt.failure?.phase === 'reference_dependency_preparation' && + receipt.failure.dependencyStage !== undefined)) && + receipt.evidence.referenceDependency === undefined && + receipt.failure?.phase !== 'evidence_retention' + ) { + context.addIssue({ + code: 'custom', + message: 'reference dependency result requires retained evidence', + }); + } + if ( + (receipt.oracle !== undefined || receipt.failure?.phase === 'oracle_calibration') && + receipt.evidence.oracleSummary === undefined && + receipt.failure?.phase !== 'evidence_retention' + ) { + context.addIssue({ + code: 'custom', + message: 'oracle calibration requires retained summary evidence', + }); + } + }); + +export type PetrinautHistoricalPreflightReceipt = z.infer; + +export interface PetrinautHistoricalPreflightInput { + readonly sourceRepositoryDir: string; + readonly parentTargetDir: string; + readonly referenceTargetDir: string; + readonly controllerRoot: string; + readonly receiptFile: string; +} + +interface ClosedOraclePack { + readonly oracleId: typeof ORACLE_ID; + readonly packSha256: string; +} + +export interface PetrinautHistoricalPreflightDependencies extends HistoricalReplayTargetDependencies { + /** Test-only frozen-case substitution for self-contained Git fixtures. */ + readonly selectedCase?: ResolvedExecutionCase; + /** Test-only reference substitution for self-contained Git fixtures. */ + readonly referenceCommit?: string; + readonly loadOraclePack?: (caseDir: string) => Promise; + readonly runOracle?: (input: { + readonly candidateRoot: string; + readonly caseDir: string; + readonly onPreparationResult?: ( + observation: PetrinautOraclePreparationObservation, + ) => Promise | void; + }) => Promise; +} + +export async function runPetrinautHistoricalPreflight( + input: PetrinautHistoricalPreflightInput, + dependencies: PetrinautHistoricalPreflightDependencies = {}, +): Promise<{ + readonly receiptFile: string; + readonly receipt: PetrinautHistoricalPreflightReceipt; +}> { + const roots = await validateInput(input); + const receiptHandle = await open(roots.receiptFile, 'wx'); + const runner = dependencies.runner ?? runCommand; + const dependencyInstallRunner = dependencies.dependencyInstallRunner ?? runCommand; + const evidenceWriter = createPreflightEvidenceWriter({ + outputRoot: roots.outputRoot, + redactedRoots: [ + roots.sourceRepositoryDir, + roots.parentTargetDir, + roots.referenceTargetDir, + roots.controllerRoot, + roots.outputRoot, + ], + }); + let phase: z.infer = 'parent_preparation'; + let parent: + | { + readonly sourceCommit: string; + readonly sourceTree: string; + readonly materializedCommit: string; + readonly packetCommit: string; + readonly dependencyPreparation: PetrinautDependencyPreparationOutcome; + } + | undefined; + let reference: + | { + readonly sourceCommit: string; + readonly sourceTree: string; + readonly dependencyPreparation?: PetrinautDependencyPreparationOutcome; + } + | undefined; + let oracle: PetrinautHistoricalPreflightReceipt['oracle']; + let parentObservation: PetrinautDependencyPreparationObservation | undefined; + let referenceObservation: PetrinautDependencyPreparationObservation | undefined; + let oracleSummary: + | { + readonly kind: 'report'; + readonly report: PetrinautOptimizationOracleReport; + } + | { readonly kind: 'failure'; readonly error: unknown } + | undefined; + let parentRemovedEarly = false; + const oraclePreparation: PetrinautOraclePreparationObservation[] = []; + let evidence: { + readonly parentDependency?: RetainedPreflightEvidence; + readonly referenceDependency?: RetainedPreflightEvidence; + readonly oracleSummary?: RetainedPreflightEvidence; + } = {}; + let status: PetrinautHistoricalPreflightReceipt['status'] = 'setup_failed'; + let failure: PetrinautHistoricalPreflightReceipt['failure']; + const retainDependencyObservation = async ( + scope: 'parent' | 'reference', + observation: PetrinautDependencyPreparationObservation, + ): Promise => { + if (scope === 'parent') parentObservation = observation; + else referenceObservation = observation; + await dependencies.onPetrinautDependencyPreparation?.(observation); + }; + + try { + const selectedCase = + dependencies.selectedCase ?? (await resolveExecutionCase(CASE_REFERENCE, DEFAULT_CASES_ROOT)); + if (selectedCase.caseId !== CASE_REFERENCE) { + throw new Error('Petrinaut preflight received a different frozen case'); + } + const parentReady = await prepareHistoricalReplayTarget( + { + lane: 'claude_code', + selectedCase, + sourceRepositoryDir: roots.sourceRepositoryDir, + targetDir: roots.parentTargetDir, + controllerRoot: roots.controllerRoot, + forbiddenRoots: [roots.referenceTargetDir, roots.outputRoot], + }, + { + runner, + dependencyInstallRunner, + onPetrinautDependencyPreparation: async (observation) => + await retainDependencyObservation('parent', observation), + ...(dependencies.createVerifier === undefined ? {} : { createVerifier: dependencies.createVerifier }), + }, + ); + if (parentReady.dependencyPreparation.recipe !== 'petrinaut-yarn-immutable-v1') { + throw new Error('Petrinaut parent used a different dependency recipe'); + } + parent = { + sourceCommit: parentReady.sourceCommit, + sourceTree: parentReady.sourceTree, + materializedCommit: parentReady.materializedCommit, + packetCommit: parentReady.baseSha, + dependencyPreparation: parentReady.dependencyPreparation, + }; + + phase = 'reference_materialization'; + const materializedReference = await materializePinnedSourceTree({ + sourceRepositoryDir: roots.sourceRepositoryDir, + sourceCommit: dependencies.referenceCommit ?? PETRINAUT_HISTORICAL_REFERENCE_COMMIT, + targetDir: roots.referenceTargetDir, + runner, + }); + reference = { + sourceCommit: materializedReference.sourceCommit, + sourceTree: materializedReference.sourceTree, + }; + await assertParentContainsNoReference( + roots.parentTargetDir, + materializedReference.sourceCommit, + roots.referenceTargetDir, + runner, + ); + phase = 'cleanup'; + if ((await removeOwnedWorkspace(roots.parentTargetDir)) !== 'removed') { + throw new Error('owned parent workspace cleanup failed'); + } + parentRemovedEarly = true; + + phase = 'reference_dependency_preparation'; + const referencePreparation = await preparePetrinautHistoricalReplayDependencies({ + targetDir: roots.referenceTargetDir, + runner, + dependencyInstallRunner, + onObservation: async (observation) => { + reference = { + sourceCommit: materializedReference.sourceCommit, + sourceTree: materializedReference.sourceTree, + dependencyPreparation: observation.outcome, + }; + await retainDependencyObservation('reference', observation); + }, + }); + reference = { + sourceCommit: materializedReference.sourceCommit, + sourceTree: materializedReference.sourceTree, + dependencyPreparation: referencePreparation, + }; + + phase = 'oracle_pack'; + const oraclePack = await (dependencies.loadOraclePack ?? loadClosedOraclePack)(selectedCase.caseDir); + if (oraclePack.oracleId !== ORACLE_ID || !HASH.test(oraclePack.packSha256)) { + throw new Error('Petrinaut preflight received a different compiled oracle pack'); + } + + phase = 'oracle_calibration'; + let report: PetrinautOptimizationOracleReport; + try { + report = await (dependencies.runOracle ?? runPetrinautOptimizationOracle)({ + candidateRoot: roots.referenceTargetDir, + caseDir: selectedCase.caseDir, + onPreparationResult: (observation) => { + oraclePreparation.push(observation); + }, + }); + } catch (error) { + oracleSummary = { kind: 'failure', error }; + throw error; + } + oracleSummary = { kind: 'report', report }; + const reportBytes = `${JSON.stringify(report)}\n`; + oracle = { + id: ORACLE_ID, + packSha256: oraclePack.packSha256, + reportSha256: sha256(reportBytes), + reportStatus: report.status, + }; + status = report.status; + } catch (error) { + status = 'setup_failed'; + const dependencyError = findDependencyPreparationError(error); + failure = { + phase, + ...(dependencyError === undefined ? {} : { dependencyStage: dependencyError.outcome.failureStage }), + messageSha256: sha256(redactedError(error, roots)), + }; + } + + const cleanup = await cleanupWorkspaces(roots, parentRemovedEarly); + if (cleanup.parentWorkspace === 'failed' || cleanup.referenceWorkspace === 'failed') { + status = 'setup_failed'; + failure = { + phase: 'cleanup', + messageSha256: sha256('owned workspace cleanup failed'), + }; + } + try { + phase = 'evidence_retention'; + if (parentObservation !== undefined) { + evidence = { + ...evidence, + parentDependency: await evidenceWriter.writeDependency('parent', parentObservation), + }; + } + if (referenceObservation !== undefined) { + evidence = { + ...evidence, + referenceDependency: await evidenceWriter.writeDependency('reference', referenceObservation), + }; + } + if (oracleSummary !== undefined) { + evidence = { + ...evidence, + oracleSummary: + oracleSummary.kind === 'report' + ? await evidenceWriter.writeOracle(oracleSummary.report, oraclePreparation) + : await evidenceWriter.writeOracleFailure(oracleSummary.error, oraclePreparation), + }; + } + } catch (error) { + status = 'setup_failed'; + failure = { + phase: 'evidence_retention', + messageSha256: sha256(redactedError(error, roots)), + }; + } + const receipt = parsePetrinautHistoricalPreflightReceipt({ + schemaVersion: 1, + caseId: CASE_REFERENCE, + status, + setupStatus: status === 'passed' ? 'valid' : 'invalid', + ...(parent === undefined ? {} : { parent }), + ...(reference === undefined ? {} : { reference }), + ...(oracle === undefined ? {} : { oracle }), + evidence, + commandTrace: COMMAND_TRACE, + cleanup, + ...(failure === undefined ? {} : { failure }), + }); + const receiptBytes = `${JSON.stringify(receipt, null, 2)}\n`; + try { + await receiptHandle.writeFile(receiptBytes, 'utf8'); + } finally { + await receiptHandle.close(); + } + return { receiptFile: roots.receiptFile, receipt }; +} + +export function parsePetrinautHistoricalPreflightReceipt( + value: unknown, +): PetrinautHistoricalPreflightReceipt { + return petrinautHistoricalPreflightReceiptSchema.parse(value); +} + +async function loadClosedOraclePack(caseDir: string): Promise { + const [{ loadControllerOraclePack }, { resolveCompiledExecutionOracle }] = await Promise.all([ + import('./oracle-pack.js'), + import('../execution-comparison-operator.js'), + ]); + const compiled = resolveCompiledExecutionOracle(ORACLE_ID); + const pack = await loadControllerOraclePack({ + caseDir, + implementationFiles: compiled.implementationFiles, + }); + if (pack.manifest.id !== ORACLE_ID) { + throw new Error('Petrinaut preflight received a different compiled oracle'); + } + return { oracleId: ORACLE_ID, packSha256: pack.packSha256 }; +} + +async function validateInput(input: PetrinautHistoricalPreflightInput): Promise<{ + readonly sourceRepositoryDir: string; + readonly parentTargetDir: string; + readonly referenceTargetDir: string; + readonly controllerRoot: string; + readonly receiptFile: string; + readonly outputRoot: string; +}> { + for (const [name, path] of Object.entries(input)) { + if (!isAbsolute(path)) { + throw new Error(`Petrinaut preflight ${name} must be an absolute path`); + } + } + const sourceRepositoryDir = await existingRealDirectory(input.sourceRepositoryDir, 'source repository'); + const controllerRoot = await existingRealDirectory(input.controllerRoot, 'controller root'); + const outputRoot = await existingRealDirectory(dirname(input.receiptFile), 'output root'); + const parentRoot = await existingRealDirectory(dirname(input.parentTargetDir), 'parent workspace root'); + const referenceRoot = await existingRealDirectory( + dirname(input.referenceTargetDir), + 'reference workspace root', + ); + const parentTargetDir = resolve(parentRoot, basename(input.parentTargetDir)); + const referenceTargetDir = resolve(referenceRoot, basename(input.referenceTargetDir)); + const receiptFile = resolve(outputRoot, basename(input.receiptFile)); + await assertAbsent(parentTargetDir, 'parent target'); + await assertAbsent(referenceTargetDir, 'reference target'); + for (const file of Object.values(PREFLIGHT_EVIDENCE_FILES)) { + await assertAbsent(resolve(outputRoot, file), `retained evidence ${file}`); + } + assertPairwiseDisjoint([ + sourceRepositoryDir, + controllerRoot, + parentTargetDir, + referenceTargetDir, + outputRoot, + ]); + return { + sourceRepositoryDir, + parentTargetDir, + referenceTargetDir, + controllerRoot, + receiptFile, + outputRoot, + }; +} + +async function existingRealDirectory(path: string, name: string): Promise { + const stat = await lstat(path); + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new Error(`Petrinaut preflight ${name} must be a real directory`); + } + return await realpath(path); +} + +async function assertAbsent(path: string, name: string): Promise { + try { + await lstat(path); + } catch (error) { + if (isNodeError(error, 'ENOENT')) return; + throw error; + } + throw new Error(`Petrinaut preflight ${name} must not already exist`); +} + +function assertPairwiseDisjoint(roots: readonly string[]): void { + for (let index = 0; index < roots.length - 1; index += 1) { + assertControllerIsolation({ + controllerRoot: roots[index]!, + targetRoots: roots.slice(index + 1), + }); + } +} + +async function assertParentContainsNoReference( + parentTargetDir: string, + referenceCommit: string, + referenceTargetDir: string, + runner: CommandRunner, +): Promise { + for (const forbidden of [referenceCommit, referenceTargetDir]) { + const result = await runner('git', ['grep', '-F', '--quiet', forbidden, '--', '.'], { + cwd: parentTargetDir, + timeoutMs: 30_000, + maxOutputBytes: 16 * 1024, + }); + if (result.exitCode === 0) { + throw new Error('historical reference leaked into the parent target'); + } + if (result.exitCode !== 1) { + throw new Error(`parent reference-leak scan failed: ${result.stderr || result.stdout}`); + } + for (const entry of await readdir(parentTargetDir, { withFileTypes: true })) { + if (!entry.isFile()) continue; + const handle = await open(resolve(parentTargetDir, entry.name), 'r'); + try { + const buffer = Buffer.alloc(64 * 1024); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (buffer.subarray(0, bytesRead).toString('utf8').includes(forbidden)) { + throw new Error('historical reference leaked into the parent target'); + } + } finally { + await handle.close(); + } + } + } +} + +async function cleanupWorkspaces( + input: { + readonly parentTargetDir: string; + readonly referenceTargetDir: string; + }, + parentRemovedEarly: boolean, +): Promise<{ + readonly parentWorkspace: 'removed' | 'not_created' | 'failed'; + readonly referenceWorkspace: 'removed' | 'not_created' | 'failed'; +}> { + const [parentWorkspace, referenceWorkspace] = await Promise.all([ + parentRemovedEarly ? Promise.resolve('removed' as const) : removeOwnedWorkspace(input.parentTargetDir), + removeOwnedWorkspace(input.referenceTargetDir), + ]); + return { parentWorkspace, referenceWorkspace }; +} + +async function removeOwnedWorkspace(path: string): Promise<'removed' | 'not_created' | 'failed'> { + try { + await lstat(path); + } catch (error) { + return isNodeError(error, 'ENOENT') ? 'not_created' : 'failed'; + } + try { + await rm(path, { recursive: true }); + return 'removed'; + } catch { + return 'failed'; + } +} + +function redactedError( + error: unknown, + roots: { + readonly sourceRepositoryDir: string; + readonly parentTargetDir: string; + readonly referenceTargetDir: string; + readonly controllerRoot: string; + readonly outputRoot: string; + }, +): string { + let message = error instanceof Error ? error.message : String(error); + for (const root of Object.values(roots)) { + message = message.split(root).join(''); + } + return message; +} + +function findDependencyPreparationError(error: unknown): PetrinautDependencyPreparationError | undefined { + if (error instanceof PetrinautDependencyPreparationError) return error; + if (error instanceof HistoricalReplayTargetPreparationError) { + return findDependencyPreparationError(error.cause); + } + return undefined; +} + +function sha256(value: string): string { + return `sha256:${createHash('sha256').update(value).digest('hex')}`; +} + +function isNodeError(error: unknown, code: string): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code?: string }).code === code + ); +} diff --git a/src/dev/execution-comparison/petrinaut-historical-preflight/evidence.ts b/src/dev/execution-comparison/petrinaut-historical-preflight/evidence.ts new file mode 100644 index 000000000..07c88d3dc --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-historical-preflight/evidence.ts @@ -0,0 +1,255 @@ +import { createHash } from 'node:crypto'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { z } from 'zod'; + +import type { PetrinautDependencyPreparationObservation } from '../historical-replay-target.js'; +import type { PetrinautOraclePreparationObservation } from '../petrinaut-optimization-oracle/runner.js'; +import type { PetrinautOptimizationOracleReport } from '../petrinaut-optimization-oracle/types.js'; + +const HASH = /^sha256:[a-f0-9]{64}$/u; +const MAX_STREAM_BYTES = 48 * 1024; +const MAX_STATUS_BYTES = 16 * 1024; +const MAX_ORACLE_STREAM_BYTES = 8 * 1024; + +export const PREFLIGHT_EVIDENCE_FILES = { + parentDependency: 'parent-dependency.json', + referenceDependency: 'reference-dependency.json', + oracleSummary: 'oracle-summary.json', +} as const; + +export const retainedPreflightEvidenceSchema = z + .object({ + file: z.enum([ + PREFLIGHT_EVIDENCE_FILES.parentDependency, + PREFLIGHT_EVIDENCE_FILES.referenceDependency, + PREFLIGHT_EVIDENCE_FILES.oracleSummary, + ]), + sha256: z.string().regex(HASH), + bytes: z.number().int().positive(), + truncated: z.boolean(), + }) + .strict(); + +export type RetainedPreflightEvidence = z.infer; + +export function createPreflightEvidenceWriter(input: { + readonly outputRoot: string; + readonly redactedRoots: readonly string[]; +}): { + readonly writeDependency: ( + scope: 'parent' | 'reference', + observation: PetrinautDependencyPreparationObservation, + ) => Promise; + readonly writeOracle: ( + report: PetrinautOptimizationOracleReport, + preparation: readonly PetrinautOraclePreparationObservation[], + ) => Promise; + readonly writeOracleFailure: ( + error: unknown, + preparation: readonly PetrinautOraclePreparationObservation[], + ) => Promise; +} { + return { + writeDependency: async (scope, observation) => { + const stdout = sanitizeBounded(observation.commandResult.stdout, input.redactedRoots, MAX_STREAM_BYTES); + const stderr = sanitizeBounded(observation.commandResult.stderr, input.redactedRoots, MAX_STREAM_BYTES); + const spawnError = + observation.commandResult.spawnError === undefined + ? undefined + : sanitizeBounded(observation.commandResult.spawnError, input.redactedRoots, MAX_STATUS_BYTES); + const trackedSourceStatus = + observation.trackedSourceStatus === undefined + ? undefined + : sanitizeBounded(observation.trackedSourceStatus, input.redactedRoots, MAX_STATUS_BYTES); + return await retain( + input.outputRoot, + scope === 'parent' + ? PREFLIGHT_EVIDENCE_FILES.parentDependency + : PREFLIGHT_EVIDENCE_FILES.referenceDependency, + { + schemaVersion: 1, + kind: 'dependency_preparation', + scope, + recipe: observation.outcome.recipe, + command: observation.outcome.command, + args: observation.outcome.args, + status: observation.outcome.status, + exitCode: observation.outcome.exitCode, + ...('failureStage' in observation.outcome + ? { failureStage: observation.outcome.failureStage } + : {}), + commandResult: { + stdout: stdout.value, + stderr: stderr.value, + ...(spawnError === undefined ? {} : { spawnError: spawnError.value }), + aborted: observation.commandResult.aborted === true, + timedOut: observation.commandResult.timedOut === true, + outputTruncated: + observation.commandResult.outputTruncated === true || stdout.truncated || stderr.truncated, + }, + ...(trackedSourceStatus === undefined ? {} : { trackedSourceStatus: trackedSourceStatus.value }), + }, + observation.commandResult.outputTruncated === true || + stdout.truncated || + stderr.truncated || + spawnError?.truncated === true || + trackedSourceStatus?.truncated === true, + ); + }, + writeOracle: async (report, preparation) => { + const setupFailure = + report.setupFailure === undefined + ? undefined + : sanitizeBounded(report.setupFailure, input.redactedRoots, MAX_STATUS_BYTES); + const consoleErrors = sanitizeList(report.consoleErrors, input.redactedRoots); + const failedRequests = sanitizeList(report.failedRequests, input.redactedRoots); + const preparationLogs = sanitizeOraclePreparation(preparation, input.redactedRoots); + return await retain( + input.outputRoot, + PREFLIGHT_EVIDENCE_FILES.oracleSummary, + { + schemaVersion: 1, + kind: 'oracle_summary', + oracleId: report.oracleId, + status: report.status, + preparation: report.preparation, + preparationLogs: preparationLogs.values, + checks: report.checks.map(({ id, claims, status, evidence }) => ({ + id, + claims, + status, + evidence: sanitizeList(evidence, input.redactedRoots).values, + })), + ...(setupFailure === undefined ? {} : { setupFailure: setupFailure.value }), + consoleErrors: consoleErrors.values, + failedRequests: failedRequests.values, + }, + setupFailure?.truncated === true || + consoleErrors.truncated || + failedRequests.truncated || + preparationLogs.truncated, + ); + }, + writeOracleFailure: async (error, preparation) => { + const failure = sanitizeBounded( + error instanceof Error ? error.message : String(error), + input.redactedRoots, + MAX_STATUS_BYTES, + ); + const preparationLogs = sanitizeOraclePreparation(preparation, input.redactedRoots); + return await retain( + input.outputRoot, + PREFLIGHT_EVIDENCE_FILES.oracleSummary, + { + schemaVersion: 1, + kind: 'oracle_summary', + oracleId: 'petrinaut-optimization-oracles-v1', + status: 'setup_failed', + thrownFailure: failure.value, + preparationLogs: preparationLogs.values, + }, + failure.truncated || preparationLogs.truncated, + ); + }, + }; +} + +function sanitizeOraclePreparation( + observations: readonly PetrinautOraclePreparationObservation[], + roots: readonly string[], +): { readonly values: readonly unknown[]; readonly truncated: boolean } { + let truncated = false; + const values = observations.map(({ id, commandResult }) => { + const stdout = sanitizeBounded(commandResult.stdout, roots, MAX_ORACLE_STREAM_BYTES); + const stderr = sanitizeBounded(commandResult.stderr, roots, MAX_ORACLE_STREAM_BYTES); + const spawnError = + commandResult.spawnError === undefined + ? undefined + : sanitizeBounded(commandResult.spawnError, roots, MAX_ORACLE_STREAM_BYTES); + truncated ||= + commandResult.outputTruncated === true || + stdout.truncated || + stderr.truncated || + spawnError?.truncated === true; + return { + id, + exitCode: commandResult.exitCode, + stdout: stdout.value, + stderr: stderr.value, + ...(spawnError === undefined ? {} : { spawnError: spawnError.value }), + aborted: commandResult.aborted === true, + timedOut: commandResult.timedOut === true, + outputTruncated: commandResult.outputTruncated === true || stdout.truncated || stderr.truncated, + }; + }); + return { values, truncated }; +} + +async function retain( + outputRoot: string, + file: RetainedPreflightEvidence['file'], + value: unknown, + truncated: boolean, +): Promise { + const bytes = `${JSON.stringify(value, null, 2)}\n`; + await writeFile(join(outputRoot, file), bytes, { + encoding: 'utf8', + flag: 'wx', + }); + return retainedPreflightEvidenceSchema.parse({ + file, + sha256: sha256(bytes), + bytes: Buffer.byteLength(bytes, 'utf8'), + truncated, + }); +} + +function sanitizeList( + values: readonly string[], + roots: readonly string[], +): { readonly values: readonly string[]; readonly truncated: boolean } { + let remaining = MAX_STREAM_BYTES; + let truncated = false; + const sanitized: string[] = []; + for (const value of values) { + if (remaining <= 0) { + truncated = true; + break; + } + const result = sanitizeBounded(value, roots, remaining); + sanitized.push(result.value); + remaining -= Buffer.byteLength(result.value, 'utf8'); + truncated ||= result.truncated; + } + return { values: sanitized, truncated }; +} + +function sanitizeBounded( + value: string, + roots: readonly string[], + maxBytes: number, +): { readonly value: string; readonly truncated: boolean } { + let sanitized = value; + for (const root of [...roots].sort((left, right) => right.length - left.length)) { + sanitized = sanitized.split(root).join(''); + } + sanitized = sanitized + .replace(/:\/\/[^/\s:@]+:[^/\s@]+@/gu, '://@') + .replace(/\b(token|password|authorization|api[_-]?key)\s*[:=]\s*[^\s,;]+/giu, '$1=') + .replace(/(^|[\s("'=])\/(?:[^/\s"'(),;]+\/)*[^/\s"'(),;]*/gmu, '$1') + .replace(/[A-Za-z]:\\(?:[^\\\s"'(),;]+\\)*[^\\\s"'(),;]*/gu, ''); + const buffer = Buffer.from(sanitized, 'utf8'); + if (buffer.byteLength <= maxBytes) { + return { value: sanitized, truncated: false }; + } + return { + value: buffer.subarray(0, maxBytes).toString('utf8'), + truncated: true, + }; +} + +function sha256(value: string): string { + return `sha256:${createHash('sha256').update(value).digest('hex')}`; +} diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle.ts index cedc9cd60..96a945468 100644 --- a/src/dev/execution-comparison/petrinaut-optimization-oracle.ts +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle.ts @@ -2,6 +2,7 @@ export { PETRINAUT_FOCUSED_PREPARATION, runPetrinautOptimizationOracle, } from './petrinaut-optimization-oracle/runner.js'; +export type { PetrinautOraclePreparationObservation } from './petrinaut-optimization-oracle/runner.js'; export type { PetrinautOptimizationCheckId, PetrinautOptimizationOracleCheck, diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts index 7e6c8cf4f..bec3a4587 100644 --- a/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts @@ -1,15 +1,23 @@ import { spawn, type ChildProcess } from 'node:child_process'; -import { access } from 'node:fs/promises'; +import { access, readFile } from 'node:fs/promises'; import { createServer } from 'node:http'; +import { fileURLToPath } from 'node:url'; -import { chromium, type Browser, type Page } from 'playwright-core'; +import { chromium, type Browser, type Locator, type Page } from 'playwright-core'; -import type { PetrinautOptimizationExecutionCasePublicContract } from '../case-contract.js'; +import type { + PetrinautMechanicalAddress, + PetrinautOptimizationExecutionCasePublicContract, +} from '../case-contract.js'; import type { PetrinautOptimizationControllerOracleManifest } from '../oracle-pack.js'; import { requirePetrinautFocusedObservation } from './claims.js'; import { startDeterministicFakeOptimizer } from './fake-optimizer.js'; import type { PetrinautOptimizationOracleCheck } from './types.js'; +const SEMANTIC_ACTION_TIMEOUT_MS = 5_000; +const NAVIGATION_TIMEOUT_MS = 30_000; +const CALIBRATION_SEED_PATH = fileURLToPath(new URL('./calibration-seed.json', import.meta.url)); + export async function runPetrinautBrowserChecks(input: { readonly candidateRoot: string; readonly contract: PetrinautOptimizationExecutionCasePublicContract; @@ -42,6 +50,7 @@ export async function runPetrinautBrowserChecks(input: { let browser: Browser | undefined; const consoleErrors: string[] = []; const failedRequests: string[] = []; + const calibrationSeed = JSON.parse(await readFile(CALIBRATION_SEED_PATH, 'utf8')) as unknown; try { await waitForRoute( `${candidateOrigin}${input.contract.acceptance.publicRoute}`, @@ -60,8 +69,11 @@ export async function runPetrinautBrowserChecks(input: { const definition = definitions.get(declared.id); if (definition === undefined) throw new Error(`missing Petrinaut check implementation: ${declared.id}`); const context = await browser.newContext(); + await context.addInitScript((seed) => { + localStorage.setItem('petrinaut-sdcpn', JSON.stringify({ [(seed as { id: string }).id]: seed })); + }, calibrationSeed); const page = await context.newPage(); - page.setDefaultTimeout(5_000); + page.setDefaultTimeout(SEMANTIC_ACTION_TIMEOUT_MS); page.on('console', (message) => { if (message.type() === 'error') consoleErrors.push(message.text()); }); @@ -75,7 +87,8 @@ export async function runPetrinautBrowserChecks(input: { ); try { await page.goto(`${candidateOrigin}${input.contract.acceptance.publicRoute}`, { - waitUntil: 'networkidle', + waitUntil: 'domcontentloaded', + timeout: NAVIGATION_TIMEOUT_MS, }); const evidence = await definition(page); checks.push({ id: declared.id, claims: declared.claims, status: 'passed', evidence }); @@ -106,57 +119,47 @@ function checkDefinitions(input: { readonly fakeOrigin: string; readonly requests: { readonly body: unknown; readonly aborted: boolean }[]; }): ReadonlyMap { + const addresses = input.contract.mechanicalAddresses; return new Map([ [ 'route-and-accessibility', async (page) => { - await requireCount(page.getByRole('heading', { name: 'Optimizations', exact: true }), 1, 'view'); - await requireCount(page.getByRole('tab', { name: 'Optimizations', exact: true }), 1, 'tab'); - await page.getByRole('button', { name: 'Create optimization', exact: true }).click(); - await requireCount(page.getByRole('combobox', { name: 'Scenario', exact: true }), 1, 'scenario'); - await page.getByRole('combobox', { name: 'Scenario', exact: true }).selectOption('baseline'); - let controlsReachable = true; - for (const [role, name] of [ - ['combobox', 'Objective metric'], - ['combobox', 'Objective direction'], - ['button', 'Run optimization'], - ] as const) { - const locator = page.getByRole(role, { name, exact: true }); - await locator.focus(); - controlsReachable &&= await locator.evaluate((element) => element === document.activeElement); - } + await navigateToOptimizations(page, addresses); + await requireCount(locate(page, addresses.viewTitle), 1, 'viewTitle'); + await requireCount(locate(page, addresses.create), 1, 'create'); + await openCreateDrawer(page, addresses); + await requireCount(locate(page, addresses.createDrawer), 1, 'createDrawer'); + await requireCount(locate(page, addresses.scenario), 1, 'scenario'); + await selectScenarioOption(page, addresses, /Seasonal Flu/u); + await requireCount(locate(page, addresses.metric), 1, 'metric'); + await requireCount(locate(page, addresses.directionMaximize), 1, 'directionMaximize'); + await requireCount(locate(page, addresses.run), 1, 'run'); requirePetrinautFocusedObservation({ check: 'route-and-accessibility', pathname: new URL(page.url()).pathname, expectedPathname: input.contract.acceptance.publicRoute, - controlsReachable, }); return [ 'public /optimization route ready', - 'required controls expose stable roles and keyboard focus', + 'required controls expose stable source-backed mechanical addresses', ]; }, ], [ 'scenario-configuration', async (page) => { - await openConfiguration(page); - const optimize = page.getByRole('checkbox', { name: 'Optimize rate', exact: true }); - await optimize.check(); - await page.getByRole('spinbutton', { name: 'rate minimum', exact: true }).fill('2'); - await page - .getByRole('combobox', { name: 'Objective direction', exact: true }) - .selectOption('minimize'); - await page.getByRole('combobox', { name: 'Scenario', exact: true }).selectOption('surge'); - assert(!(await optimize.isChecked()), 'scenario change retained optimized binding'); + await openConfiguration(page, addresses); + const optimize = page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }); + await optimize.click({ force: true }); + assert(await optimize.isChecked(), 'optimize toggle did not enable'); + await locate(page, addresses.directionMinimize).click({ force: true }); + await selectScenarioOption(page, addresses, /High Virulence Outbreak/u); assert( - (await page.getByRole('spinbutton', { name: 'rate fixed value', exact: true }).inputValue()) === - '8', - 'scenario change did not reset fixed value', + !(await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).isChecked()), + 'scenario change retained optimized binding', ); assert( - (await page.getByRole('combobox', { name: 'Objective direction', exact: true }).inputValue()) === - 'maximize', + !(await locate(page, addresses.directionMinimize).isChecked()), 'scenario change retained metric direction', ); return [ @@ -168,54 +171,67 @@ function checkDefinitions(input: { [ 'request-contract', async (page) => { - await openConfiguration(page); + await openConfiguration(page, addresses); const savedRequestIndex = input.requests.length; - await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); - await status(page, /Complete/u); + await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ + force: true, + }); + await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await locate(page, addresses.directionMaximize).click({ force: true }); + await setOptimizationName(page, 'saved metric proof'); + await locate(page, addresses.run).click(); + await waitForAddress(page, addresses.statusComplete); const savedBody = input.requests[savedRequestIndex]?.body; assert(record(savedBody), 'fake optimizer did not capture the saved-metric request'); const savedObjective = savedBody['objective']; assert( record(savedObjective) && savedObjective['direction'] === 'maximize' && - record(savedObjective['metric']) && - savedObjective['metric']['source'] === 'saved', + typeof savedObjective['metricId'] === 'string', 'saved objective missing', ); + await dismissOverlayDrawers(page); + await openCreateDrawer(page, addresses); + await selectScenarioOption(page, addresses, /Seasonal Flu/u); const customRequestIndex = input.requests.length; - await page.getByRole('textbox', { name: 'Optimization name', exact: true }).fill('request proof'); - await page.getByRole('checkbox', { name: 'Optimize rate', exact: true }).check(); - await page.getByRole('spinbutton', { name: 'rate minimum', exact: true }).fill('2'); - await page.getByRole('spinbutton', { name: 'rate maximum', exact: true }).fill('9'); - await page.getByRole('spinbutton', { name: 'demand fixed value', exact: true }).fill('7'); - await page.getByRole('combobox', { name: 'Objective metric', exact: true }).selectOption('custom'); - await page.getByRole('textbox', { name: 'Custom metric code', exact: true }).fill('return 42;'); - await page - .getByRole('combobox', { name: 'Objective direction', exact: true }) - .selectOption('minimize'); - await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); - await status(page, /Complete/u); + await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ + force: true, + }); + await selectComboboxOption(page, addresses.metric, /Custom code/u); + await locate(page, addresses.metricCode).fill('return 42;'); + await locate(page, addresses.directionMinimize).click({ force: true }); + await setOptimizationName(page, 'custom metric proof'); + await locate(page, addresses.run).click(); + await waitForAddress(page, addresses.statusComplete); const body = input.requests[customRequestIndex]?.body; assert(record(body), 'fake optimizer did not capture a JSON request'); assert( - record(body['scenario']) && body['scenario']['id'] === 'baseline', + record(body['scenario']) && typeof body['scenario']['id'] === 'string', 'scenario missing from request', ); const bindings = record(body['scenario']) ? body['scenario']['parameterBindings'] : undefined; assert(record(bindings), 'parameter bindings missing from request'); assert( - record(bindings['rate']) && bindings['rate']['kind'] === 'optimize', + Object.values(bindings).some((binding) => record(binding) && binding['kind'] === 'optimize'), 'optimized binding missing', ); - assert(record(bindings['demand']) && bindings['demand']['value'] === 7, 'fixed binding missing'); + assert( + Object.values(bindings).some((binding) => record(binding) && binding['kind'] === 'fixed'), + 'fixed binding missing', + ); const objective = body['objective']; assert(record(objective) && objective['direction'] === 'minimize', 'objective direction missing'); + assert(record(objective) && typeof objective['metricId'] === 'string', 'custom objective missing'); + const model = body['model']; assert( - record(objective['metric']) && - objective['metric']['source'] === 'custom' && - objective['metric']['code'] === 'return 42;', - 'custom objective missing', + record(model) && + record(model['definition']) && + Array.isArray(model['definition']['metrics']) && + model['definition']['metrics'].some( + (metric) => record(metric) && metric['code'] === 'return 42;', + ), + 'custom metric code missing from request model', ); return [ 'captured flat fixed/optimized bindings', @@ -226,19 +242,23 @@ function checkDefinitions(input: { [ 'progress-and-completion', async (page) => { - await openConfiguration(page); - await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); - await page.getByText('Trial 1: 12', { exact: true }).waitFor(); - await page.getByText('Best so far: 12', { exact: true }).first().waitFor(); - await status(page, /Complete/u); + await openConfiguration(page, addresses); + await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ + force: true, + }); + await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await locate(page, addresses.directionMaximize).click({ force: true }); + await setOptimizationName(page, 'progress proof'); + await locate(page, addresses.run).click(); + await waitForAddress(page, addresses.statusComplete); + const progressiveTrialCount = await page.getByText(/^\d+$/u).count(); + const bestSoFarVisible = (await page.getByText('Best', { exact: true }).count()) > 0; + const completionVisible = (await locate(page, addresses.statusComplete).count()) > 0; requirePetrinautFocusedObservation({ check: 'progress-and-completion', - progressiveTrialCount: await page.getByText(/^Trial \d+: /u).count(), - bestSoFarVisible: (await page.getByText(/^Best so far: /u).count()) > 0, - completionVisible: await page - .getByRole('status', { name: 'Optimization status', exact: true }) - .filter({ hasText: /Complete/u }) - .isVisible(), + progressiveTrialCount, + bestSoFarVisible, + completionVisible, }); return ['progressive trial rendered', 'best-so-far rendered', 'completion rendered']; }, @@ -246,10 +266,15 @@ function checkDefinitions(input: { [ 'service-error', async (page) => { - await openConfiguration(page); - await page.getByRole('textbox', { name: 'Optimization name', exact: true }).fill('service failure'); - await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); - await status(page, /Error: Deterministic optimizer failure/u); + await openConfiguration(page, addresses); + await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ + force: true, + }); + await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await locate(page, addresses.directionMaximize).click({ force: true }); + await setOptimizationName(page, 'service failure'); + await locate(page, addresses.run).click(); + await waitForAddress(page, addresses.statusError); return ['service error rendered distinctly']; }, ], @@ -257,18 +282,33 @@ function checkDefinitions(input: { 'cancel-and-abort', async (page) => { const requestIndex = input.requests.length; - await openConfiguration(page); - await page.getByRole('textbox', { name: 'Optimization name', exact: true }).fill('cancel proof'); - await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); - const cancel = page.getByRole('button', { name: 'Cancel optimization', exact: true }); + await openConfiguration(page, addresses); + await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ + force: true, + }); + await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await locate(page, addresses.directionMaximize).click({ force: true }); + await setOptimizationName(page, 'cancel proof'); + await locate(page, addresses.run).click(); + // Create closes and the view drawer opens on the active record (initializing/running). + await page + .getByText('Running', { exact: true }) + .filter({ visible: true }) + .or(page.getByText('Initializing', { exact: true }).filter({ visible: true })) + .first() + .waitFor(); + const cancel = locate(page, addresses.cancel); await cancel.waitFor(); await cancel.focus(); assert( - await cancel.evaluate((element) => element === document.activeElement), + await cancel.evaluate((element) => { + const active = document.activeElement; + return active === element || (active !== null && element.contains(active)); + }), 'cancel is not focusable', ); await cancel.press('Enter'); - await status(page, /Cancelled/u); + await waitForAddress(page, addresses.statusCancelled); await waitFor( () => input.requests[requestIndex]?.aborted === true, 'upstream request was not aborted', @@ -277,10 +317,7 @@ function checkDefinitions(input: { check: 'cancel-and-abort', cancelControlVisible: true, hostRequestAborted: input.requests[requestIndex]?.aborted === true, - cancelledVisible: await page - .getByRole('status', { name: 'Optimization status', exact: true }) - .filter({ hasText: /Cancelled/u }) - .isVisible(), + cancelledVisible: (await locate(page, addresses.statusCancelled).count()) > 0, }); return ['cancel keyboard control rendered', 'host request aborted', 'cancelled state rendered']; }, @@ -290,9 +327,15 @@ function checkDefinitions(input: { async (page) => { const browserRequests: string[] = []; page.on('request', (request) => browserRequests.push(request.url())); - await openConfiguration(page); - await page.getByRole('button', { name: 'Run optimization', exact: true }).click(); - await status(page, /Complete/u); + await openConfiguration(page, addresses); + await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ + force: true, + }); + await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await locate(page, addresses.directionMaximize).click({ force: true }); + await setOptimizationName(page, 'secrecy proof'); + await locate(page, addresses.run).click(); + await waitForAddress(page, addresses.statusComplete); requirePetrinautFocusedObservation({ check: 'private-origin-secrecy', candidateOrigin: input.candidateOrigin, @@ -306,28 +349,150 @@ function checkDefinitions(input: { ]); } -async function openConfiguration(page: Page): Promise { - await page.getByRole('button', { name: 'Create optimization', exact: true }).click(); - const scenario = page.getByRole('combobox', { name: 'Scenario', exact: true }); +async function navigateToOptimizations( + page: Page, + addresses: PetrinautOptimizationExecutionCasePublicContract['mechanicalAddresses'], +): Promise { + const skip = locate(page, addresses.skipTour); + if ((await skip.count()) > 0) await skip.click({ force: true }); + const dismiss = locate(page, addresses.dismissAssistant); + if ((await dismiss.count()) > 0) await dismiss.click({ force: true }); + await locate(page, addresses.simulateMode).click({ force: true }); + await locate(page, addresses.optimizationsNav).click({ force: true }); + await locate(page, addresses.viewTitle).waitFor(); +} + +async function openConfiguration( + page: Page, + addresses: PetrinautOptimizationExecutionCasePublicContract['mechanicalAddresses'], +): Promise { + await navigateToOptimizations(page, addresses); + await openCreateDrawer(page, addresses); assert( - (await page.getByRole('button', { name: 'Run optimization', exact: true }).count()) === 0, + (await page.getByRole('checkbox', { name: /^Optimize /u }).count()) === 0, 'configuration appeared before scenario selection', ); - await scenario.selectOption('baseline'); + await selectScenarioOption(page, addresses, /Seasonal Flu/u); } -async function status(page: Page, pattern: RegExp): Promise { - await page - .getByRole('status', { name: 'Optimization status', exact: true }) - .filter({ hasText: pattern }) - .waitFor(); +async function openCreateDrawer( + page: Page, + addresses: PetrinautOptimizationExecutionCasePublicContract['mechanicalAddresses'], +): Promise { + if ((await locate(page, addresses.createDrawer).count()) === 0) { + await dismissOverlayDrawers(page); + await locate(page, addresses.create).click({ force: true }); + } + await locate(page, addresses.createDrawer).waitFor(); + await locate(page, addresses.run).waitFor(); + assert((await locate(page, addresses.run).count()) === 1, 'create drawer did not expose Run control'); } -async function requireCount( - locator: ReturnType, - expected: number, - label: string, +async function dismissOverlayDrawers(page: Page): Promise { + for (let attempt = 0; attempt < 3; attempt += 1) { + if ((await page.getByRole('dialog').count()) === 0) return; + await page.keyboard.press('Escape'); + try { + await page.getByRole('dialog').first().waitFor({ state: 'hidden', timeout: 500 }); + } catch { + // Keep dismissing until no dialog remains or attempts exhaust. + } + } +} + +function locate(page: Page, address: PetrinautMechanicalAddress): Locator { + switch (address.kind) { + case 'roleName': + return page.getByRole(address.role as Parameters[0], { + name: address.name, + exact: true, + }); + case 'roleValue': + return page.locator(`${cssRoleSelector(address.role)}[value="${cssEscape(address.value)}"]`); + case 'roleContents': + return page.getByRole(address.role as Parameters[0]).filter({ + hasText: address.contents, + }); + case 'exactText': + // exactText never resolves role=tooltip nodes (nav tooltips reuse titles). + return page + .getByText(address.text, { exact: true }) + .and(page.locator(':not([role="tooltip"])')) + .filter({ visible: true }); + } +} + +async function selectScenarioOption( + page: Page, + addresses: PetrinautOptimizationExecutionCasePublicContract['mechanicalAddresses'], + optionPattern: RegExp, ): Promise { + const drawer = locate(page, addresses.createDrawer); + await drawer.waitFor(); + const empty = locate(page, addresses.scenario); + // Empty-state address when present; after selection the same dialog combobox + // keeps the scenario control (placeholder text no longer matches). + const combobox = (await empty.count()) > 0 ? empty : drawer.getByRole('combobox').first(); + const tagName = await combobox.evaluate((element) => element.tagName); + if (tagName === 'SELECT') { + const value = await combobox.evaluate((element, source) => { + const pattern = new RegExp(source, 'u'); + for (const option of Array.from((element as HTMLSelectElement).options)) { + if (pattern.test(option.textContent ?? '')) return option.value; + } + return null; + }, optionPattern.source); + assert(value !== null && value.length > 0, `no select option matched ${optionPattern}`); + await combobox.selectOption(value); + return; + } + await combobox.click({ force: true }); + await page.getByRole('option').filter({ hasText: optionPattern }).first().click(); +} + +function cssRoleSelector(role: string): string { + if (role === 'radio') return 'input[type="radio"]'; + return `[role="${cssEscape(role)}"]`; +} + +function cssEscape(value: string): string { + return value.replaceAll('\\', '\\\\').replaceAll('"', '\\"'); +} + +async function selectComboboxOption( + page: Page, + address: PetrinautMechanicalAddress, + optionPattern: RegExp, +): Promise { + const combobox = locate(page, address); + const tagName = await combobox.evaluate((element) => element.tagName); + if (tagName === 'SELECT') { + const value = await combobox.evaluate((element, source) => { + const pattern = new RegExp(source, 'u'); + for (const option of Array.from((element as HTMLSelectElement).options)) { + if (pattern.test(option.textContent ?? '')) return option.value; + } + return null; + }, optionPattern.source); + assert(value !== null && value.length > 0, `no select option matched ${optionPattern}`); + await combobox.selectOption(value); + return; + } + await combobox.click({ force: true }); + await page.getByRole('option').filter({ hasText: optionPattern }).first().click(); +} + +async function setOptimizationName(page: Page, name: string): Promise { + const dialog = page.getByRole('dialog', { name: 'Create an optimization', exact: true }); + const nameField = dialog.getByRole('textbox').first(); + await nameField.fill(name); +} + +async function waitForAddress(page: Page, address: PetrinautMechanicalAddress): Promise { + await locate(page, address).first().waitFor(); +} + +async function requireCount(locator: Locator, expected: number, label: string): Promise { const actual = await locator.count(); assert(actual === expected, `${label}: expected ${expected}, received ${actual}`); } diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/calibration-seed.json b/src/dev/execution-comparison/petrinaut-optimization-oracle/calibration-seed.json new file mode 100644 index 000000000..a549582a5 --- /dev/null +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/calibration-seed.json @@ -0,0 +1,165 @@ +{ + "id": "sir-calibration", + "title": "SIR Epidemic Model", + "sdcpn": { + "places": [ + { + "id": "place__susceptible", + "name": "Susceptible", + "colorId": null, + "dynamicsEnabled": false, + "differentialEquationId": null, + "showAsInitialState": true, + "x": -435, + "y": 150 + }, + { + "id": "place__infected", + "name": "Infected", + "colorId": null, + "dynamicsEnabled": false, + "differentialEquationId": null, + "showAsInitialState": true, + "x": -195, + "y": 285 + }, + { + "id": "place__recovered", + "name": "Recovered", + "colorId": null, + "dynamicsEnabled": false, + "differentialEquationId": null, + "x": 375, + "y": 195 + } + ], + "transitions": [ + { + "id": "transition__infection", + "name": "Infection", + "inputArcs": [ + { "placeId": "place__susceptible", "weight": 1, "type": "standard" }, + { "placeId": "place__infected", "weight": 1, "type": "standard" } + ], + "outputArcs": [{ "placeId": "place__infected", "weight": 2 }], + "lambdaType": "stochastic", + "lambdaCode": "// Mass-action infection: fires at the configured infection rate whenever a\n// Susceptible and an Infected are both present (the two standard input arcs).\nexport default Lambda((tokens, parameters) => parameters.infection_rate)", + "transitionKernelCode": "// Consumes 1 Susceptible + 1 Infected and produces 2 Infected (the output\n// arc has weight 2), encoding the S + I -> 2I reaction: the susceptible has\n// become newly infected. Places are untyped, so tokens carry no attributes.\nexport default TransitionKernel(() => {\n return {\n Infected: [{}, {}],\n };\n});", + "x": -150, + "y": 75 + }, + { + "id": "transition__recovery", + "name": "Recovery", + "inputArcs": [{ "placeId": "place__infected", "weight": 1, "type": "standard" }], + "outputArcs": [{ "placeId": "place__recovered", "weight": 1 }], + "lambdaType": "stochastic", + "lambdaCode": "// Each Infected recovers at the configured recovery rate. The ratio of\n// infection_rate to recovery_rate sets the basic reproduction number R0.\nexport default Lambda((tokens, parameters) => parameters.recovery_rate)", + "transitionKernelCode": "// Move one Infected to Recovered (1-to-1). Recovered individuals are immune,\n// so they never re-enter the Susceptible or Infected places.\nexport default TransitionKernel(() => {\n return {\n Recovered: [{}],\n };\n});", + "x": 90, + "y": 240 + } + ], + "types": [], + "differentialEquations": [], + "parameters": [ + { + "id": "param__infection_rate", + "name": "Infection Rate", + "variableName": "infection_rate", + "type": "real", + "defaultValue": "3" + }, + { + "id": "param__recovery_rate", + "name": "Recovery Rate", + "variableName": "recovery_rate", + "type": "real", + "defaultValue": "1" + } + ], + "scenarios": [ + { + "id": "scenario__seasonal_flu", + "name": "Seasonal Flu", + "description": "Moderate outbreak with R₀ ≈ 1.5. Models a typical seasonal influenza wave in a small community.", + "scenarioParameters": [ + { "type": "integer", "identifier": "population", "default": 1000 }, + { "type": "ratio", "identifier": "infected_ratio", "default": 0.01 } + ], + "parameterOverrides": { "param__infection_rate": "1.5", "param__recovery_rate": "0.8" }, + "initialState": { + "type": "per_place", + "content": { + "place__susceptible": "scenario.population * (1 - scenario.infected_ratio)", + "place__infected": "scenario.population * scenario.infected_ratio", + "place__recovered": "0" + } + } + }, + { + "id": "scenario__high_virulence", + "name": "High Virulence Outbreak", + "description": "Aggressive pathogen with R₀ ≈ 6 and slow recovery, modelling rapid spread before interventions.", + "scenarioParameters": [ + { "type": "integer", "identifier": "population", "default": 10000 }, + { "type": "ratio", "identifier": "infected_ratio", "default": 0.0001 } + ], + "parameterOverrides": { "param__infection_rate": "6", "param__recovery_rate": "0.5" }, + "initialState": { + "type": "per_place", + "content": { + "place__susceptible": "scenario.population * (1 - scenario.infected_ratio)", + "place__infected": "scenario.population * scenario.infected_ratio", + "place__recovered": "0" + } + } + }, + { + "id": "scenario__contained_outbreak", + "name": "Contained Outbreak", + "description": "Sub-threshold spread with R₀ < 1 (recovery outpaces infection), so the outbreak fizzles out instead of taking off.", + "scenarioParameters": [ + { "type": "integer", "identifier": "population", "default": 1000 }, + { "type": "ratio", "identifier": "infected_ratio", "default": 0.05 } + ], + "parameterOverrides": { "param__infection_rate": "0.6", "param__recovery_rate": "1.2" }, + "initialState": { + "type": "per_place", + "content": { + "place__susceptible": "scenario.population * (1 - scenario.infected_ratio)", + "place__infected": "scenario.population * scenario.infected_ratio", + "place__recovered": "0" + } + } + }, + { + "id": "scenario__pandemic_wave", + "name": "Pandemic Wave", + "description": "A large, mostly-susceptible population seeded with a few cases and R₀ ≈ 5 — useful for watching the classic epidemic curve build and burn out at scale.", + "scenarioParameters": [ + { "type": "integer", "identifier": "population", "default": 100000 }, + { "type": "ratio", "identifier": "infected_ratio", "default": 0.00005 } + ], + "parameterOverrides": { "param__infection_rate": "2.5", "param__recovery_rate": "0.5" }, + "initialState": { + "type": "per_place", + "content": { + "place__susceptible": "scenario.population * (1 - scenario.infected_ratio)", + "place__infected": "scenario.population * scenario.infected_ratio", + "place__recovered": "0" + } + } + } + ], + "metrics": [ + { + "id": "metric__infected_fraction", + "name": "Infected Fraction", + "description": "Share of the population currently infected.", + "code": "const s = state.places.Susceptible.count;\nconst i = state.places.Infected.count;\nconst r = state.places.Recovered.count;\nconst total = s + i + r;\nreturn total === 0 ? 0 : i / total;" + } + ] + }, + "lastUpdated": "2026-07-22T16:29:57.811Z" +} diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/claims.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/claims.ts index fdb204c68..2d3067521 100644 --- a/src/dev/execution-comparison/petrinaut-optimization-oracle/claims.ts +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/claims.ts @@ -3,7 +3,6 @@ export type PetrinautFocusedObservation = readonly check: 'route-and-accessibility'; readonly pathname: string; readonly expectedPathname: string; - readonly controlsReachable: boolean; } | { readonly check: 'progress-and-completion'; @@ -30,10 +29,7 @@ export function assessPetrinautFocusedObservation( ): readonly string[] { switch (observation.check) { case 'route-and-accessibility': - return [ - ...(observation.pathname === observation.expectedPathname ? [] : ['focused route missing']), - ...(observation.controlsReachable ? [] : ['required controls are not keyboard reachable']), - ]; + return observation.pathname === observation.expectedPathname ? [] : ['focused route missing']; case 'progress-and-completion': return [ ...(observation.progressiveTrialCount > 0 ? [] : ['progressive trials missing']), diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts index 301e3a522..6f6fba9f8 100644 --- a/src/dev/execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/fake-optimizer.ts @@ -5,6 +5,15 @@ export interface CapturedOptimizationRequest { aborted: boolean; } +/** + * Deterministic Petrinaut Opt stand-in that speaks the upstream Optuna SSE + * contract decoded by `@local/petrinaut-optimizer-client`. + * + * Request `name` steers the branch: + * - includes `failure` → terminal `event: error` + * - includes `cancel` → hold the stream open (no terminal event) + * - otherwise → two COMPLETE trials + `event: done` + */ export async function startDeterministicFakeOptimizer(): Promise<{ readonly origin: string; readonly requests: CapturedOptimizationRequest[]; @@ -35,46 +44,28 @@ export async function startDeterministicFakeOptimizer(): Promise<{ }; request.on('aborted', markAborted); response.on('close', markAborted); - response.writeHead(200, { 'content-type': 'application/x-ndjson' }); + response.writeHead(200, { 'content-type': 'text/event-stream' }); const name = record(body) && typeof body['name'] === 'string' ? body['name'] : ''; - writeEvent(response, { type: 'started', requestedTrials: 2 }); if (name.includes('cancel')) return; if (name.includes('failure')) { - writeEvent(response, { - type: 'error', - code: 'deterministic_failure', - message: 'Deterministic optimizer failure', - retryable: false, - }); + writeEvent(response, { message: 'Deterministic optimizer failure' }, 'error'); response.end(); return; } - const best = { trial: 0, parameters: { rate: 4 }, objective: 12 }; writeEvent(response, { - type: 'trial', - trial: 0, - parameters: { rate: 4 }, - objective: 12, - state: 'complete', - best, + step: 0, + params: { rate: 4 }, + metric: 12, + state: 'COMPLETE', }); await new Promise((resolve) => setTimeout(resolve, 20)); writeEvent(response, { - type: 'trial', - trial: 1, - parameters: { rate: 6 }, - objective: 10, - state: 'complete', - best, - }); - writeEvent(response, { - type: 'complete', - requestedTrials: 2, - completedTrials: 2, - prunedTrials: 0, - failedTrials: 0, - best, + step: 1, + params: { rate: 6 }, + metric: 10, + state: 'COMPLETE', }); + writeEvent(response, {}, 'done'); response.end(); }); await new Promise((resolve, reject) => { @@ -90,7 +81,7 @@ export async function startDeterministicFakeOptimizer(): Promise<{ origin: `http://127.0.0.1:${address.port}`, requests, close: async () => { - for (const response of openResponses) response.destroy(); + for (const open of openResponses) open.destroy(); await new Promise((resolveClose, reject) => { server.close((error) => (error === undefined ? resolveClose() : reject(error))); }); @@ -98,8 +89,8 @@ export async function startDeterministicFakeOptimizer(): Promise<{ }; } -function writeEvent(response: ServerResponse, event: unknown): void { - response.write(`${JSON.stringify(event)}\n`); +function writeEvent(response: ServerResponse, event: unknown, name?: 'done' | 'error'): void { + response.write(`${name === undefined ? '' : `event: ${name}\n`}data: ${JSON.stringify(event)}\n\n`); } function record(value: unknown): value is Record { diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/fixture.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/fixture.ts index 30052f764..0cdf6aaa3 100644 --- a/src/dev/execution-comparison/petrinaut-optimization-oracle/fixture.ts +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/fixture.ts @@ -5,10 +5,17 @@ const PACKAGE_SCRIPTS = [ ['packages/ds-components', '@hashintel/ds-components', ['codegen', 'build']], ['packages/petrinaut-core', '@hashintel/petrinaut-core', ['build']], ['packages/optimizer-client', '@local/petrinaut-optimizer-client', ['build']], + ['packages/refractive', '@hashintel/refractive', ['build']], ['packages/petrinaut', '@hashintel/petrinaut', ['build']], ] as const; -export async function createKnownGoodPetrinautCandidate(root: string): Promise { +export async function createKnownGoodPetrinautCandidate( + root: string, + options: { + readonly backgroundRequestDurationMs?: number; + readonly inventedLabelsRival?: boolean; + } = {}, +): Promise { await mkdir(join(root, 'scripts'), { recursive: true }); await writeFile( join(root, 'package.json'), @@ -22,7 +29,13 @@ export async function createKnownGoodPetrinautCandidate(root: string): Promise { + await createKnownGoodPetrinautCandidate(root, { inventedLabelsRival: true }); +} + function preparationMarkerSource(): string { return String.raw` import { readFile, writeFile } from "node:fs/promises"; @@ -61,6 +79,7 @@ const expected = [ "@hashintel/ds-components:build", "@hashintel/petrinaut-core:build", "@local/petrinaut-optimizer-client:build", + "@hashintel/refractive:build", "@hashintel/petrinaut:build", ]; let completed = []; @@ -76,7 +95,42 @@ await writeFile(path, JSON.stringify(completed)); `; } -function candidateServerSource(): string { +function candidateServerSource(options: { + readonly backgroundRequestDurationMs: number; + readonly inventedLabelsRival: boolean; +}): string { + const invented = options.inventedLabelsRival; + const viewTitle = invented ? '

Optimizations

' : 'Optimizations'; + const nav = invented + ? '
' + : `
+ + +
+
+ + + +
`; + const createLabel = invented ? 'Create optimization' : 'Create'; + const runLabel = invented ? 'Run optimization' : 'Run'; + const cancelLabel = invented ? 'Cancel optimization' : 'Cancel'; + const scenarioLabel = invented ? 'Scenario' : ''; + const scenarioPlaceholder = invented ? 'Select scenario' : 'Select a scenario'; + const metricLabel = invented ? 'Objective metric' : ''; + const metricPlaceholder = invented ? '' : 'Select a metric'; + const directionBlock = invented + ? `` + : `
+ + +
`; + const statusBlock = invented + ? '
Idle
' + : '
Idle
Best
'; + return String.raw` import { createServer, request as httpRequest } from "node:http"; @@ -88,107 +142,168 @@ const host = args.get("--host") ?? "127.0.0.1"; const port = Number(args.get("--port")); const optimizerOrigin = new URL(process.env.PETRINAUT_OPT_ORIGIN); const optimizerProvider = process.env.VITE_PETRINAUT_OPT_PROVIDER; -const requiredPreparation = 5; +const requiredPreparation = 6; +const backgroundRequestDurationMs = ${options.backgroundRequestDurationMs}; const html = String.raw${'`'}
-
-

Optimizations

- - -
Idle
-
+ + + + ${statusBlock}
${'`'}; const server = createServer(async (request, response) => { + if (request.method === "GET" && request.url === "/background-readiness-rival") { + setTimeout(() => { + response.writeHead(204).end(); + }, backgroundRequestDurationMs); + return; + } if (request.method === "GET" && request.url === "/favicon.ico") { response.writeHead(204).end(); return; @@ -234,11 +364,21 @@ const server = createServer(async (request, response) => { if (request.method === "POST" && request.url === "/api/petrinaut-opt/optimize/all") { const upstream = httpRequest(new URL("/optimize/all", optimizerOrigin), { method: "POST", - headers: { "content-type": "application/json" }, + headers: { "content-type": "application/json", accept: "text/event-stream" }, }, (upstreamResponse) => { - response.writeHead(upstreamResponse.statusCode ?? 500, { "content-type": "application/x-ndjson" }); + response.writeHead(upstreamResponse.statusCode ?? 500, { + "content-type": upstreamResponse.headers["content-type"] ?? "text/event-stream", + }); + upstreamResponse.on("error", () => { + if (!response.writableEnded) response.end(); + }); upstreamResponse.pipe(response); }); + upstream.on("error", () => { + if (!response.headersSent) response.writeHead(502); + if (!response.writableEnded) response.end(); + }); + request.on("error", () => upstream.destroy()); request.pipe(upstream); request.on("aborted", () => upstream.destroy()); response.on("close", () => { diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/runner.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/runner.ts index af0394b97..701e68d17 100644 --- a/src/dev/execution-comparison/petrinaut-optimization-oracle/runner.ts +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/runner.ts @@ -1,4 +1,4 @@ -import { runCommand } from '../../../app/command-runner.js'; +import { runCommand, type CommandResult } from '../../../app/command-runner.js'; import { isPetrinautOptimizationExecutionCaseContract, loadPublicCasePacket } from '../case-contract.js'; import { isPetrinautOptimizationControllerOracleManifest, @@ -28,6 +28,11 @@ export const PETRINAUT_FOCUSED_PREPARATION = [ command: 'yarn', args: ['workspace', '@local/petrinaut-optimizer-client', 'build'], }, + { + id: 'refractive-build', + command: 'yarn', + args: ['workspace', '@hashintel/refractive', 'build'], + }, { id: 'petrinaut-ui-build', command: 'yarn', @@ -35,9 +40,15 @@ export const PETRINAUT_FOCUSED_PREPARATION = [ }, ] as const; +export interface PetrinautOraclePreparationObservation { + readonly id: (typeof PETRINAUT_FOCUSED_PREPARATION)[number]['id']; + readonly commandResult: CommandResult; +} + export async function runPetrinautOptimizationOracle(input: { readonly candidateRoot: string; readonly caseDir: string; + readonly onPreparationResult?: (observation: PetrinautOraclePreparationObservation) => Promise | void; }): Promise { const [packet, manifest] = await Promise.all([ loadPublicCasePacket(input.caseDir), @@ -57,6 +68,10 @@ export async function runPetrinautOptimizationOracle(input: { timeoutMs: 10 * 60_000, maxOutputBytes: 256 * 1024, }); + await input.onPreparationResult?.({ + id: step.id, + commandResult: result, + }); preparation.push({ id: step.id, status: result.exitCode === 0 ? 'passed' : 'failed', diff --git a/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json b/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json index 1847d94ca..c4bd00245 100644 --- a/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json +++ b/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json @@ -80,10 +80,10 @@ "id": "AC1", "publicConcern": "Accessible public workflow", "origin": "public_baseline", - "publicWording": "Expose stable named controls and result regions on /optimization.", + "publicWording": "Expose the Optimizations view, Create action, and Create an optimization drawer controls on /optimization.", "controller": { - "wording": "Tab, form, start/cancel, and result controls have stable roles/names and accept keyboard focus.", - "expectedState": "Every required interactive control is rendered once and is keyboard reachable." + "wording": "Closed mechanical addresses for Simulate navigation, Create, scenario/metric selectors and metric editor, Maximize/Minimize, Run/Cancel, and Complete/Error/Cancelled remain exact.", + "expectedState": "Every required source-backed mechanical address resolves exactly once." } } ] diff --git a/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json b/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json index bd11a20f1..421adc4da 100644 --- a/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json +++ b/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json @@ -12,11 +12,11 @@ }, "requirementRegistry": { "path": "testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json", - "sha256": "sha256:395f28cad01954be1b68be799f659a2292f28dc267c26eb50ebb68e9fce8ed05" + "sha256": "sha256:e2b08e6079b69bd1fbcbd8fb3590a5d0fa487b220f9441bccc648989d174455e" }, "executionContractTemplate": { "path": "testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json", - "sha256": "sha256:1d8e9190749bd4107831aeaf50da35c73fec6196a2f902a8a57eaa64ba47d864" + "sha256": "sha256:5d73dc3c9c76c350a22b9e5d2a06c9a9741a2474523feffe9bb4ba4e5076b321" }, "oracle": { "id": "petrinaut-optimization-oracles-v1", diff --git a/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json b/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json index 387b40788..584ad0fb1 100644 --- a/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json +++ b/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json @@ -3,7 +3,7 @@ "case": { "id": "petrinaut-optimization-v1", "specification": "spec.md", - "specificationSha256": "a949cdeeca3093c51f36d42169d582141280f3561c9770de2b6665e28fac3c15", + "specificationSha256": "3fb0772f0e955f81240b58af9f53736393b1f8dccf55315371fc5755e801dc43", "provider": "anthropic", "model": "claude-opus-4-8", "product": "petrinaut", @@ -30,17 +30,88 @@ "sameOriginApi": "/api/petrinaut-opt/optimize/all", "executionTerminal": "promotion_prepared" }, - "accessibility": { - "view": { "role": "heading", "name": "Optimizations" }, - "tab": { "role": "tab", "name": "Optimizations" }, - "create": { "role": "button", "name": "Create optimization" }, - "scenario": { "role": "combobox", "name": "Scenario" }, - "metric": { "role": "combobox", "name": "Objective metric" }, - "direction": { "role": "combobox", "name": "Objective direction" }, - "run": { "role": "button", "name": "Run optimization" }, - "cancel": { "role": "button", "name": "Cancel optimization" }, - "status": { "role": "status", "name": "Optimization status" }, - "results": { "role": "region", "name": "Optimization results" } + "mechanicalAddresses": { + "skipTour": { + "kind": "roleName", + "role": "button", + "name": "Skip tour" + }, + "dismissAssistant": { + "kind": "roleName", + "role": "button", + "name": "Dismiss" + }, + "simulateMode": { + "kind": "roleName", + "role": "radio", + "name": "Simulate" + }, + "optimizationsNav": { + "kind": "roleValue", + "role": "radio", + "value": "optimizations" + }, + "viewTitle": { + "kind": "exactText", + "text": "Optimizations" + }, + "create": { + "kind": "roleName", + "role": "button", + "name": "Create" + }, + "createDrawer": { + "kind": "roleName", + "role": "dialog", + "name": "Create an optimization" + }, + "scenario": { + "kind": "roleContents", + "role": "combobox", + "contents": "Select a scenario" + }, + "metric": { + "kind": "roleContents", + "role": "combobox", + "contents": "Select a metric" + }, + "metricCode": { + "kind": "roleName", + "role": "textbox", + "name": "Editor content" + }, + "directionMaximize": { + "kind": "roleName", + "role": "radio", + "name": "Maximize" + }, + "directionMinimize": { + "kind": "roleName", + "role": "radio", + "name": "Minimize" + }, + "run": { + "kind": "roleName", + "role": "button", + "name": "Run" + }, + "cancel": { + "kind": "roleName", + "role": "button", + "name": "Cancel" + }, + "statusComplete": { + "kind": "exactText", + "text": "Complete" + }, + "statusError": { + "kind": "exactText", + "text": "Error" + }, + "statusCancelled": { + "kind": "exactText", + "text": "Cancelled" + } }, "rules": [ "Work only in the target repository.", diff --git a/testing/execution-comparisons/cases/petrinaut-optimization/spec.md b/testing/execution-comparisons/cases/petrinaut-optimization/spec.md index 6233b5e59..f9f754761 100644 --- a/testing/execution-comparisons/cases/petrinaut-optimization/spec.md +++ b/testing/execution-comparisons/cases/petrinaut-optimization/spec.md @@ -45,6 +45,7 @@ Browser traffic and rendered content must not expose or contact the private upst ### AC1 Accessible public workflow -The `/optimization` route exposes a named Optimizations view. Its tab, scenario selector, parameter and -metric form controls, start/cancel controls, and result regions use stable accessible roles and names and -are keyboard reachable. +The `/optimization` route exposes an Optimizations view title and Create action. Opening Create reveals a +Create an optimization drawer whose scenario selector, metric selector, Maximize/Minimize direction +controls, Run/Cancel actions, and Complete/Error/Cancelled status text use the closed mechanical +addresses declared by the public contract. From 3327b6afd9c51fbbd9a2c4991e4ad8179a0c738e Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 21:05:04 +0200 Subject: [PATCH 05/15] FE-1241: Resolve plan restack markers Preserve the landed FE-1250 and FE-1251 frontier state alongside the latest brownfield comparison status. Co-authored-by: Cursor --- memory/PLAN.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/memory/PLAN.md b/memory/PLAN.md index 2b8fb9f02..fccf7b419 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -117,13 +117,9 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand - `execution-comparison-tracer` ([FE-1230](https://linear.app/hash/issue/FE-1230/greenfield-execution-comparison-tracer)) — **active on `ka/fe-1230-independent-oracle-journeys`:** the frozen case, immutable-attempt contract, retained Brunch/Claude v1 pair, and versioned independent browser-oracle journeys are built. Next: restore the exact retained output paths, replay `petri-editor-browser-v2` unchanged against both, then review and promote the bounded evidence; no mutants, judging campaign, or repetitions in this frontier. - `end-to-end-comparison-tracer` ([FE-1239](https://linear.app/hash/issue/FE-1239/trace-elicitation-through-execution)) — **complete on `ka/fe-1239-end-to-end-comparison-tracer`, stacked on FE-1230:** two rigorous Petri-editor elicitation outputs crossed exact immutable handoffs into the staged 2×2 Brunch/Claude matrix; all four valid failed cells, unchanged-oracle evidence, and the validity-first requirement ledger are promoted. Definition and retained witness below. - `executor-slice-admission-parity` ([FE-1240](https://linear.app/hash/issue/FE-1240/prevent-invalid-scoped-slices-from-reaching-execution)) — **complete on `ka/fe-1240-slice-admission-parity`, restacked on `next` after FE-1239 landed:** incomplete scoped-slice worker context is rejected during deterministic plan admission, exact findings enter bounded repair before `slice_execute`, and the execution boundary remains fail-closed. Definition below. -<<<<<<< HEAD - `executor-plan-coherence` ([FE-1250](https://linear.app/hash/issue/FE-1250/build-coherent-execution-plans)) — **implementation complete on `ka/fe-1250-coherent-execution-plans`, based on FE-1240 via `next`:** frontier-verified multi-slice epics now require one ordinary terminal member over every sibling, planner conduct establishes shared foundations, and workers preserve cumulative public contracts for the canonical harness. Fast verification passes; next is the separately authorized unchanged Petri comparison rerun. No new browser gate, durable plan kind, or executor lifecycle phase. Definition below. - `comparison-publication-workflow` ([FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports)) — **active on `ka/fe-1251-comparison-publication`, stacked on FE-1250:** capture immutable controller and release provenance before comparison lanes start, then explicitly publish retained validity-first reports into the canonical Notion database through a guarded idempotent skill. Definition below. -- `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases)) — **active on `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240:** A49-L's versioned fail-closed historical-replay admission path and the frozen Brunch backend host-landing profile/controller oracle are complete. Preserve the minimal Petri-net editor as the only greenfield case, then replay Petrinaut optimization UI PR #9051 in the full `hashintel/hash` repository. A50-L is invalidated and D136-L selects the focused `/optimization` route for the common hard gate. Next: build the revised Petrinaut profile/oracle before any historical provider run. -======= - `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases)) — **active on `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240:** both frozen brownfield profiles/oracles and D137-L's admitted, lane-ready two-commit historical replay path are built and contract-hardened. Final admission now follows D136-L's tracked-clean dependency boundary, pinned case recipes are exhaustive with no default, and the production deep operation has a reachable-network rival. Preserve the minimal Petri-net editor as the only greenfield case. Next: scope and run the fresh historical provider preflight. ->>>>>>> 5e1ed22b (FE-1241: Admit pinned historical replay targets) - **Carved from FE-1167 (2026-07-13):** the Execute-mode evidence sub-list — Execute entry beats on thin vs rich seeds (assessment honesty: Ask on thin, Proceed on rich) and the FE-1107/KA residue (close-or-narrow, demo/walkthrough session via `TESTING_PLAN.md`, post-KA plan pass). The former sticky-posture question is no longer KA residue: FE-1187 `remediation-4` owns the persistent Specify elicitation-style audit/SPEC revision, and its Continue lexical audit owns the old `continue` ambiguity. Full context in the archived FE-1167 definition (`docs/archive/PLAN_HISTORY.md`). - `planning-process-model` — **moved to the KA stream 2026-07-13; reshaped by D126-L**: the durable scope handoff is settled, so this item now owns only plan projection and epistemic-horizon questions beyond committed scopes. Definition below. - **[1.x data-model handoff owed by FE-1187](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md):** a concise colleague-facing note must distinguish current canon from directional vocabulary before the consolidated outer checkpoint. Current: `{milestone, frontier, scope}` with executor-derived slices; basis (`explicit | implicit`) orthogonal to settlement (`advisory | settled`); no persisted readiness grade or spec-global elicitation-gap table; active-branch Pi JSONL reads; one CommandExecutor mutation authority with spec-local LSN/change log; no new projected `vv_obligation` (legacy rows remain readable). Directional only: explain stored `thesis` as pitch/concept without renaming it yet, and avoid new coupling to spec-local `term` while its possible workspace lift remains future work. Persisted judgment-shaped reconciliation needs remain current, but are YAGNI-suspect: add no kinds/consumers/orchestration dependency without fresh evidence; re-evaluate derivation/removal when KA work first needs that table. Delivered note: [`Brunch 1.x data-model handoff`](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md); it does not block the deterministic TUI queue. From f1b083a4be1af9b6f3c582acfdb0fb00a8a40c12 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 21:22:15 +0200 Subject: [PATCH 06/15] FE-1241: Install Linux PTY shell dependency Run the expect-backed PTY integration suite on Ubuntu with the zsh executable its fixtures invoke. Co-authored-by: Cursor --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 977889c54..4e0d6a0c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ jobs: - name: Install PTY test dependency run: | sudo apt-get update - sudo apt-get install --yes expect + sudo apt-get install --yes expect zsh - name: Install run: npm ci From 531fa328b88740706f8727736bda776e34cb36eb Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 21:39:29 +0200 Subject: [PATCH 07/15] FE-1241: Tolerate settled host-landing startup Accept both first-run mode selection and an already-settled controller session before driving the public landing command. Co-authored-by: Cursor --- .../host-landing-oracle/runner.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/dev/execution-comparison/host-landing-oracle/runner.ts b/src/dev/execution-comparison/host-landing-oracle/runner.ts index 14f4905cc..cbb059ab7 100644 --- a/src/dev/execution-comparison/host-landing-oracle/runner.ts +++ b/src/dev/execution-comparison/host-landing-oracle/runner.ts @@ -91,9 +91,16 @@ async function driveCandidateTui(input: { sendKeys(name, ['Down', 'Enter']); await wait(name, 'Does this build on the existing code here?'); sendKeys(name, ['Enter']); - await wait(name, 'Choose how Specify mode should work'); - sendKeys(name, ['Esc']); - terminalEvidence.push(...(await wait(name, 'Settled controller session.'))); + const settledOrModeChoice = await wait( + name, + /Choose how Specify mode should work|Settled controller session\./u, + ); + if (settledOrModeChoice.join('\n').includes('Choose how Specify mode should work')) { + sendKeys(name, ['Esc']); + terminalEvidence.push(...(await wait(name, 'Settled controller session.'))); + } else { + terminalEvidence.push(...settledOrModeChoice); + } const commandText = input.scenario === 'greenfield_success' ? `/brunch:land ${HOST_LANDING_RUN_ID} ${input.fixture.targetDir}` @@ -153,7 +160,7 @@ async function driveCandidateTui(input: { } } -async function wait(name: string, text: string): Promise { +async function wait(name: string, text: string | RegExp): Promise { const status = sessionStatus(name); if (!status) throw new Error(`TUI driver session ${name} disappeared`); const result = await waitForScreenText(status.logPath, status.cols, status.rows, text, { From bf28c7d9e2ce95c97a69aca8ab9619e29edd989e Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 21:59:35 +0200 Subject: [PATCH 08/15] FE-1241: Harden comparison path and selector checks Canonicalize bounded reads across filesystem aliases and replace runtime regex construction with fixed-text option matching while retaining redaction coverage. Co-authored-by: Cursor --- .../__tests__/agent-runtime-runtime.test.ts | 13 +++++- .../extensions/agent-runtime/runtime/index.ts | 8 +--- .../petrinaut-historical-preflight.test.ts | 8 ++-- .../petrinaut-optimization-oracle/browser.ts | 46 +++++++++---------- 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts b/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts index 2b35a1097..b20e1cd2e 100644 --- a/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts +++ b/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -207,8 +207,10 @@ describe('Brunch agent runtime-state projection', () => { const workspace = join(root, 'workspace'); const packageRoot = join(root, 'package'); const skillPath = join(packageRoot, 'SKILL.md'); + const workspaceFile = join(workspace, 'inside.md'); await mkdir(workspace); await mkdir(packageRoot); + await writeFile(workspaceFile, '# Inside\n'); await writeFile(skillPath, '# Skill\n'); type RegisteredRead = { @@ -243,6 +245,15 @@ describe('Brunch agent runtime-state projection', () => { cwd: workspace, }), ).resolves.toBeDefined(); + await expect( + registerRead(await realpath(workspace)).execute( + 'aliased-in-root-read', + { path: workspaceFile }, + undefined, + undefined, + { cwd: workspace }, + ), + ).resolves.toBeDefined(); await expect( registerRead(workspace).execute('bounded-read', { path: skillPath }, undefined, undefined, { cwd: workspace, diff --git a/src/.pi/extensions/agent-runtime/runtime/index.ts b/src/.pi/extensions/agent-runtime/runtime/index.ts index 21ae16bad..09b887b96 100644 --- a/src/.pi/extensions/agent-runtime/runtime/index.ts +++ b/src/.pi/extensions/agent-runtime/runtime/index.ts @@ -318,12 +318,8 @@ export function registerBrunchOperationalModePolicy( } async function assertBoundedReadPath(root: string, requestedPath: string): Promise { - const normalizedRoot = resolve(root); - const target = resolve(normalizedRoot, requestedPath); - if (target !== normalizedRoot && !target.startsWith(`${normalizedRoot}${sep}`)) { - throw new Error(`read-only tool path escapes target root: ${requestedPath}`); - } - const [realRoot, realTarget] = await Promise.all([realpath(normalizedRoot), realpath(target)]); + const target = resolve(root, requestedPath); + const [realRoot, realTarget] = await Promise.all([realpath(root), realpath(target)]); if (realTarget !== realRoot && !realTarget.startsWith(`${realRoot}${sep}`)) { throw new Error(`read-only tool path escapes target root through symlink: ${requestedPath}`); } diff --git a/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts index 0688bee74..95c4b2c99 100644 --- a/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts +++ b/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts @@ -1,4 +1,4 @@ -import { createHash } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -228,7 +228,7 @@ describe('Petrinaut historical provider preflight', () => { const fixture = await createFixture(); let installs = 0; let oracleCalled = false; - const secret = 'token=reference-secret-value'; + const sensitiveMarker = `token=${randomUUID()}`; const result = await runFixturePreflight(fixture, { dependencyInstallRunner: async () => { @@ -238,7 +238,7 @@ describe('Petrinaut historical provider preflight', () => { : { exitCode: 7, stdout: 'x'.repeat(200_000), - stderr: `reference install failed in ${fixture.referenceTargetDir}; ${secret}\n`, + stderr: `reference install failed in ${fixture.referenceTargetDir}; ${sensitiveMarker}\n`, }; }, runOracle: async () => { @@ -283,7 +283,7 @@ describe('Petrinaut historical provider preflight', () => { expect(evidenceBytes).toContain('"exitCode": 7'); expect(evidenceBytes).toContain('reference install failed'); expect(evidenceBytes).not.toContain(fixture.referenceTargetDir); - expect(evidenceBytes).not.toContain('reference-secret-value'); + expect(evidenceBytes).not.toContain(sensitiveMarker); expect(JSON.stringify(result.receipt)).not.toContain(fixture.referenceTargetDir); }, 30_000); diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts index bec3a4587..074db59f3 100644 --- a/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts @@ -130,7 +130,7 @@ function checkDefinitions(input: { await openCreateDrawer(page, addresses); await requireCount(locate(page, addresses.createDrawer), 1, 'createDrawer'); await requireCount(locate(page, addresses.scenario), 1, 'scenario'); - await selectScenarioOption(page, addresses, /Seasonal Flu/u); + await selectScenarioOption(page, addresses, 'Seasonal Flu'); await requireCount(locate(page, addresses.metric), 1, 'metric'); await requireCount(locate(page, addresses.directionMaximize), 1, 'directionMaximize'); await requireCount(locate(page, addresses.run), 1, 'run'); @@ -153,7 +153,7 @@ function checkDefinitions(input: { await optimize.click({ force: true }); assert(await optimize.isChecked(), 'optimize toggle did not enable'); await locate(page, addresses.directionMinimize).click({ force: true }); - await selectScenarioOption(page, addresses, /High Virulence Outbreak/u); + await selectScenarioOption(page, addresses, 'High Virulence Outbreak'); assert( !(await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).isChecked()), 'scenario change retained optimized binding', @@ -176,7 +176,7 @@ function checkDefinitions(input: { await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ force: true, }); - await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); await locate(page, addresses.directionMaximize).click({ force: true }); await setOptimizationName(page, 'saved metric proof'); await locate(page, addresses.run).click(); @@ -193,12 +193,12 @@ function checkDefinitions(input: { await dismissOverlayDrawers(page); await openCreateDrawer(page, addresses); - await selectScenarioOption(page, addresses, /Seasonal Flu/u); + await selectScenarioOption(page, addresses, 'Seasonal Flu'); const customRequestIndex = input.requests.length; await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ force: true, }); - await selectComboboxOption(page, addresses.metric, /Custom code/u); + await selectComboboxOption(page, addresses.metric, 'Custom code'); await locate(page, addresses.metricCode).fill('return 42;'); await locate(page, addresses.directionMinimize).click({ force: true }); await setOptimizationName(page, 'custom metric proof'); @@ -246,7 +246,7 @@ function checkDefinitions(input: { await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ force: true, }); - await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); await locate(page, addresses.directionMaximize).click({ force: true }); await setOptimizationName(page, 'progress proof'); await locate(page, addresses.run).click(); @@ -270,7 +270,7 @@ function checkDefinitions(input: { await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ force: true, }); - await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); await locate(page, addresses.directionMaximize).click({ force: true }); await setOptimizationName(page, 'service failure'); await locate(page, addresses.run).click(); @@ -286,7 +286,7 @@ function checkDefinitions(input: { await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ force: true, }); - await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); await locate(page, addresses.directionMaximize).click({ force: true }); await setOptimizationName(page, 'cancel proof'); await locate(page, addresses.run).click(); @@ -331,7 +331,7 @@ function checkDefinitions(input: { await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ force: true, }); - await selectComboboxOption(page, addresses.metric, /Infected Fraction/u); + await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); await locate(page, addresses.directionMaximize).click({ force: true }); await setOptimizationName(page, 'secrecy proof'); await locate(page, addresses.run).click(); @@ -372,7 +372,7 @@ async function openConfiguration( (await page.getByRole('checkbox', { name: /^Optimize /u }).count()) === 0, 'configuration appeared before scenario selection', ); - await selectScenarioOption(page, addresses, /Seasonal Flu/u); + await selectScenarioOption(page, addresses, 'Seasonal Flu'); } async function openCreateDrawer( @@ -425,7 +425,7 @@ function locate(page: Page, address: PetrinautMechanicalAddress): Locator { async function selectScenarioOption( page: Page, addresses: PetrinautOptimizationExecutionCasePublicContract['mechanicalAddresses'], - optionPattern: RegExp, + optionText: string, ): Promise { const drawer = locate(page, addresses.createDrawer); await drawer.waitFor(); @@ -435,19 +435,18 @@ async function selectScenarioOption( const combobox = (await empty.count()) > 0 ? empty : drawer.getByRole('combobox').first(); const tagName = await combobox.evaluate((element) => element.tagName); if (tagName === 'SELECT') { - const value = await combobox.evaluate((element, source) => { - const pattern = new RegExp(source, 'u'); + const value = await combobox.evaluate((element, expectedText) => { for (const option of Array.from((element as HTMLSelectElement).options)) { - if (pattern.test(option.textContent ?? '')) return option.value; + if ((option.textContent ?? '').includes(expectedText)) return option.value; } return null; - }, optionPattern.source); - assert(value !== null && value.length > 0, `no select option matched ${optionPattern}`); + }, optionText); + assert(value !== null && value.length > 0, `no select option matched ${optionText}`); await combobox.selectOption(value); return; } await combobox.click({ force: true }); - await page.getByRole('option').filter({ hasText: optionPattern }).first().click(); + await page.getByRole('option').filter({ hasText: optionText }).first().click(); } function cssRoleSelector(role: string): string { @@ -462,24 +461,23 @@ function cssEscape(value: string): string { async function selectComboboxOption( page: Page, address: PetrinautMechanicalAddress, - optionPattern: RegExp, + optionText: string, ): Promise { const combobox = locate(page, address); const tagName = await combobox.evaluate((element) => element.tagName); if (tagName === 'SELECT') { - const value = await combobox.evaluate((element, source) => { - const pattern = new RegExp(source, 'u'); + const value = await combobox.evaluate((element, expectedText) => { for (const option of Array.from((element as HTMLSelectElement).options)) { - if (pattern.test(option.textContent ?? '')) return option.value; + if ((option.textContent ?? '').includes(expectedText)) return option.value; } return null; - }, optionPattern.source); - assert(value !== null && value.length > 0, `no select option matched ${optionPattern}`); + }, optionText); + assert(value !== null && value.length > 0, `no select option matched ${optionText}`); await combobox.selectOption(value); return; } await combobox.click({ force: true }); - await page.getByRole('option').filter({ hasText: optionPattern }).first().click(); + await page.getByRole('option').filter({ hasText: optionText }).first().click(); } async function setOptimizationName(page: Page, name: string): Promise { From 3a2914655b96628bc4793595bdd2de961953e39c Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 22:10:42 +0200 Subject: [PATCH 09/15] FE-1241: Close bounded-read existence oracle Deny lexical escapes uniformly while permitting canonical in-root aliases and retaining symlink containment checks. Co-authored-by: Cursor --- .../__tests__/agent-runtime-runtime.test.ts | 12 ++++++++++- .../extensions/agent-runtime/runtime/index.ts | 20 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts b/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts index b20e1cd2e..c6677c7a2 100644 --- a/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts +++ b/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts @@ -208,6 +208,7 @@ describe('Brunch agent runtime-state projection', () => { const packageRoot = join(root, 'package'); const skillPath = join(packageRoot, 'SKILL.md'); const workspaceFile = join(workspace, 'inside.md'); + const missingOutsidePath = join(root, 'missing-outside.md'); await mkdir(workspace); await mkdir(packageRoot); await writeFile(workspaceFile, '# Inside\n'); @@ -258,7 +259,16 @@ describe('Brunch agent runtime-state projection', () => { registerRead(workspace).execute('bounded-read', { path: skillPath }, undefined, undefined, { cwd: workspace, }), - ).rejects.toThrow('read-only tool path escapes target root'); + ).rejects.toThrow(`read-only tool path escapes target root: ${skillPath}`); + await expect( + registerRead(workspace).execute( + 'missing-outside-read', + { path: missingOutsidePath }, + undefined, + undefined, + { cwd: workspace }, + ), + ).rejects.toThrow(`read-only tool path escapes target root: ${missingOutsidePath}`); } finally { await rm(root, { recursive: true, force: true }); } diff --git a/src/.pi/extensions/agent-runtime/runtime/index.ts b/src/.pi/extensions/agent-runtime/runtime/index.ts index 09b887b96..06a657bd7 100644 --- a/src/.pi/extensions/agent-runtime/runtime/index.ts +++ b/src/.pi/extensions/agent-runtime/runtime/index.ts @@ -318,9 +318,23 @@ export function registerBrunchOperationalModePolicy( } async function assertBoundedReadPath(root: string, requestedPath: string): Promise { - const target = resolve(root, requestedPath); - const [realRoot, realTarget] = await Promise.all([realpath(root), realpath(target)]); - if (realTarget !== realRoot && !realTarget.startsWith(`${realRoot}${sep}`)) { + const normalizedRoot = resolve(root); + const target = resolve(normalizedRoot, requestedPath); + if (!isContainedPath(normalizedRoot, target)) { + try { + const [realRoot, realTarget] = await Promise.all([realpath(normalizedRoot), realpath(target)]); + if (isContainedPath(realRoot, realTarget)) return; + } catch { + // Lexical escapes always receive the same denial, whether or not they exist. + } + throw new Error(`read-only tool path escapes target root: ${requestedPath}`); + } + const [realRoot, realTarget] = await Promise.all([realpath(normalizedRoot), realpath(target)]); + if (!isContainedPath(realRoot, realTarget)) { throw new Error(`read-only tool path escapes target root through symlink: ${requestedPath}`); } } + +function isContainedPath(root: string, target: string): boolean { + return target === root || target.startsWith(`${root}${sep}`); +} From ecf998f6e04eb84e164f67e0ded65770acbff6c8 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 22:21:58 +0200 Subject: [PATCH 10/15] FE-1241: Delegate missing in-root reads Validate the nearest existing ancestor so absent in-root paths reach the underlying tool without weakening alias or symlink boundaries. Co-authored-by: Cursor --- .../__tests__/agent-runtime-runtime.test.ts | 10 +++++ .../extensions/agent-runtime/runtime/index.ts | 44 ++++++++++++++----- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts b/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts index c6677c7a2..ee5979bff 100644 --- a/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts +++ b/src/.pi/extensions/__tests__/agent-runtime-runtime.test.ts @@ -208,6 +208,7 @@ describe('Brunch agent runtime-state projection', () => { const packageRoot = join(root, 'package'); const skillPath = join(packageRoot, 'SKILL.md'); const workspaceFile = join(workspace, 'inside.md'); + const missingInsidePath = join(workspace, 'missing-inside.md'); const missingOutsidePath = join(root, 'missing-outside.md'); await mkdir(workspace); await mkdir(packageRoot); @@ -255,6 +256,15 @@ describe('Brunch agent runtime-state projection', () => { { cwd: workspace }, ), ).resolves.toBeDefined(); + await expect( + registerRead(workspace).execute( + 'missing-inside-read', + { path: missingInsidePath }, + undefined, + undefined, + { cwd: workspace }, + ), + ).rejects.toMatchObject({ code: 'ENOENT', syscall: 'access' }); await expect( registerRead(workspace).execute('bounded-read', { path: skillPath }, undefined, undefined, { cwd: workspace, diff --git a/src/.pi/extensions/agent-runtime/runtime/index.ts b/src/.pi/extensions/agent-runtime/runtime/index.ts index 06a657bd7..3fffd4e38 100644 --- a/src/.pi/extensions/agent-runtime/runtime/index.ts +++ b/src/.pi/extensions/agent-runtime/runtime/index.ts @@ -8,7 +8,7 @@ import { realpath } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { resolve, sep } from 'node:path'; +import { dirname, resolve, sep } from 'node:path'; import type { ExtensionAPI } from '@earendil-works/pi-coding-agent'; import { @@ -320,21 +320,45 @@ export function registerBrunchOperationalModePolicy( async function assertBoundedReadPath(root: string, requestedPath: string): Promise { const normalizedRoot = resolve(root); const target = resolve(normalizedRoot, requestedPath); - if (!isContainedPath(normalizedRoot, target)) { - try { - const [realRoot, realTarget] = await Promise.all([realpath(normalizedRoot), realpath(target)]); - if (isContainedPath(realRoot, realTarget)) return; - } catch { - // Lexical escapes always receive the same denial, whether or not they exist. + const lexicallyContained = isContainedPath(normalizedRoot, target); + let realRoot: string; + let realTargetBoundary: string; + try { + [realRoot, realTargetBoundary] = await Promise.all([ + realpath(normalizedRoot), + realpathNearestExistingAncestor(target), + ]); + } catch (error) { + if (!lexicallyContained) { + throw new Error(`read-only tool path escapes target root: ${requestedPath}`); } + throw error; + } + if (isContainedPath(realRoot, realTargetBoundary)) return; + if (!lexicallyContained) { throw new Error(`read-only tool path escapes target root: ${requestedPath}`); } - const [realRoot, realTarget] = await Promise.all([realpath(normalizedRoot), realpath(target)]); - if (!isContainedPath(realRoot, realTarget)) { - throw new Error(`read-only tool path escapes target root through symlink: ${requestedPath}`); + throw new Error(`read-only tool path escapes target root through symlink: ${requestedPath}`); +} + +async function realpathNearestExistingAncestor(target: string): Promise { + let candidate = target; + for (;;) { + try { + return await realpath(candidate); + } catch (error) { + if (!isErrorWithCode(error, 'ENOENT')) throw error; + const parent = dirname(candidate); + if (parent === candidate) throw error; + candidate = parent; + } } } +function isErrorWithCode(error: unknown, code: string): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error && error.code === code; +} + function isContainedPath(root: string, target: string): boolean { return target === root || target.startsWith(`${root}${sep}`); } From bb3c7b05306c438cbb10d6db296c7d53af593305 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 23:05:58 +0200 Subject: [PATCH 11/15] FE-1241: Reconcile brownfield comparison state Retire consumed scope cards and preserve the real-reference preflight as an explicit future gate rather than completed evidence. Co-authored-by: Cursor --- docs/praxis/comparison-runs.md | 16 +- memory/PLAN.md | 18 +- memory/SPEC.md | 10 +- ...son-cases--admission-contract-hardening.md | 100 -------- ...ses--historical-replay-target-admission.md | 173 -------------- ...ses--petrinaut-optimization-case-oracle.md | 177 -------------- ...son-cases--petrinaut-provider-preflight.md | 224 ------------------ ...son-cases--petrinaut-reference-fidelity.md | 203 ---------------- src/dev/TOPOLOGY.md | 6 +- 9 files changed, 25 insertions(+), 902 deletions(-) delete mode 100644 memory/cards/brownfield-comparison-cases--admission-contract-hardening.md delete mode 100644 memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md delete mode 100644 memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md delete mode 100644 memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md delete mode 100644 memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md diff --git a/docs/praxis/comparison-runs.md b/docs/praxis/comparison-runs.md index 5dcf5f48a..e9d69bba0 100644 --- a/docs/praxis/comparison-runs.md +++ b/docs/praxis/comparison-runs.md @@ -111,15 +111,13 @@ filenames, SHA-256, byte count, and truncation state. The receipt is evidence, n `setup_failed` or `assertion_failed` blocks historical provider work; never bypass the failed phase or replace an invalid receipt. -The 2026-07-22 real-source run is currently blocked on historical browser semantics: both immutable -installs, tracked-source checks, and the six focused builds (including the directly declared -Refractive build immediately before Petrinaut UI) pass. Browser navigation uses bounded -`domcontentloaded` readiness independently from the unchanged 5-second semantic action budget, and -the real run retains no failed request or navigation timeout. The pinned historical shell does not -expose the fixture-authored exact `Optimizations` heading / `Create optimization` control contract; -its optimization view labels the action `Create`. Re-entry must respecify real public semantics before -provider work rather than weakening locators in place. Exact retained evidence is recorded in the -[active scope card](../../memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md). +The 2026-07-22 diagnostic run proved both immutable installs, tracked-source checks, and all six +focused builds, then exposed that fixture-authored labels were stronger than PR #9051's public UI. +The frozen packet, typed addresses, and synthetic rivals were rebaselined to source-backed semantics +before any provider attempt. PR #362 deliberately shipped without rerunning the expensive real +post-rebaseline preflight, so no passing merged-reference receipt exists. Before a future first +Petrinaut provider attempt, rerun the command above and require a passing receipt under D138-L; the +waiver used to close the implementation PR is not reusable as campaign evidence. ## End-to-end comparison tracer diff --git a/memory/PLAN.md b/memory/PLAN.md index fccf7b419..d53e9db25 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -21,7 +21,7 @@ Brunch-next has delivered the original composition spine: the host, sealed Pi pr **Parallel lanes.** Group 3 agent-layer work is pickup-ready. `agent-control-plane-closure` is one earned coverage frontier over foreground and background prompt ingress: it absorbs the former `graph-assurance-conduct` and `subagent-skill-access` items, reconciles the stale eager-context fossils left by D58-L's load-on-demand cutover, and closes prompt replacement/resource-invocation contradictions before capture experiments amplify them. FE-1187 retains only the concern-grouped review renderer. FE-1208's reshaped `automation-observability-dx` closure landed on the restacked base; its invalid warrant pilots remain historical evidence, while the useful evaluator/report primitives are available to `capture-ledger-tracer`. Two evaluation use cases stay deliberately distinct: **seed-based intra-product testing** (Brunch on its own terms — seeds encode Brunch graph/spec state only Brunch understands) and **mission-driven agent-as-user cross-product comparison** (`agent-as-user-comparison` — competitor CLIs share no Brunch state, so the comparable artifact is the "ready" spec/plan document a mission produces). FE-1210 closed the rigorous technical tracer on 2026-07-17; the separately planned operator workflow (`operator-comparison-workflow`, FE-1215) makes saved missions, visible comparison-harness framing, conversational launch, and a readable free-form report approachable without weakening the retained regression/evaluation machinery. Its first operator-led run falsified the nested fresh-actor topology: the top-level project Pi session must act as the simulated user and drive one direct harness subshell at a time, with stock-Pi text interaction as the portable baseline; the corrected real witness is deferred to `saved-mission-comparison-witness`. The KA stream owns executor/orchestrator/Execute-mode work. FE-1192 and FE-1195 completed the attempt, isolated fan-in, durable parallel authority, and epic integration sequence; `executor-plan-coherence` now uses those settled seams to test whether multi-slice plans converge on one working result. Broader instrumentation remains trigger-gated under Later. -**End-to-end comparison lane.** FE-1239 materializes the handoff that the completed reporting procedure names but does not run: two fresh rigorous elicitation lanes produce exact approved specifications, each specification crosses an immutable content-addressed handoff into both Brunch and Claude execution, and the unchanged Petri-editor oracle joins all four outputs into requirement-level evidence. Shared interface requirements are disclosed before elicitation and remain controlled baseline, while private/reveal requirements stay controller-only. The first tracer supports bounded within-case contrasts only — no winner, reliability claim, or cross-case causal conclusion. The planned `brownfield-comparison-cases` frontier keeps that Petri editor as the sole greenfield case and extends the same exact-handoff discipline to one pinned backend feature in Brunch and one pinned frontend feature in Petrinaut; Clay is not part of the campaign. +**End-to-end comparison lane.** FE-1239 materializes the handoff that the completed reporting procedure names but does not run: two fresh rigorous elicitation lanes produce exact approved specifications, each specification crosses an immutable content-addressed handoff into both Brunch and Claude execution, and the unchanged Petri-editor oracle joins all four outputs into requirement-level evidence. Shared interface requirements are disclosed before elicitation and remain controlled baseline, while private/reveal requirements stay controller-only. The first tracer supports bounded within-case contrasts only — no winner, reliability claim, or cross-case causal conclusion. FE-1241 keeps that Petri editor as the sole greenfield case and adds frozen Brunch/Petrinaut brownfield packets, admitted historical-replay targets, and controller-owned oracles; Clay remains outside the campaign. No Petrinaut provider attempt was run. D138-L still requires an explicit passing merged-reference preflight before any future first attempt; that real run was waived for PR #362 and is not claimed as evidence. **Current seams.** Brunch ships on the `1.0.0-alpha.x` line. One-shot `ask` is the only interactive structured-exchange terminal; D125-L's live ask registry provides headless discovery/answering, while the transcript-backed pending projection remains a compatibility surface for live offer tools after the legacy `present_question` pending branch retired. Sweep classification remains fail-closed and compile-time anchored to the exchange-schema terminal names (D117-L), while the larger capture-conditional watermark question remains A40-L. @@ -84,9 +84,9 @@ Close the entire first batch of walkthrough-related findings: remediation, the o ### Recently Completed +- 2026-07-22 `brownfield-comparison-cases` (FE-1241) — **✓ implementation complete:** frozen Brunch and Petrinaut brownfield packets, the two-commit lane-ready replay boundary, compile-time preparation/oracle registries, deterministic sensitivity rivals, and portable CI are built. The real post-rebaseline Petrinaut preflight/provider campaign was explicitly not run; D138-L remains its re-entry gate. - 2026-07-21 `executor-slice-admission-parity` (FE-1240) — **✓ complete:** candidate admission now rejects scoped slices without executable criterion, design, or verification-machinery context; exact findings enter bounded repair, every admitted repaired slice survives preview/worker-context parsing, and the populated-plan execution guard remains fail-closed. - 2026-07-20 `comparison-reporting-skills` (FE-1232) — **✓ implementation complete, building on landed FE-1230:** added separate project-shared Notion publication and comparison-evidence reporting skills; elicitation, execution, end-to-end, and frozen campaign-strategy references; and executable guardrails for active-procedure precedence, safe mutation, validity-first interpretation, reproducible judging, failure retention, and audience-safe controller-only redaction. -- 2026-07-17 `agent-control-plane-closure` (FE-1216) — **✓ closed**: all seven required control-surface rows are built; foreground block replacement, load-on-demand context topology, distinct control ownership, observable resource reads, capability-honest background grants, process-lifetime immutable body/manifest caching, and one private posture renderer now satisfy D135-L/I68-L without schema, fixture, reviewer, or capture-experiment widening. Older completion history (including FE-1192/FE-1195 executor topology closure): [`docs/archive/PLAN_HISTORY.md`](../docs/archive/PLAN_HISTORY.md). ### Next — deferred operator evidence, shared host convergence, then Group 3 agent layer @@ -119,7 +119,7 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand - `executor-slice-admission-parity` ([FE-1240](https://linear.app/hash/issue/FE-1240/prevent-invalid-scoped-slices-from-reaching-execution)) — **complete on `ka/fe-1240-slice-admission-parity`, restacked on `next` after FE-1239 landed:** incomplete scoped-slice worker context is rejected during deterministic plan admission, exact findings enter bounded repair before `slice_execute`, and the execution boundary remains fail-closed. Definition below. - `executor-plan-coherence` ([FE-1250](https://linear.app/hash/issue/FE-1250/build-coherent-execution-plans)) — **implementation complete on `ka/fe-1250-coherent-execution-plans`, based on FE-1240 via `next`:** frontier-verified multi-slice epics now require one ordinary terminal member over every sibling, planner conduct establishes shared foundations, and workers preserve cumulative public contracts for the canonical harness. Fast verification passes; next is the separately authorized unchanged Petri comparison rerun. No new browser gate, durable plan kind, or executor lifecycle phase. Definition below. - `comparison-publication-workflow` ([FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports)) — **active on `ka/fe-1251-comparison-publication`, stacked on FE-1250:** capture immutable controller and release provenance before comparison lanes start, then explicitly publish retained validity-first reports into the canonical Notion database through a guarded idempotent skill. Definition below. -- `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases)) — **active on `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240:** both frozen brownfield profiles/oracles and D137-L's admitted, lane-ready two-commit historical replay path are built and contract-hardened. Final admission now follows D136-L's tracked-clean dependency boundary, pinned case recipes are exhaustive with no default, and the production deep operation has a reachable-network rival. Preserve the minimal Petri-net editor as the only greenfield case. Next: scope and run the fresh historical provider preflight. +- `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases)) — **implementation complete on `ka/fe-1241-brownfield-comparison-cases`, based on `next` after FE-1240 landed:** both frozen brownfield profiles/oracles and D137-L's admitted two-commit replay path are built and contract-hardened; pinned recipes are exhaustive and the deterministic suite is green. No active scope remains. Any future Petrinaut provider attempt re-enters through D138-L's real merged-reference preflight. - **Carved from FE-1167 (2026-07-13):** the Execute-mode evidence sub-list — Execute entry beats on thin vs rich seeds (assessment honesty: Ask on thin, Proceed on rich) and the FE-1107/KA residue (close-or-narrow, demo/walkthrough session via `TESTING_PLAN.md`, post-KA plan pass). The former sticky-posture question is no longer KA residue: FE-1187 `remediation-4` owns the persistent Specify elicitation-style audit/SPEC revision, and its Continue lexical audit owns the old `continue` ambiguity. Full context in the archived FE-1167 definition (`docs/archive/PLAN_HISTORY.md`). - `planning-process-model` — **moved to the KA stream 2026-07-13; reshaped by D126-L**: the durable scope handoff is settled, so this item now owns only plan projection and epistemic-horizon questions beyond committed scopes. Definition below. - **[1.x data-model handoff owed by FE-1187](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md):** a concise colleague-facing note must distinguish current canon from directional vocabulary before the consolidated outer checkpoint. Current: `{milestone, frontier, scope}` with executor-derived slices; basis (`explicit | implicit`) orthogonal to settlement (`advisory | settled`); no persisted readiness grade or spec-global elicitation-gap table; active-branch Pi JSONL reads; one CommandExecutor mutation authority with spec-local LSN/change log; no new projected `vv_obligation` (legacy rows remain readable). Directional only: explain stored `thesis` as pitch/concept without renaming it yet, and avoid new coupling to spec-local `term` while its possible workspace lift remains future work. Persisted judgment-shaped reconciliation needs remain current, but are YAGNI-suspect: add no kinds/consumers/orchestration dependency without fresh evidence; re-evaluate derivation/removal when KA work first needs that table. Delivered note: [`Brunch 1.x data-model handoff`](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md); it does not block the deterministic TUI queue. @@ -460,10 +460,10 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu ### brownfield-comparison-cases - **Name:** Add Brunch and Petrinaut brownfield end-to-end comparison cases -- **Linear / branch:** [FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases), child of FE-1211; `ka/fe-1241-brownfield-comparison-cases`, stacked on FE-1240. +- **Linear / branch:** [FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases), child of FE-1211; `ka/fe-1241-brownfield-comparison-cases`, based on `next` after FE-1240 landed. - **Kind:** structural evaluation expansion — parameterized case/substrate/oracle composition plus two repository-backed end-to-end cases; one frontier with several scoped slices, not one frontier per case. - **Certainty:** proving. -- **Status:** blocked 2026-07-22 after the diagnostic real no-provider Petrinaut preflight. D137-L is materialized as the sole pinned preparation path and its admission contracts are hardened. The controller preflight operation and CLI now compose the real parent preparation, disjoint controller-only merged-reference materialization, the same compiled immutable recipe, unchanged focused oracle, write-once path-redacted receipt plus bounded retained phase evidence, and owned cleanup without a provider turn. The real frozen parent and merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0` both passed immutable install and tracked-source cleanliness. The focused oracle includes the directly declared Refractive workspace build before Petrinaut UI; all six preparation steps pass without broad graph preparation. Browser navigation now separates a 30-second `domcontentloaded` budget from 5-second semantic actions, and real evidence contains no aborted request or navigation timeout. Calibration still returns `assertion_failed`: the exact `Optimizations` heading is absent and every downstream check cannot find an exact `Create optimization` button. Pinned historical source proves that the real view labels its action `Create` and that `/optimization` mounts the ordinary local-storage Petrinaut shell, so the synthetic fixture's semantic contract requires respecification before provider work. Receipt SHA-256 `0ddbd4bffd4e57ce7426bc254bd8780e6d399c1eaf98f27499c261ac174a8046` retains the invalid assertion result and all three bounded evidence files, and both workspaces were removed. +- **Status:** implementation complete 2026-07-22; PR #362 is conflict-free and its full CI, Bugbot, and Semgrep gates pass. D137-L is the sole pinned preparation path, both frozen case/oracle stacks are built, and Petrinaut's public contract was rebaselined from fixture-authored labels to source-backed semantics before any provider attempt. The post-rebaseline real merged-reference preflight was explicitly waived, so no passing reference receipt or provider-campaign evidence is claimed. D138-L remains the mandatory re-entry gate before a first Petrinaut provider attempt. - **Objective:** prove that FE-1239's exact elicitation → immutable handoff → crossed Brunch/Claude execution → requirement-ledger path works against real pinned brownfield repositories without weakening controller isolation or turning the Petri tracer into generic campaign machinery. - **Lights up:** pinned repository + private feature mission + shared public baseline → two codebase-aware elicitation lanes → two exact handoffs applied to the same base tree → four isolated execution cells → repository-specific controller oracle → validity-first requirement ledger, once for Brunch and once for Petrinaut. - **Stabilizes:** one data-driven case profile across `greenfield` and `brownfield`; immutable repository/base-tree identity beside immutable spec identity; clean same-base materialization for every lane; an exhaustive code-owned dependency recipe per pinned case, including the closed controller-only immutable install before runtime denial; pinned Brunch graph seeding and launch identity without tracked-source mutation; compile-time oracle dispatch; and diff/evidence collection that excludes harness seed files without widening FE-1230's `ExecutionAttempt`. @@ -473,12 +473,12 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Retired:** A49-L; later historical case launches must consume this admission path and may not fall back to prompt-only conduct. - **Why now / unlocks:** FE-1239 proved the full chain only on an empty greenfield browser app, and FE-1240 closes the slice-admission defect that stopped both Brunch execution cells. The next load-bearing unknown is whether the same contracts survive repository inspection, pinned pre-existing code, and feature-delta verification. - **Boundary:** retain `minimal-petri-net-editor` unchanged as the sole `greenfield / whole_application / frontend` case; add exactly the derived FE-1201 host-landing slice as `brunch / brownfield / single_feature / backend` and PR #9051 as `petrinaut / brownfield / single_feature / frontend`. Petrinaut receives the full pinned HASH monorepo, not an extracted library fixture. The framework may parameterize case id, frozen axes, repository substrate, delivery contract, and sealed oracle id, but it must not accept arbitrary manifest shell commands or make controller material or historical solution refs target-visible. -- **Acceptance:** the existing Petri case, hashes, browser-oracle behavior, and promoted witness remain valid; both brownfield cases freeze content-addressed mission/baseline/registry/execution/oracle packets; every elicitation and execution lane starts from a separate clean checkout of the same declared base tree with no later refs/remotes or reachable historical-solution services; exact approved spec and contract bytes cross unchanged; pinned cases require an explicit safe controller source and record the compiled immutable install without tracked-source mutation before runtime network denial; pinned Brunch preparation returns a launchable `specId` whose brownfield graph preserves the exact approved bytes and is Execute-plan ready; harness seed commits are identified separately from implementation diffs; controller roots remain unreachable; a compile-time registry dispatches each known oracle; known-good fixtures pass and one focused negative fixture per case proves oracle sensitivity; each real case retains exactly two elicitation handoffs and four execution attempts with complete cleanup and requirement evidence or `not_assessable`. -- **Verification:** per `memory/SPEC.md` §Verification Design. Inner — case-profile, base/tree identity, materialization, strict solution-isolation, path-isolation, oracle-dispatch, and handoff/matrix contracts. Middle — unchanged Petri regression; Brunch black-box `/brunch:land` TUI journeys resume a settled session under `PI_OFFLINE=1` and are judged by an independent full-range Git model over disposable repositories; Petrinaut deterministic fake-provider browser, host/iframe bridge, and accessibility journeys; focused final-commit-only/behavioral rivals prove sensitivity. Outer — optional real-optimizer smoke, masked Petrinaut visual review, masked code-quality review, then one fresh staged 2×2 run and promoted bounded report per brownfield case. Historical code/diff/architecture similarity is never an oracle. +- **Acceptance:** the existing Petri case, hashes, browser-oracle behavior, and promoted witness remain valid; both brownfield cases freeze content-addressed mission/baseline/registry/execution/oracle packets; every lane-ready target starts from a separate clean checkout of the declared base tree with no later refs/remotes or reachable historical-solution services; exact approved spec and contract bytes cross unchanged; pinned cases require an explicit safe controller source and record only their compiled dependency recipe without tracked-source mutation before runtime network denial; pinned Brunch preparation returns an Execute-plan-ready `specId`; harness seed commits remain distinct from implementation diffs; controller roots remain unreachable; compile-time registries dispatch every recipe and oracle; known-good fixtures and focused rivals prove deterministic sensitivity. A future provider campaign, if authorized, owns the two elicitation handoffs/four execution attempts, complete cleanup, and requirement evidence; Petrinaut cannot enter it until D138-L passes. +- **Verification:** per `memory/SPEC.md` §Verification Design. Inner — case-profile, base/tree identity, materialization, strict solution isolation, lexical/canonical/symlink path isolation, oracle dispatch, and handoff/matrix contracts. Middle — unchanged Petri regression; Brunch black-box `/brunch:land` TUI journeys over disposable repositories plus an independent full-range Git model; Petrinaut deterministic fake-provider browser/accessibility journeys and focused behavioral rivals. Outer — no FE-1241 provider campaign was run. A future authorized campaign must first produce a passing D138-L real-reference receipt; optional real-optimizer, fully provisioned host/iframe, and masked visual/code review remain non-gating. - **Cross-cutting obligations:** public baseline controls only addressability needed by the common oracle and is never credited as elicitation gain; existing repository knowledge may be inspected equally by both elicitation lanes; private mission/reveal/oracle files never enter target workspaces; failed and invalid attempts remain retained; Brunch stops at `promotion_prepared` and never lands into either source repository; provider/model/budget/intervention rules remain frozen per study. - **Explicitly out:** Clay/Pi Campaign Machine; any second greenfield case; whole-repository rewrites; evolving-plan studies; reliability repetitions; automatic scoring or cross-case winner; live production deployment; Cursor/Codex lanes; arbitrary oracle plugins; production `/compare-end-to-end`; landing provider outputs into either source repository; and FE-1230 `ExecutionAttempt` schema widening. - **Traceability:** D70-L fixture taxonomy; D134-L/I67-L controller/mission isolation; D40-L/D120-L/I62-L execution and no-landing boundaries; FE-1210/FE-1230/FE-1232/FE-1239/FE-1240; [`docs/praxis/comparison-runs.md`](../docs/praxis/comparison-runs.md); Notion `brunch Test Plan` axes. -- **Current execution pointer:** [`memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md`](cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md) is active. It owns D138-L's one pre-provider source-fidelity audit and rebaseline, typed mechanical-address contract/hash regeneration, synthetic sensitivity rivals, and unchanged real merged-reference pass. The prior [`petrinaut-provider-preflight`](cards/brownfield-comparison-cases--petrinaut-provider-preflight.md) remains the retained blocked receipt/evidence source; no provider matrix may start until the new card closes. +- **Current execution pointer:** none. All tracked scope files are consumed. Re-enter only if a Petrinaut provider attempt is authorized; D138-L then requires the real merged-reference preflight to pass before any lane starts. ### comparison-reporting-skills @@ -669,7 +669,7 @@ KA stream: reuses: canonical harness | isolated integration | epic verification excludes: browser-specific executor gate | durable plan kind | new lifecycle phase -[hard]-> brownfield-comparison-cases - status: active FE-1241; A49-L and Brunch profile/oracle complete, Petrinaut scope revised to standalone hard gate after A50-L invalidation + status: implementation complete FE-1241; no provider campaign run; future Petrinaut attempts gate on D138-L lights_up: same-base brownfield elicitation -> exact handoff -> crossed execution -> case oracle cases: minimal-petri-net-editor = sole greenfield reference | FE-1201-derived brunch backend | PR #9051 petrinaut frontend excludes: Clay | more greenfields | repetitions | aggregate winner | ExecutionAttempt widening diff --git a/memory/SPEC.md b/memory/SPEC.md index 43b0df6e9..887ad64ab 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -128,7 +128,7 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | A47-L | Pi's valuable `InteractiveMode` TUI behavior can be preserved while one independent cwd-scoped Brunch session host owns the sole writable sealed Pi runtime, JSONL session manager, graph command authority, and semantic live-event fan-out used by both TUI and React clients. The unknown is the TUI attachment seam: Pi exports `InteractiveMode` over an in-process `AgentSessionRuntime`, not a remote TUI client. Validation must prove a real TUI + browser target without a second writable runtime, raw-Pi browser contract, or permanent second relay. Frontier: `shared-session-host-tracer`; retirement/cutover: `shared-session-host-cutover`. | medium | open | D39-L, D132-L, D133-L; I64-L, I65-L | | A48-L | A read-only semantic preflight can improve orientation-menu availability over deterministic graph-fact heuristics while returning within a ≤3-second interaction budget often enough to justify a model-backed path. Admission is limited to the configured soft recommended evaluator model; other foreground selections, missing evaluator auth, timeout, malformed output, or failure use the deterministic safe subset without restricting Pi-native `/model`. The path must consume no foreground turn, write no transcript/graph/session truth, and cache only in process by `{specId, lsn, operationalMode}`. A Brunch-owned reconciliation-blocker reader participates in gating now with an explicit empty implementation; injected non-empty blockers veto availability so future derived/persisted blocker wiring cannot be forgotten. Validation: a tracked human-approved contrastive catalog plus structured scratch tracer report; first run three uncached feasibility calls, then only if plausible run ten uncached labeled cases, requiring ≥8/10 within 3 seconds, exact flags on every completed response, at least one named deterministic-fallback miss corrected, and zero false-positive moves. If the budget or quality gate fails, retire only the model path, not the deterministic fallback. | low | open | D74-L, D109-L, D123-L; `walkthrough-remediation-2` | -| A50-L | The real HASH `/processes/draft` host/iframe route can be the Petrinaut comparison's common hard-gate address under focused controller preparation, with unrelated auth/backend behavior cheaply stubbed. **Invalidated 2026-07-21:** the route source accepts a backend-free draft, but the Next application shell fails before rendering without generated Panda and GraphQL artifacts plus built workspace dependencies. An immutable install fetched 3,570 packages (2.79 GiB); frontend dependency preparation fanned into 61 packages including Rust/toolchain work and stopped on a pinned mise-version mismatch. By contrast, focused design-system/Petrinaut builds brought the merged standalone `/optimization` route to an accessible Optimizations view without HASH backend state. | high | invalidated | D136-L; `brownfield-comparison-cases` | + ### Active Decisions @@ -306,9 +306,9 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D132-L | Standalone interactive web uses one cwd-scoped combined Brunch host with a target-addressed inventory of sealed, in-process Pi `AgentSession`s (2026-07-14). The same process serves React assets/WebSocket Brunch RPC and owns coordinator/graph authority; it does not construct `InteractiveMode`, expose raw Pi RPC, spawn one Pi child per session, host multiple projects, or promise in-flight survival across host restart. One durable session target has one driver/many observers and cannot be opened as duplicate writable runtimes; write leases wait for real same-session contention. Hosted-session mutations return the complete `LiveSessionHostResult` discriminated `{status}` union as JSON-RPC success payloads, including domain refusals; only malformed boundary input and thrown host failures use JSON-RPC errors. FE-1200 materialized the one-target path and validated simultaneous target isolation (A43-L). Depends on: D5-L, D10-L, D33-L, D39-L, D84-L; req 4, req 31. Supersedes: D10-L/D72-L read-only-sidecar posture and D84-L singleton/TUI-owned target topology. | [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md), [`src/session/TOPOLOGY.md`](../src/session/TOPOLOGY.md), [`src/rpc/TOPOLOGY.md`](../src/rpc/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — target-addressed host and concurrent-session isolation materialized 2026-07-14 | | D133-L | Web and TUI share transport-neutral presentation semantics, not platform components or separately-authored domain decoders (2026-07-14). Validated Brunch `toolResult.details` project to a shared semantic presentation model; LLM-context, TUI, and React adapters render that meaning for their audiences. Web hydrates from a named JSONL-derived product projection and overlays target-addressed live events, then refetches canonical truth at settlement/reconnect. The host emits neither ANSI/TUI strings nor ready-made HTML, and no chat mirror/event store is introduced. FE-1200 materialized the full required persisted family inventory: ordinary text; free-text, choice, choices, and bounded-questionnaire terminal read-back; candidate, review-set, and digest offers/continuations; and receipt-bearing review settlement. Live React controls answer free text and listed single/multi choices; bounded questionnaires remain answerable headlessly through D38-L's schema-tagged string/JSON envelope, without a dedicated React questionnaire form. Depends on: A44-L, D17-L, D19-L, D104-L; req 12, req 17, req 32. | [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — full required-family coverage materialized 2026-07-15 | | D134-L | The approachable `/compare-specs` control topology is one top-level project Pi session acting as the simulated user and driving exactly one comparison-harness interactive subshell at a time. It does not spawn a Pi actor that then opens a nested interactive shell. The top-level agent alone receives the private mission; each harness receives only its approved minimal framing and the user's natural messages. Shared top-level context and lane order are acceptable and disclosed for this exploratory workflow; isolation-sensitive studies continue to use FE-1210's separate rigorous campaign recipe. Ordinary text interaction is the stock-Pi baseline for choices and approvals; a custom structured-question tool may enhance presentation but is never required. Setup checks are bounded to actual selected-harness prerequisites—no throwaway Pi/Claude provider turns or synthetic actor launches on every run. Supersedes: FE-1215's unmaterialized fresh-nested-actor design note. | [`.pi/prompts/compare-specs.md`](../.pi/prompts/compare-specs.md), [`testing/comparisons/missions/README.md`](../testing/comparisons/missions/README.md); PLAN `operator-comparison-workflow` | active — remediation materialized and focused Brunch smoke witnessed 2026-07-17; full comparison witness deferred | -| D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller preparation boundary. Before the network-denied candidate lane, the controller materializes the declared parent tree and exact handoff, then runs only `corepack yarn install --immutable --mode=skip-build`; this closed immutable install is the sole package-registry-network allowance and must leave tracked source clean. After the lane terminates at `promotion_prepared`, the controller runs the closed focused package builds and launches the standalone website `/optimization` route. A deterministic loopback fake optimizer grades capability-present UI, scenario-first configuration, fixed/optimized flat bindings, one saved/custom metric plus direction, request construction, progressive trials/best/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and the source task's observable accessibility semantics. D138-L's merged-reference calibration is the authority for whether this historical replay's mechanical contract is setup-valid; synthetic fixtures prove sensitivity but cannot invent exact labels, roles, or interaction shapes absent from the source task/reference. The real HASH `/processes/draft` host/iframe integration is not a common hard gate: its global Next shell requires unrelated GraphQL/codegen and broad 61-package platform/toolchain preparation, so one fully provisioned host/iframe smoke plus masked code/visual review remain explicit non-gating outer evidence and report `not_assessable` when unavailable. Depends on: A50-L, D134-L, D138-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases`; Petrinaut scope card | active — materialized 2026-07-22; reference-fidelity correction active | +| D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller boundary. Before a network-denied candidate lane, the controller materializes the parent tree and exact handoff, runs only `corepack yarn install --immutable --mode=skip-build`, and requires tracked source to remain clean. After the lane terminates at `promotion_prepared`, closed focused builds launch the standalone `/optimization` route; a deterministic loopback optimizer grades scenario-first configuration, fixed/optimized bindings, saved/custom objective direction, request construction, progressive/completion/error/cancellation behavior, upstream abort, same-origin secrecy, and source-backed accessibility semantics. D138-L owns merged-reference calibration; synthetic fixtures prove sensitivity but cannot invent labels, roles, or interaction shapes. The broad `/processes/draft` host/iframe shell remains non-gating outer evidence because it requires unrelated platform preparation. Depends on: D134-L, D138-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases` | active — implementation and deterministic source-fidelity rebaseline materialized 2026-07-22; real merged-reference pass not retained | | D137-L | Historical replay preparation has one deep public operation that owns the security-sensitive order from pinned-source materialization through exact handoff, closed case-owned dependency preparation, strict admission, and lane finalization (2026-07-22). Its final Git authority is the declared two-commit synthetic prefix clarified under A49-L; admission validates the root source identity, packet-only child, strict Claude/Brunch policy pair, network/path probes, and absence of any third history, remote, ref, residue, or escaping symlink. Public callers receive only a lane-discriminated ready descriptor (`Brunch` includes the seeded brownfield `specId`; Claude does not); private stage types may enforce internal ordering, but no public phase machine, arbitrary command/plugin surface, or serializable security token is introduced. A compile-time case registry selects either no dependency preparation for Brunch host landing or D136-L's sole immutable Petrinaut recipe. Depends on: A49-L retirement, D136-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases` | active — materialized 2026-07-22 | -| D138-L | A historical replay profile must pass a no-provider-turn behavioral calibration against its declared merged reference before its first provider attempt (2026-07-22). The source issue/PR and merged behavior—not a synthetic fixture—own the replay task's observable semantics; the merged patch is calibration evidence, never a golden diff or architecture oracle. Before any attempt exists, a contradicted public packet/oracle may be rebaselined to those source semantics with all hashes and rivals regenerated. Synthetic fixtures remain sensitivity rivals and may not author exact labels, roles, control shapes, or stronger requirements absent from the replay source. Any reference mismatch blocks provider work rather than being relabeled setup-valid; after the first provider attempt, frozen packet/oracle bytes may not be rewritten. Depends on: D70-L, D136-L, D137-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases`; Petrinaut provider-preflight card | active — Petrinaut reference-fidelity rebaseline selected 2026-07-22 | +| D138-L | A historical replay profile must pass a no-provider-turn behavioral calibration against its declared merged reference before its first provider attempt (2026-07-22). The source issue/PR and merged behavior—not a synthetic fixture—own observable semantics; the patch is calibration evidence, never a golden diff or architecture oracle. Before any attempt exists, a contradicted packet/oracle may be rebaselined with hashes and rivals regenerated. Synthetic fixtures remain sensitivity rivals and may not author stronger requirements absent from the source. Any reference mismatch blocks provider work; after the first provider attempt, frozen bytes may not be rewritten. Depends on: D70-L, D136-L, D137-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases` | active gate — source-fidelity rebaseline materialized 2026-07-22; post-rebaseline real calibration explicitly not run, so no first provider attempt is authorized | | D60-L | Agent context splits into pull / projection / render / surface, distinguishes graph-truth from active-context reads, and keeps `workspace.state` separate. Agent context (what the agent reasons over) spans `cwd` (filesystem kickoff heuristic — `.brunch?`, session count/length, README/markdown sizes, file counts), `graph` (overview/list/query), and `node` (variable-hop neighborhood). ... | [`src/graph/TOPOLOGY.md`](../src/graph/TOPOLOGY.md), [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md) | active | | D83-L | Context-render house style: a markdown frame (md-pen) with TOON for uniform data and a fenced ASCII tree for hierarchy, wrapped in `
` tags; agent context clusters into `` / `` / `` scopes. Refines D60-L's RENDER stage. LLM-facing agent-context renders adopt one consistent dialect instead of ad-hoc `[bracket]` + bullet lists: - **Audience scope.** `src/agents/contexts/` owns the LLM agent-context dialect (the `
` scope-clustering plus TOON data blocks and t ... | See archive snapshot for full rationale. | active | | D85-L | Suspended prompt-resource axis model: strategy/lens/method are no longer runtime state. A 2026-06-18 grill consolidation of the `agents/skills/` topology and the D58-L manifest axes, implemented across FE-893, FE-861, and FE-898, produced useful prompt-resource content and path topology. ... | [`src/agents/TOPOLOGY.md`](../src/agents/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md) | retired/suspended | @@ -693,7 +693,7 @@ For agent-as-user evaluation, the primary behavioral claim is **consequential-fa **FE-1230 greenfield execution-comparison assessment (2026-07-20).** Observability is **partial**: build/test output, git trees, browser DOM/accessibility state, console errors, JSON downloads, Brunch run/Petri artifacts, and target-visible interactions are text-native, while ordinary visual hierarchy and drag feel remain human judgments; product-private reasoning and diagnostics are excluded from common evidence. Reproducibility is **partial**: the approved Petri-editor specification, empty-repository base, public automation contract, Opus 4.8 model, budgets, and hidden oracle bytes freeze before the first valid lane, but generated implementation shape and model conduct vary; three Brunch repetitions detect gross instability rather than reliability tails. Controllability is **partial**: the controller can install and run unchanged build, browser, reference-model, round-trip, and negative-space checks against fresh lane outputs, while provider availability and product-native planning remain external. A minimal public accessibility contract gives both lanes stable roles/names without revealing hidden scenarios or expected results. -**Brownfield end-to-end comparison assessment (active 2026-07-21).** Observability is **partial**: pinned base/final trees, diffs, test/build output, controller results, and Brunch temporary-host Git state are text-native; Petrinaut interaction quality and progress presentation remain partly visual. Reproducibility is **partial**: historical parent commits, sanitized missions, exact handoffs, deterministic fake optimization responses, and temp-repository fixtures can freeze the mechanical conditions, while elicitation/execution model conduct still varies and the optional real optimizer remains external. Controllability is **partial with a fail-closed isolation route**: Brunch host landing can run entirely against temporary Git repositories and Petrinaut's hard gates can use a fake provider; A49-L is retired by one versioned admission contract over asymmetric Claude/Brunch controls, history-free source materialization, target-bounded reads, and network-denied verifier commands. For the first Brunch slice specifically, Git outcomes are highly observable and reproducible: the frozen profile and compiled claim-linked oracle now resume a settled session under `PI_OFFLINE=1`, drive `/brunch:land` through the public TUI, and judge disposable repositories with an independent Git model; fresh sessions fail closed because they can auto-kick the configured provider. Petrinaut/profile and all historical provider work remain gated behind later scoped tracers. +**Brownfield end-to-end comparison assessment (updated 2026-07-22).** Observability is **partial**: pinned trees, diffs, build/test output, controller verdicts, temporary-host Git state, browser DOM/accessibility state, request/abort evidence, and cleanup are text-native; visual hierarchy and provider conduct remain external. Reproducibility is **high for deterministic mechanics and unproven for campaigns**: frozen Brunch/Petrinaut packets, the two-commit historical target, compile-time recipes/oracles, fake optimizer, and contrastive rivals are stable, but no Petrinaut provider campaign or post-rebaseline real-reference pass was retained. Controllability is **partial with fail-closed isolation**: Brunch host landing runs under `PI_OFFLINE=1` against disposable repositories and an independent Git model; Petrinaut runs focused builds and same-origin browser checks only after lane termination. D138-L blocks a future first Petrinaut provider attempt until an explicit real merged-reference preflight passes. **Standalone-web tracer assessment (2026-07-14).** Observability is **partial**: JSONL, target-addressed RPC, semantic event frames, and accessible DOM/text states make structural session behavior observable; stream cadence, visual hierarchy, and error feel remain manual. Reproducibility is **high** for this tracer: paired temporary production boots use the deterministic faux provider and an existing JSONL session, not a live model or a static transcript golden. Controllability is **partial**: the middle loop controls the browser journey and cache/overlay loss, while a workbench walkthrough owns the bounded UX verdict. The normalizer may remove only declared nondeterministic ids/timestamps; it must not mask Brunch binding/runtime/exchange differences. @@ -853,7 +853,7 @@ The first required probe is M0: after manual TUI interaction, a checker proves ` - **Operator-led cross-product comparisons (FE-1215; D134-L remediation before later `saved-mission-comparison-witness` evidence).** The PM-facing comparison door is deliberately distinct from rigorous frozen-packet evaluation. One project Pi prompt conversationally creates or revises a rich private **agent-as-user mission** for a simulated user: their objective, context, priorities, preferences, constraints, knowledge, uncertainty, decision latitude, and conversational posture. The invoking top-level Pi agent receives that mission and directly performs the user's side of each interaction while driving exactly one comparison-harness subshell at a time; it must not spawn a Pi actor that opens another interactive shell. Each comparison harness receives only minimal visible framing plus the opening user message and subsequent mission-grounded answers. Harness selection and framing are run setup, not mission content. Ordinary conversational text is the portable baseline for operator choices and approvals; environment-specific structured-question tools are optional presentation only. Setup checks cover actual selected-harness prerequisites without synthetic actor/provider turns on every run. Editable missions live outside `.fixtures/` under `testing/comparisons/missions/`; each run snapshots the private mission, separately identified target-visible setup/interactions, and outputs under `.fixtures/runs/agent-as-user-comparison/` while temporary lane work stays in scratch. The readable operator report may expose the full private mission so elicitation can be compared against what each harness actually learned, but it keeps that baseline separate from target-visible evidence and declares no automatic winner or prescribed rubric. Because the top-level actor context spans sequential harnesses, this approachable workflow discloses order and does not claim the per-lane actor-process isolation required by rigorous frozen-packet studies; frozen reveal policies, matched budgets, blinding, fresh-per-lane actor sessions, structured adjudication, multi-run statistics, and scripted judges remain separate tools for focused improvement/regression claims. - **FE-1230 execution-comparison oracle boundary.** Execution cases are distinct from private elicitation missions: the human-approved specification and a minimal public runtime/accessibility contract are visible to every lane, while exact browser journeys, reference-model states, expected results, claim mapping, and adversarial fixtures remain controller-only and outside every lane cwd. The public contract requires a static production build at `dist/`, `npm run build`, `npm test`, and stable accessible roles/names for the canvas and named controls; it does not prescribe framework, source topology, implementation decomposition, test library, or internal state model. The versioned `petri-editor-browser-v2` suite tests/builds once, then runs every declared journey from a fresh browser context with public-only setup, per-journey runtime evidence, and non-blocking claim-linked verdicts that distinguish harness/setup failure from product assertion failure. Brunch stops at `promotion_prepared` and never lands. The retained first pair remains immutable; replay/promotion waits for its exact artifact paths. Mutants, masked/process judging, repetitions, and generalized campaign machinery are deferred until one valid end-to-end path exists. The tracer pins `anthropic/claude-opus-4-8` in both products, but same model does not imply equivalent hidden prompting or thinking controls. -- **Brownfield historical-replay oracle boundary (`brownfield-comparison-cases`).** Historical implementations bootstrap independent claims, fixtures, and focused wrong rivals; matching their code, module decomposition, diff shape, or architecture is never a correctness gate and the references never enter masked review. Brunch derives a backend-only host-landing mission from FE-1201/PR #336 at parent `f5a423b19f76cf345d88053456870a126e451618`; because the original PR also changed presentation surfaces, its whole merged diff is explicitly not a golden patch. Its common mechanical address is the disclosed `/brunch:land` command: the controller resumes a settled session fixture under `PI_OFFLINE=1`, drives the built candidate through the real TUI, and lets an independent Git model judge pre-confirm and post-apply temporary-repository snapshots. The compiled v1 case/oracle now closes that route over brownfield, greenfield, refusal, stale-acceptance, full-range, and bookkeeping rivals; fresh-session setup fails closed because it can auto-kick the configured provider. Existing `landing.ts`/`GitHostLandPort` tests may bootstrap fixtures and wrong rivals but cannot enter the comparison oracle. Petrinaut replays only optimization UI PR #9051 in the full `hashintel/hash` checkout at parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978`; optimizer backend/API capabilities already present there are disclosed baseline, not implementation work or elicitation gain. D136-L makes the focused public `/optimization` website route the deterministic fake-provider hard gate; the real HASH `/processes/draft` host/iframe integration is a fully provisioned non-gating outer smoke because the global Next shell requires unrelated GraphQL/codegen and broad platform/toolchain preparation. Public baselines expose only mechanical addressability and pre-existing capabilities required by common oracles. Missions omit historical ids, titles, URLs, and solution wording; target trees are archive/materialization products with no later refs or remotes. The retired A49-L recipe is now executable admission: Claude uses fail-closed native sandbox settings, strict empty MCP, an explicit non-web tool set, and domain denial; Brunch comparison launches disable web/research/Specify-subagent access, use target-bounded foreground and child file tools, and sandbox verifier commands with runtime network forbidden. The selected historical patches add no external package dependency unavailable at their parent trees, so focused dependency preparation can finish controller-side before denial. A real optimizer smoke, the full HASH host/iframe smoke, and masked review remain non-gating outer evidence. +- **Brownfield historical-replay oracle boundary (`brownfield-comparison-cases`).** Historical implementations bootstrap independent claims and focused rivals; matching code, decomposition, diff shape, or architecture is never a gate. Brunch replays only FE-1201's backend host-landing behavior through public `/brunch:land`, a controller-supplied settled session under `PI_OFFLINE=1`, and an independent temporary-repository Git model. Petrinaut replays only PR #9051's optimization UI in the full pinned HASH checkout; pre-existing optimizer backend/API capability is disclosed baseline. D136-L makes standalone `/optimization` the deterministic fake-provider hard gate after the one compiled immutable install and closed focused builds; broad `/processes/draft` integration is non-gating outer evidence. D137-L admits only the declared source-root plus packet-child prefix with exhaustive recipes and strict Claude/Brunch isolation. D138-L makes source behavior—not synthetic labels—the semantic authority and requires a passing controller-only merged-reference calibration before a first provider attempt. The deterministic source-fidelity rebaseline is built, but the real post-rebaseline calibration and provider campaign were explicitly not run. Real optimizer, fully provisioned host/iframe, and masked reviews remain non-gating. - **Shared-session-host convergence oracle (planned; `shared-session-host-tracer` → `shared-session-host-cutover`).** FE-1200's standalone proof is necessary but not sufficient for architectural replacement. The proving oracle must launch the production host as the sole owner of one writable sealed Pi runtime, attach a real TUI presentation and React observer/driver to the same durable target, exercise an ordinary turn plus one extension-owned structured ask and one TUI-only product interaction, detach/restart a client without ending the hosted runtime, and reject a duplicate writable open or conflicting driver. Durable settlement must converge through the same JSONL presentation, and browser traffic must remain Brunch semantic RPC/events rather than raw Pi RPC/events. Once A47-L retires, the cutover becomes a closed coverage sweep: every required TUI/web lifecycle, command/UI, exchange, transcript, graph-update, model/auth, and shutdown row has one host-owned path and a closure oracle; only then may `SessionEventRelay`, `brunch.sessionEvent`, `/rpc/driver`, and their sidecar harnesses be deleted. Human outer evidence must confirm that the TUI remains useful rather than becoming a thin degraded shell. - **Standalone-web compound oracle (2026-07-14; coverage completed 2026-07-15, `standalone-web-session-host`).** The initial five complementary oracles prove the host tracer: (1) inner RPC/host negative-space contracts require `(specId, sessionId)` on every lifecycle/driver/ask/event path and reject targetless fallback, duplicate writable opens, second drivers, and mismatched targets; (2) projection shape/malformed-detail tests keep JSONL and raw Pi/detail shapes behind the named semantic presentation; (3) a production-wired standalone-web browser journey, controlled only by a deterministic faux provider, asserts accessible hydration, streamed text, one `ask`, answer, and `agent_settled` states; (4) paired temporary web/TUI runs use a narrow declared normalizer to prove both live→settled→fresh JSONL hydration and equivalent Brunch binding/runtime/exchange semantics—no static JSONL golden; (5) a workbench manual checklist judges only stream cadence, busy/settled truthfulness, ask interaction, reload, and error feel. Browser cache/overlay loss is part of the middle-loop journey. The follow-on production-host concurrency differential retires A43-L with overlapping graph writes, asks, failure/recovery, target-local event sequences, reconnect, and separate JSONL readback; shared graph changes appear only through canonical `worldUpdate` continuity. The completed coverage pass adds projection no-loss/malformed tests for every required persisted ask terminal shape (including questionnaire read-back), React render/answer tests for free text and listed single/multi choices, headless schema-envelope questionnaire answering coverage (with no dedicated React questionnaire form), distinct candidate/review-set/digest production settlement/reconnect witnesses, concurrency/target isolation, and receipt-bearing review settlement. Live-provider conduct and process-restart survival remain explicitly deferred to their named later work; no second truth plane or raw Pi browser contract is introduced. - **Trace → eval → score → regression flywheel.** The first combined trajectory/evaluation proof is a controlled Brunch A/B over the existing consequential-fact claim, not a generic tracing platform or omnibus architecture score. One realistic non-inferable scenario carries a human-authored hidden-fact ledger, forbidden rivals, and a controlled reveal policy. The only intervention is an eval/dev ablation of the warrant-before-commit directive at the real prompt-composition seam; all other run conditions remain fixed. Three real-provider TUI-driven runs per arm produce a joined legibility envelope: stamped run configuration, directive inventory and content hashes, advertised/read/provider-visible directive state, ordered model/tool/exchange/TUI/graph effects, atomic evaluator judgments with evidence and rubric identity, and a replayable report. Completion requires promoted evidence that discriminates the full directive from the ablated rival; the report is an instrument, not the completion claim. This proves bounded evaluator discrimination, not competitor superiority or broad prompt quality. After the tracer, mine one real walkthrough failure into the same corpus. Keep OTel, broad subagent span joining, provider matrices, interaction-quality scoring, and competitor campaigns trigger-gated until a named claim requires them. diff --git a/memory/cards/brownfield-comparison-cases--admission-contract-hardening.md b/memory/cards/brownfield-comparison-cases--admission-contract-hardening.md deleted file mode 100644 index 8d68ed0de..000000000 --- a/memory/cards/brownfield-comparison-cases--admission-contract-hardening.md +++ /dev/null @@ -1,100 +0,0 @@ -# Harden historical replay admission contracts - -Frontier: brownfield-comparison-cases -Status: done -Mode: single -Created: 2026-07-22 - -Posture: proving (inherited from `brownfield-comparison-cases`). - -## Objective - -Historical replay admission enforces D136-L and D137-L without rejecting declared dependency artifacts or defaulting an undecided pinned case. - -## Cold-start reads - -- `memory/SPEC.md` — D136-L and D137-L -- `memory/PLAN.md` — frontier `brownfield-comparison-cases` -- `src/dev/TOPOLOGY.md` — Historical Replay Isolation -- `memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md` — completed tracer and preserved invariants -- `src/dev/execution-comparison/historical-replay-target.ts` — dependency selection and deep-operation composition -- `src/dev/end-to-end-comparison/solution-isolation.ts` — final-prefix cleanliness and network probes - -## Acceptance Criteria - -```text -admission contract hardening -├── ✓ historical-replay-target.test.ts — a dependency recipe may leave untracked artifacts while tracked source remains clean -├── ✓ historical-replay-target.test.ts — tracked dependency mutation still fails during dependency_preparation -├── ✓ historical-replay-target.test.ts — a branded verifier that reaches a required network probe fails during admission with network_probe_reachable -├── ✓ historical-replay-target.test.ts + TypeScript build — every pinned case id has one explicit code-owned dependency recipe and no default branch -├── ✓ solution-isolation.test.ts — packet drift, third commits, remotes, refs, and tracked worktree mutation remain rejected -└── ✓ card/link check — completed Petrinaut evidence points to the live historical-replay tests rather than deleted preparation tests -``` - -## Completion evidence - -| Leaf | Outcome | Evidence | -| ---- | ------- | -------- | -| Non-ignored untracked dependency artifacts remain admissible | met | `historical-replay-target.test.ts` runs the compiled Petrinaut install seam, leaves `.pnp.cjs` untracked, and reaches Claude readiness; final admission now asks Git only for tracked worktree changes | -| Tracked source or packet mutation still fails | met | the Petrinaut tracked-package rival fails in `dependency_preparation`; packet drift fails in `admission`; `solution-isolation.test.ts` now mutates tracked `package.json` and retains `git_worktree_changes_present` | -| Reachable required network probe prevents readiness | met | a deep-operation branded-verifier rival reaches `https://github.com`, fails in `admission` with `network_probe_reachable`, returns no descriptor, and removes the owned target | -| Every pinned case id has one explicit code-owned recipe | met | `PinnedExecutionCaseId` projects the contract owner union; `DEPENDENCY_RECIPE_BY_PINNED_CASE satisfies Record<...>` names Brunch `none` and Petrinaut immutable Yarn with no default; the TypeScript production build passes | -| Existing topology and isolation rivals remain rejected | met | focused historical-replay and solution-isolation suites retain wrong-parent, packet-drift, third-commit, remote/ref, tracked-worktree, path/symlink, policy, brand, and host rejection | -| Petrinaut evidence points to live tests | met | the completed Petrinaut card now cites `historical-replay-target.test.ts` and `solution-isolation.test.ts`; no `pinned-source-preparation.test.ts` reference remains | - -Untracked-artifact red: `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts -t "admits non-ignored untracked Petrinaut"` — 1 failed in `admission` with `git_worktree_changes_present` for `?? .pnp.cjs`. Green: the same command — 1 passed, 12 skipped by the name filter. - -Reachable-network sensitivity red: with the required `https://github.com` probe temporarily removed, `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts -t "branded verifier reaches a required network probe"` — 1 failed because preparation incorrectly resolved `ready`. Green after restoring the required probe: the same command — 1 passed, 14 skipped by the name filter. - -Exhaustive-map red: a temporary source sentinel failed while the production module still used the Petrinaut type guard/default-none branch. After the exhaustive `satisfies Record` map and runtime coverage for both current cases landed, the sentinel was removed as a representation lock; the TypeScript production build is the durable exhaustiveness oracle. - -Focused green before representation-only sentinel removal: `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts src/dev/execution-comparison/__tests__/operator-cli.test.ts src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts src/dev/end-to-end-comparison/__tests__/case-profile.test.ts src/dev/execution-comparison/__tests__/case-contract.test.ts src/dev/execution-comparison/__tests__/brunch-lane.test.ts` — 7 files, 45 tests passed. - -Checkpoint green after refactor: `npm run verify:full` — default 328 files/2,572 tests passed with 1 file/2 tests skipped; slow 9 files/68 tests passed; TypeScript and production builds passed. `npm run check` and `git diff --check` passed. - -Remaining provider-preflight debt: resolve the declared commit/tree against a real HASH checkout, run the real immutable install and retain its exact result, then prove the focused standalone `/optimization` route reaches setup-valid readiness before interpreting any candidate outcome. The separately named `/processes/draft` host/iframe evidence remains non-gating. - -## Verification Approach - -- **Inner:** contrastive deep-operation tests for allowed untracked dependency output, forbidden tracked mutation, reachable network, and exhaustive case-recipe selection. -- **Middle:** existing solution-isolation adversarial suite and full D137 historical-replay target suite. -- **Outer:** none for this hardening slice. The first fresh Petrinaut provider preflight owns actual HASH commit/tree resolution, real immutable install, and focused-route setup validity before any result is interpreted. -- **Checkpoint:** `npm run verify:full`, `npm run check`, `git diff --check`, and skipped-test delta versus `20709cdd`. - -## Cross-cutting obligations - -- Dependency preparation remains one compiled recipe; public contracts and manifests never supply commands. -- Only controller-owned pre-lane preparation may create dependency artifacts; source and packet files remain immutable. -- Production composition still creates the real fail-if-unavailable verifier; test composition may inject only a branded verifier factory. -- Setup rejection remains separate from candidate assertion failure. -- Brunch no-install behavior, strict asymmetric policies, no-landing, and the sole greenfield Petri path remain unchanged. - -## Assumption dependency - -None — this slice reconciles two already-settled decisions and adds the missing adversarial oracle. - -## Explicitly Out - -- A real HASH checkout/install/browser run; owned by the next provider preflight. -- General dependency plugins, persisted phase receipts, or a third brownfield case. -- Broader oracle registry or replay-module refactoring. - -## Expected touched paths (tentative) - -```text -memory/ -├── PLAN.md ~ -└── cards/ - ├── brownfield-comparison-cases--admission-contract-hardening.md - ├── brownfield-comparison-cases--historical-replay-target-admission.md ~ - └── brownfield-comparison-cases--petrinaut-optimization-case-oracle.md ~ -src/dev/ -├── TOPOLOGY.md ~ -├── end-to-end-comparison/ -│ ├── solution-isolation.ts ~ -│ └── __tests__/solution-isolation.test.ts ~ -└── execution-comparison/ - ├── historical-replay-target.ts ~ - └── __tests__/historical-replay-target.test.ts ~ -``` diff --git a/memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md b/memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md deleted file mode 100644 index 649f92161..000000000 --- a/memory/cards/brownfield-comparison-cases--historical-replay-target-admission.md +++ /dev/null @@ -1,173 +0,0 @@ -# Admit lane-ready historical replay targets - -Frontier: brownfield-comparison-cases -Status: done -Mode: single -Created: 2026-07-22 - -Posture: proving (inherited from `brownfield-comparison-cases`). - -## Orientation - -- **Containing seam:** D137-L historical replay preparation/admission between frozen case selection and Brunch/Claude execution launch. -- **Frontier:** FE-1241 `brownfield-comparison-cases`; keep this work on `ka/fe-1241-brownfield-comparison-cases`. -- **Volatile state:** the Petrinaut profile/oracle and pinned Brunch graph seed are built; review found `admitHistoricalReplay` test-only, one-commit-only, and absent from the production prepare path. -- **Main risk:** weakening A49-L while reconciling its source-materialization proof with FE-1239's separate exact-handoff commit. - -## Target Behavior - -Every pinned brownfield execution lane is returned only after D137-L's deep operation admits its complete declared synthetic prefix. - -## Cold-start reads - -- `memory/SPEC.md` — A49-L retirement clarification; D136-L; D137-L -- `memory/PLAN.md` — frontier `brownfield-comparison-cases` -- `src/dev/TOPOLOGY.md` — Historical Replay Isolation and Brownfield Comparison Oracles -- `src/dev/end-to-end-comparison/solution-isolation.ts` — current admission checks and one-commit mismatch -- `src/dev/end-to-end-comparison/pinned-source-preparation.ts` — current two-commit preparation and dependency install -- `src/dev/execution-comparison/operator-cli.ts` — current Petrinaut-only pinned dispatch -- `src/dev/end-to-end-comparison/{brunch-adapter,claude-adapter}.ts` — current greenfield execution launch adapters - -## Boundary Crossings - -```text -→ frozen case contract + lane + source/target/controller/forbidden roots -→ compile-time historical-replay profile resolution -→ pinned source root commit -→ exact packet-only handoff child commit -→ case-owned dependency preparation or explicit no-op -→ strict policy-pair and runtime-boundary admission -→ lane finalization -→ BrunchReady { baseSha, specId, launch } | ClaudeReady { baseSha, launch } -``` - -## Completion evidence - -| Leaf | Outcome | Evidence | -| ---- | ------- | -------- | -| One source root plus one packet-only child is admitted | met | `historical-replay-target.test.ts` known-good temporary Git fixture; `solution-isolation.test.ts` two-commit admission | -| `HEAD` and returned `baseSha` equal the handoff child | met | `historical-replay-target.test.ts` exact `HEAD`, parent, count, and packet-delta assertions | -| Wrong parent, packet drift, or third commit rejects before readiness | met | `historical-replay-target.test.ts` production-operation rivals retain admission reasons and remove the partial target | -| Brunch host landing performs no dependency install | met | self-contained Brunch production-operation test supplies a fail-if-called install runner and reaches readiness | -| Petrinaut selects only immutable Yarn install and rejects tracked mutation | met | self-contained synthetic pinned-Git tests assert exact `corepack yarn install --immutable --mode=skip-build`, admit its non-ignored untracked dependency artifact under D136-L, and fail tracked mutation in `dependency_preparation` | -| Strict policy/path/network/ref/symlink isolation prevents readiness | met | production-operation reachable-network, remote/ref, symlink, and forbidden-root rivals plus `solution-isolation.test.ts` weakened-policy, readable-root, tracked-worktree, network, and branded-verifier rivals | -| Pinned production callers cannot bypass the deep operation | met | `operator-cli.ts` dispatches every parsed `pinned_git` contract to `prepareHistoricalReplayTarget`; obsolete half-ready preparation module/export removed | -| Brunch returns positive brownfield Execute-plan-ready `specId` | met | self-contained target test queries the exact handoff requirement, projects brownfield Execute state, and passes `assertExecuteProjectionPlanReady` | -| Claude has no `specId` and carries strict launch policy | met | production dispatch test asserts the lane discriminator, strict policy, empty MCP/settings/plugins/network allowance, and absence of `specId` | -| Ready descriptor reaches execution while Petri remains unchanged | met-with-divergence | The Claude runner directly consumes the ready descriptor and retains output from the packet-child `baseSha`; Brunch readiness carries its adapter-produced launch descriptor; no automatic pinned actor-recipe dispatcher is implemented or claimed; greenfield `execution-adapters.test.ts` remains green | -| Linux CI can exercise the production operation without weakening production defaults | met | every deep-operation test injects a factory whose verifier is branded by `createNetworkDeniedCommandRunner`; the portability oracle executes the complete ready path with a process-independent sandbox fake, raw substitution rejects, and the default unsupported-host oracle remains fail-closed | -| Setup rejection is structured and leaves no launchable partial target | met | `HistoricalReplayTargetPreparationError` carries `status`, `phase`, and admission reasons; failure tests prove owned-target removal and pre-existing-target preservation | - -Portability red: `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts -t "injected branded verifier factory"` — 1 failed because the production operation ignored the injected factory (`expected 0 to be 1`). - -Portability green: the same command — 1 passed, 12 skipped by the name filter. The paired unsupported-host oracle also passed: `npm test -- src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts -t "fails closed when the host cannot provide"` — 1 passed, 3 skipped by the name filter. - -Focused green: `npm test -- src/dev/execution-comparison/__tests__/historical-replay-target.test.ts src/dev/execution-comparison/__tests__/operator-cli.test.ts src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts src/dev/end-to-end-comparison/__tests__/case-profile.test.ts src/dev/execution-comparison/__tests__/case-contract.test.ts src/dev/execution-comparison/__tests__/brunch-lane.test.ts` — 7 files, 43 tests passed. - -Checkpoint green: `npm run verify:full` — default 328 files/2,571 tests passed with 1 file/2 tests skipped; slow 9 files/68 tests passed; build passed. `npm run check` and `git diff --check` passed. Skipped-test delta versus `20709cdd`: 0 changed `.skip`/`.todo` markers. - -## Risks and Assumptions - -- **RISK:** changing `commitCount === 1` to `2` could admit arbitrary synthetic history. - - **MITIGATION:** validate the exact root/child relationship, source identity, packet-only child delta, `HEAD === baseSha`, and absence of any third commit. -- **RISK:** generalizing pinned dispatch could run Petrinaut's install for Brunch host landing. - - **MITIGATION:** an exhaustive code-owned map gives every pinned case id exactly one `none` or `petrinaut-yarn-immutable-v1` recipe with no default; the public contract never supplies commands. -- **RISK:** tests could pass only because the developer has a sibling HASH checkout. - - **MITIGATION:** the admission tracer must use self-contained temporary Git fixtures at the deep-module boundary; optional real-repository evidence cannot be the only oracle. -- **ASSUMPTION:** the exact-handoff child can remain independently attributable while the two commits are admitted as one closed synthetic prefix. - - **IMPACT IF FALSE:** D137-L's selected shape fails and preparation must collapse to one commit or adopt a different execution-base model. - - **VALIDATE:** one known-good two-commit fixture plus packet-drift, wrong-parent, and third-commit rivals through the production operation. - - **SPEC:** settled by D137-L; this slice is the falsifier. - -## Posture check - -This tracer scores on all three proving axes: - -- **Proof of life:** a prepared pinned target reaches a real lane launch descriptor only through production admission. -- **Invariant:** A49-L isolation is enforced over the final execution base rather than a pre-handoff intermediate. -- **Uncertainty:** contrastive Git-topology rivals can falsify D137-L before provider work. - -No spike is cheaper: the production slice itself is a temporary-repository experiment with no provider, browser, or external service. - -## Acceptance Criteria - -```text -historical replay target -├── declared prefix -│ ├── ✓ historical-replay-target.test.ts — one source root plus one packet-only child is admitted -│ ├── ✓ historical-replay-target.test.ts — HEAD and returned baseSha equal the handoff child -│ └── ✓ historical-replay-target.test.ts — wrong parent, packet drift, or any third commit fails before lane readiness -├── closed preparation -│ ├── ✓ historical-replay-target.test.ts — Brunch host landing performs no dependency install -│ └── ✓ historical-replay-target.test.ts — Petrinaut selects exactly corepack yarn install --immutable --mode=skip-build and rejects tracked mutation -├── strict isolation -│ ├── ✓ historical-replay-target.test.ts — weakened policy, readable forbidden root, reachable network probe, remote/ref, or escaping symlink prevents a ready result -│ └── ✓ production-call-site guard — no pinned production caller can bypass the deep operation for a half-ready workspace -├── lane finalization -│ ├── ✓ historical-replay-target.test.ts — Brunch returns a positive specId whose exact spec graph is brownfield and Execute-plan ready -│ ├── ✓ historical-replay-target.test.ts — Claude returns no specId and carries the strict launch policy -│ └── △ execution adapters — Claude directly consumes its ready descriptor and Brunch carries adapter-produced launch metadata; no automatic actor-recipe dispatcher is claimed; Petri greenfield remains unchanged -└── failure hygiene - └── ✓ historical-replay-target.test.ts — rejection retains structured phase/reason evidence and leaves no launchable partial target -``` - -## Invariants preserved - -- Exact approved `spec.md` and `public-contract.json` bytes cross unchanged — guarded by: existing handoff/public-packet tests plus `historical-replay-target.test.ts`. -- No historical source refs, remotes, solution services, controller roots, or case-private oracle material become target-reachable — guarded by: `solution-isolation.test.ts` and the new production-path rivals. -- Brunch and Claude retain their asymmetric strict policies; neither is reduced to a lowest-common-denominator sandbox — guarded by: `solution-isolation.test.ts` and launch-descriptor assertions. -- Brunch stops at `promotion_prepared` and never lands provider output into a source repository — guarded by: existing execution-adapter and no-landing suites. -- `ExecutionAttempt` remains unchanged — guarded by: existing end-to-end matrix/attempt tests. -- Minimal Petri remains the sole greenfield case and its preparation path remains byte-identical — guarded by: existing case-profile and execution-adapter tests. -- Stop the line: a red on exact handoff bytes, controller isolation, strict policy shape, or no-landing is a respec signal, not a fixture update. - -## Verification Approach - -- **Inner:** self-contained temporary-Git state-machine rivals through the public deep operation; focused case-policy, operator, adapter, and launch-descriptor tests. -- **Middle:** existing full solution-isolation adversarial suite plus pinned Brunch graph/preflight integration; no provider invocation. -- **Outer:** none for this infrastructure slice. FE-1241 owns the later fresh Petrinaut provider matrix and separately named host-integration evidence after this admission gate closes. -- **Checkpoint:** `npm run verify:full`, `npm run check`, `git diff --check`, and skipped-test delta versus `20709cdd`. - -## Cross-cutting obligations - -- D137-L exposes one deep operation, not a public phase machine or serializable security token. -- Dependency commands remain code-owned and case-closed; no manifest command/plugin escape hatch. -- Admission validates exactly one strict Claude/Brunch policy pair even though the returned descriptor is lane-specific. -- Setup rejection remains distinct from candidate assertion failure and happens before provider work. -- Failed/invalid evidence remains inspectable while incomplete targets cannot be launched. - -## Explicitly Out - -- Real Petrinaut provider runs, full HASH `/processes/draft`, and optional real optimizer evidence. -- Closing the separate review finding that the synthetic Petrinaut browser fixture does not prove the real pinned HASH route. -- General replay plugins, resumable persisted phase engines, arbitrary dependency recipes, or a third brownfield case. -- Refactoring the oracle registry or unrelated cleanup/retry infrastructure. - -## Expected touched paths (tentative) - -```text -memory/ -├── SPEC.md ~ -├── PLAN.md ~ -└── cards/brownfield-comparison-cases--historical-replay-target-admission.md -src/dev/ -├── TOPOLOGY.md ~ -├── end-to-end-comparison.ts ~ -├── end-to-end-comparison/ -│ ├── solution-isolation.ts ~ -│ ├── pinned-source-preparation.ts -|~ -│ ├── brunch-adapter.ts ~ -│ ├── claude-adapter.ts ~ -│ └── __tests__/ -│ ├── solution-isolation.test.ts ~ -│ └── execution-adapters.test.ts ~ -└── execution-comparison/ - ├── historical-replay-target.ts + - ├── historical-replay-target/ ? - ├── case-contract.ts ~ - ├── operator-cli.ts ~ - ├── brunch-lane.ts ~ - └── __tests__/ - ├── historical-replay-target.test.ts + - └── operator-cli.test.ts ~ -``` diff --git a/memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md b/memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md deleted file mode 100644 index b124dfb9c..000000000 --- a/memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md +++ /dev/null @@ -1,177 +0,0 @@ -# Petrinaut optimization brownfield case oracle - -Frontier: brownfield-comparison-cases -Status: done -Mode: single -Created: 2026-07-21 - -## Orientation - -- Containing seam: controller-owned brownfield case profiles, pinned-source preparation, compile-time oracle dispatch, and FE-1239 exact handoffs over unchanged FE-1230 attempt leaves. -- Frontier: `brownfield-comparison-cases` (FE-1241), on `ka/fe-1241-brownfield-comparison-cases`; this is another slice on the existing issue/branch, not a new frontier. -- Volatile state: A49-L isolation and the Brunch host-landing profile/oracle are complete. The Petri editor remains the sole greenfield case. Petrinaut packets, target preparation, and oracle code do not exist yet; no historical provider run is authorized. -- Spike verdict: A50-L is invalidated and D136-L is active. The full HASH `/processes/draft` Next shell is not the common hard-gate address; the focused standalone `/optimization` route is. The authenticated HASH host/iframe integration remains explicit non-gating outer evidence. - -Posture: proving (inherited from `brownfield-comparison-cases`). - -Cross-cutting obligations: - -- consume the A49-L history-free source identity and fail-closed Claude/Brunch policies without duplicating or weakening them; -- give every elicitation and execution cell a separate materialization of the same full HASH parent tree `a3e08cf75e00cc9016c931f4665341506e03533e` at commit `5c7a2d9db5caa851c38938f4b1bac19005b0e978`; -- preserve FE-1239 exact approved-spec bytes and FE-1230 `ExecutionAttempt` without schema widening; -- keep historical ids, URLs, merged refs, solution code, controller fixtures, expected requests/events, and oracle source outside target-visible packets and roots; -- retain `minimal-petri-net-editor` unchanged as the sole greenfield case and the completed Brunch case/oracle unchanged; -- judge public behavior, proxy/request provenance, and accessible rendered outcomes—never historical code, module placement, diff shape, or architecture. - -## Target Behavior - -A frozen Petrinaut brownfield case produces claim-linked controller verdicts from deterministic fake-optimizer browser journeys through the candidate's public `/optimization` route. - -## Cold-start reads - -- `memory/SPEC.md` — A50-L, D136-L; Brownfield assessment; §Verification Design rows for case identity, Petrinaut standalone browser/accessibility, and outer evidence; historical-replay boundary and blind spots -- `memory/PLAN.md` — frontier: `brownfield-comparison-cases` -- `src/dev/TOPOLOGY.md` — historical replay isolation, compiled oracle ownership, and public-root/private-subtree layout -- `docs/praxis/comparison-runs.md` — controller isolation, attempt validity, retention, and no-landing rules -- `src/dev/end-to-end-comparison/{study-contract,solution-isolation}.ts` — closed case identity and A49-L materialization/admission -- `src/dev/execution-comparison/{case-contract,operator-cli,oracle-pack,browser-oracle}.ts` and `src/dev/execution-comparison-operator.ts` — current profile, preparation, browser journey, report, and compile-time dispatch seams -- `testing/{end-to-end-comparisons,execution-comparisons}/cases/{minimal-petri-net-editor,brunch-host-landing}/` — immutable predecessor packet shapes; do not copy browser-specific fields into the backend case -- PR #9051 at parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978` and merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0` — bootstrap behavior/rivals only; no historical file is oracle authority or target-visible input -- Base-tree `@hashintel/petrinaut-core` optimization schemas — public pre-existing backend/API capability; do not credit it as UI implementation work - -## Boundary Crossings - -```text -content-addressed Petrinaut brownfield study - -> closed frontend profile + full HASH parent commit/tree - -> A49-L history-free materialization per lane - -> exact public specification/contract seed - -> controller runs corepack yarn install --immutable --mode=skip-build - -> tracked source remains clean; dependency result is retained outside ExecutionAttempt - -> network-denied execution lane starts and terminates at promotion_prepared - -> execution attempt is retained unchanged - -> controller runs focused package builds after lane termination - -> controller starts the candidate Petrinaut website - -> /optimization exposes the capability-present optimization UI - -> controller fakes optimizer responses behind the same-origin proxy - -> browser drives scenario, parameters, metric, stream, error, and cancel - -> proxy/request provenance + accessibility + rendered-state assertions - <- claim-linked report with setup_failed distinct from assertion_failed -``` - -## Risks and Assumptions - -- RISK: the oracle imports PR #9051 modules or requires their file layout and becomes an architecture matcher → MITIGATION: launch only declared candidate commands and observe browser roles/names, HTTP requests, aborts, and rendered outcomes; source guards scan the whole oracle subtree. -- RISK: the standalone route passes while the authenticated HASH host/iframe bridge is absent → MITIGATION: do not claim the common gate proves that bridge. Before the first provider matrix, the frontier owns one fully provisioned `/processes/draft` host/iframe smoke plus masked code review; unavailable evidence is `not_assessable`. -- RISK: target code can read fake events, expected manifests, wrong rivals, or historical solution material → MITIGATION: create controller fixtures after lane termination in disjoint roots and retain only aggregate claim results in audience-safe packets. -- RISK: browser/Vite startup or fixture import failure is mistaken for product failure → MITIGATION: report `setup_failed` separately with focused-build, process, route, and console evidence; assertion failures begin only after the declared app and fixture are ready. -- RISK: full HASH dependency preparation widens runtime network → MITIGATION: after materialization and exact handoff seeding, run only the compiled immutable Yarn install as a controller-owned pre-lane step, reject tracked-source mutation, then admit the target under A49-L; execution and verifier commands remain network-denied, while the post-lane oracle permits loopback only. -- RISK: visual polish is overclaimed by DOM assertions → MITIGATION: mechanical accessibility/interaction checks gate this card; masked qualitative visual review remains separately owned outer evidence. -- ASSUMPTION: the spike's focused preparation set—design-system codegen/build plus Petrinaut core, optimizer-client, and UI builds—is sufficient for history-free candidate checkouts without broad HASH codegen/toolchain work. - → IMPACT IF FALSE: candidate setup is invalid and the focused hard gate must be respecified before provider work; do not fall back to the broad 61-package Next-shell preparation. - → VALIDATE: operator preparation must record the exact immutable install before either lane, and the known-good slow fixture must execute only the closed post-lane focused builds before `/optimization` becomes ready. - -## Posture check - -- Proof of life: lights the first full-monorepo frontend case from pinned source identity through a focused public optimization route to a controller verdict. -- Invariant: establishes one compile-time third case/oracle variant without arbitrary commands, manifest paths, plugins, or historical references. -- Uncertainty: locks the spike-selected focused preparation boundary and proves it is sufficient in a history-free candidate checkout. - -## Acceptance Criteria - -```text -Petrinaut case/oracle acceptance -├── frozen profile -│ ├── case-profile.test.ts → exact mission/baseline/registry/contract/oracle hashes and parent commit/tree -│ ├── case-profile.test.ts → target-visible bytes omit PR/issue ids, URLs, merged ref, expected events, and solution wording -│ └── case-contract.test.ts → only petrinaut/brownfield/single_feature/frontend/pinned_git parses -├── same-base preparation -│ ├── operator-cli.test.ts → the real prepare path requires an explicit safe source and routes both lanes through pinned HASH materialization -│ ├── historical-replay-target.test.ts → the deep operation materializes the declared source identity plus exact packet-only child before readiness -│ ├── historical-replay-target.test.ts → exact immutable install is recorded before lane execution, tracked source stays clean, and declared untracked dependency artifacts remain admissible -│ └── historical-replay-target.test.ts + solution-isolation.test.ts → no third commit, remotes, later refs, tracked residue, or packet drift; source identity remains separate from implementation diff -├── closed oracle -│ ├── oracle-pack.test.ts → third compiled manifest, exact check ids, complete requirement assignment -│ ├── operator-oracle-dispatch.test.ts → known id selects compiled code; unknown id fails before launch -│ └── petrinaut-optimization-oracle.test.ts → no historical/candidate-internal imports or runtime implementation selectors -├── public browser behavior -│ ├── petrinaut-optimization-oracle.slow.test.ts → focused preparation launches `/optimization` with an accessible Optimizations view -│ ├── petrinaut-optimization-oracle.slow.test.ts → scenario is explicit and changing it resets configuration -│ ├── petrinaut-optimization-oracle.slow.test.ts → flat fixed/optimized bindings and one saved/custom metric direction reach the request -│ ├── petrinaut-optimization-oracle.slow.test.ts → progressive trials, best-so-far, completion, and service failure render distinctly -│ ├── petrinaut-optimization-oracle.slow.test.ts → cancel aborts the host request and renders cancelled -│ ├── petrinaut-optimization-oracle.slow.test.ts → browser traffic stays on the same-origin proxy; requests/DOM expose no private upstream origin -│ └── petrinaut-optimization-oracle.slow.test.ts → required tab/form/result/cancel controls are keyboard-reachable with stable roles/names -├── sensitivity -│ └── petrinaut-optimization-oracle.test.ts → missing-route, final-only, direct-private-origin, and UI-only-cancel rivals fail focused claims -└── regressions - ├── existing Petri + Brunch case/profile/oracle suites remain unchanged and green - ├── exact handoff, four-cell matrix, immutable attempt, isolation, cleanup, and no-landing suites remain green - └── npm run verify:full passes -``` - -## Invariants preserved - -- Historical code/diff/architecture similarity is never an oracle — guarded by: whole-subtree dependency/source guard and browser-only candidate interaction. -- Full `hashintel/hash` is the substrate; no Petrinaut library extraction masquerades as brownfield context — guarded by: parent commit/tree and same-base preparation tests. -- Target-visible material contains no controller fixture, hidden expected request/event, private optimizer origin, or historical-solution locator — guarded by: profile redaction/hash tests plus root-disjointness admission. -- Brunch comparison execution still stops at `promotion_prepared`; controller browser execution begins only after lane termination — guarded by: existing execution terminal suites and oracle report chronology. -- `ExecutionAttempt` remains unchanged and case/oracle identity stays beside it — guarded by: `artifact-contract.test.ts`, `execution-cell.test.ts`, and operator report tests. -- Petri remains the sole greenfield case and the completed Brunch profile/oracle remains byte-identical — guarded by: existing case/study/oracle packet hash tests and both predecessor slow suites. - -## Verification Approach - -- Inner: closed schema, packet hash, source identity, A49-L preparation, claim coverage, compile-time registry, path isolation, and focused wrong-rival tests. -- Middle: operator-owned pinned materialization and immutable dependency installation precede the sealed lane; controller-owned `/optimization` browser journeys run the closed focused builds after lane termination against a synthetic contract fixture using deterministic same-origin proxy responses, no live optimizer. -- Outer: owned by `brownfield-comparison-cases` before its first Petrinaut 2×2 provider run; trigger after this mechanical oracle is frozen and one candidate output exists, then run one fully provisioned `/processes/draft` host/iframe integration smoke, identity-masked visual/code review, and an optional real-optimizer adapter smoke. Outer findings are non-gating, are `not_assessable` when unavailable, and cannot repair a mechanical failure. - -## Expected touched paths (tentative) - -```text -testing/ -├── comparisons/missions/petrinaut-optimization.md + -├── end-to-end-comparisons/cases/petrinaut-optimization/ + -└── execution-comparisons/cases/petrinaut-optimization/ + -src/dev/ -├── TOPOLOGY.md ~ -├── end-to-end-comparison/ -│ ├── study-contract.ts ~ -│ └── __tests__/case-profile.test.ts ~ -├── execution-comparison-operator.ts ~ -└── execution-comparison/ - ├── case-contract.ts ~ - ├── operator-cli.ts ~ - ├── oracle-pack.ts ~ - ├── petrinaut-optimization-oracle.ts + - ├── petrinaut-optimization-oracle/ + - └── __tests__/ - ├── case-contract.test.ts ~ - ├── historical-replay-target.test.ts ~ - ├── oracle-pack.test.ts ~ - ├── operator-oracle-dispatch.test.ts ~ - ├── petrinaut-optimization-oracle.test.ts + - └── petrinaut-optimization-oracle.slow.test.ts + -memory/ -├── PLAN.md ~ -└── SPEC.md ? -``` - -## Completion evidence - -| Leaf | Outcome | Evidence | -| --- | --- | --- | -| Frozen profile | met | `case-profile.test.ts` and `case-contract.test.ts`; exact packet hashes, pinned HASH commit/tree, closed brownfield axes, and target-visible redaction | -| Same-base preparation | met | `operator-cli.test.ts`, `historical-replay-target.test.ts`, and `solution-isolation.test.ts`; production dispatch requires a safe explicit source and routes every pinned lane through the one deep operation, which preserves exact handoff bytes over the declared two-commit prefix, records only `corepack yarn install --immutable --mode=skip-build` for Petrinaut, permits non-ignored untracked dependency artifacts while rejecting tracked source/packet mutation, and retains no third commit, remotes, or later refs. The Brunch-ready branch returns a positive Execute-plan-ready brownfield `specId`; Claude readiness has no `specId`. | -| Closed oracle | met | `oracle-pack.test.ts`, `operator-oracle-dispatch.test.ts`, and `petrinaut-optimization-oracle.test.ts`; exact manifest claims, compile-time dispatch, immutable implementation pack, and source isolation | -| Public browser behavior | met | `petrinaut-optimization-oracle.slow.test.ts`; focused builds, standalone `/optimization`, scenario reset, fixed/optimized bindings, saved/custom objectives, progressive/best/completion/error, cancel/abort, same-origin secrecy, and accessible controls | -| Sensitivity | met | `petrinaut-optimization-oracle.test.ts`; missing-route, final-only, direct-private-origin, and UI-only-cancel rivals each fail their focused claim | -| Existing invariants | met | unchanged Petri/Brunch and exact-handoff/matrix/isolation/no-landing suites passed in the full repository gate; `ExecutionAttempt` remained unchanged | -| Full repository gate | met | `npm run verify:full`: default 328 files passed and 1 skipped, 2,562 tests passed and 2 skipped; slow 9 files/68 tests passed; TypeScript and production builds passed | - -The first full gate exposed two integration regressions: the operator case-list expectation still named two cases, and concurrent temporary Git cleanup could hit a transient macOS `ENOTEMPTY`. The closed list now includes Petrinaut and generated-root cleanup retries the transient removal without changing target behavior. - -Independent review then exposed that the low-level pinned-source proof was not reached by production operator preparation and that a fresh archive lacked dependencies. The corrected operator dispatch now makes the source checkout explicit, preserves exact packet bytes, records the closed controller-only immutable install before each lane, and leaves the existing post-lane focused-build/browser oracle—including required service-provider mode—unchanged. - -The interrupted pinned-Brunch follow-through is also closed: `seedBrownfieldBrunchExecutionWorkspace` seeds the already-materialized full repository without Petri-specific execution wording or tracked-source mutation, establishes the stored brownfield posture, and returns the `specId` consumed by the network-denied comparison TUI preflight. - -Skipped-test-count delta vs the preceding FE-1241 full gate: zero; both retained two skipped tests. The fully provisioned `/processes/draft` host/iframe smoke, masked review, and optional real-optimizer smoke remain explicitly non-gating outer evidence for the later historical provider-run scope. diff --git a/memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md b/memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md deleted file mode 100644 index b1f50e74a..000000000 --- a/memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md +++ /dev/null @@ -1,224 +0,0 @@ -# Prove the real Petrinaut provider preflight - -Frontier: brownfield-comparison-cases -Status: blocked -Mode: single -Created: 2026-07-22 - -Posture: proving (inherited from `brownfield-comparison-cases`). - -## Orientation - -- **Containing seam:** D136-L's real HASH preparation and standalone `/optimization` mechanical hard gate, behind D137-L's admitted lane-ready target. -- **Frontier:** FE-1241 `brownfield-comparison-cases`; this remains on `ka/fe-1241-brownfield-comparison-cases`. -- **Volatile state:** synthetic Petrinaut preparation/oracle proofs are green; no retained controller receipt proves the frozen identities, real immutable install, and real merged-reference route together. -- **Main risk:** interpreting a provider candidate failure when the real HASH setup or oracle calibration is itself invalid. - -## Target Behavior - -The controller emits a setup-valid Petrinaut preflight receipt only after the frozen real HASH parent and a disjoint controller-only merged reference pass their distinct preparation and calibration gates. - -## Cold-start reads - -- `memory/SPEC.md` — A49-L retirement clarification; D136-L; D137-L -- `memory/PLAN.md` — frontier `brownfield-comparison-cases` -- `src/dev/TOPOLOGY.md` — Historical Replay Isolation and Brownfield Comparison Oracles -- `memory/cards/brownfield-comparison-cases--admission-contract-hardening.md` — remaining provider-preflight debt -- `src/dev/execution-comparison/historical-replay-target.ts` — real parent preparation/admission -- `src/dev/execution-comparison-operator.ts` and `operator-cli.ts` — controller command surface -- `src/dev/execution-comparison/petrinaut-optimization-oracle.ts` and private subtree — focused builds/browser calibration -- `testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json` — frozen parent identity -- `testing/execution-comparisons/cases/petrinaut-optimization/controller/oracle-manifest.json` — hard-gate identity - -## Boundary Crossings - -```text -→ explicit safe real HASH source checkout -→ frozen parent commit/tree resolution -→ D137 lane-ready parent preparation with the real immutable install -→ disjoint controller-only merged-reference materialization -→ the same code-owned immutable install -→ closed Petrinaut focused-build/browser oracle -→ redacted write-once receipt + bounded hashed logs -→ cleanup of both owned workspaces -``` - -The parent workspace is the only future provider target. The merged-reference workspace calibrates controller setup and oracle sensitivity; it is always a forbidden root and is never returned as lane-ready input. - -## Risks and Assumptions - -- **RISK:** reference calibration leaks historical solution material into the provider target. - - **MITIGATION:** use disjoint roots, include the reference root in admission denial, retain no absolute path in the receipt, and scan the target/receipt for reference commit and path leakage. -- **RISK:** the heavy real install mutates tracked source or depends on ambient tooling. - - **MITIGATION:** reuse the one code-owned immutable recipe, retain bounded stdout/stderr digests, and fail before calibration/provider work on nonzero exit or tracked mutation. -- **RISK:** the current synthetic oracle does not match the real merged UI. - - **MITIGATION:** run the unchanged compile-time oracle against the controller-only merged reference; any setup or claim failure invalidates the preflight instead of weakening the oracle. -- **ASSUMPTION:** the real merged PR #9051 tree can satisfy D136-L's focused builds and standalone `/optimization` route under the immutable install. - - **IMPACT IF FALSE:** historical provider runs remain blocked and the Petrinaut oracle/profile must be respecified or corrected. - - **VALIDATE:** this slice's real no-provider-turn preflight is the falsifier. - -## Posture check - -This tracer scores on all proving axes: - -- **Proof of life:** the production parent preparation and real merged-reference browser oracle run through one controller command. -- **Invariant:** the historical reference remains controller-only while the parent target receives strict admission. -- **Uncertainty:** setup-validity is settled before provider budget or candidate interpretation. - -No separate spike is cheaper: the real preflight command is itself the minimum experiment and leaves a structured receipt. - -## Acceptance Criteria - -```text -Petrinaut real-source preflight -├── controller composition -│ ├── ✓ petrinaut-historical-preflight.test.ts — the closed case resolves the frozen parent and code-owned merged-reference commit -│ ├── ✓ operator-cli.test.ts — preflight requires absolute disjoint source/work/output roots and dispatches no arbitrary command -│ └── ✓ petrinaut-historical-preflight.test.ts — command trace contains no Claude or Brunch provider-lane launch -├── parent gate -│ ├── ✓ petrinaut-historical-preflight.test.ts — D137 returns a lane-ready parent with exact commit/tree, packet child, admission, and real-recipe result -│ └── ✓ petrinaut-historical-preflight.test.ts — parent setup failure yields setup_failed and no calibration -├── controller-only calibration -│ ├── ✓ petrinaut-historical-preflight.test.ts — reference materialization is disjoint and never returned as lane-ready -│ ├── ✓ petrinaut-historical-preflight.test.ts — install/build/oracle failure yields setup_failed rather than candidate assertion evidence -│ └── ✓ real preflight command — unchanged petrinaut-optimization-oracles-v1 passes against merged PR #9051 -├── retained evidence -│ ├── ✓ petrinaut-historical-preflight.test.ts — receipt is write-once, schema-validated, path-redacted, and hashes bounded install/oracle logs -│ └── ✓ petrinaut-historical-preflight.test.ts — receipt binds case, parent commit/tree, reference commit/tree, dependency recipe, oracle id/pack hash, and final setup status -└── isolation and cleanup - ├── ✓ petrinaut-historical-preflight.test.ts — target and receipt contain no reference commit/path or controller-private packet - └── ✓ petrinaut-historical-preflight.test.ts — pass and failure remove only owned parent/reference workspaces while retaining the receipt/evidence -``` - -## Invariants preserved - -- Historical reference material never becomes provider-target-visible — guarded by: receipt/target negative scans and D137 forbidden-root admission. -- Public packets continue to name only the parent source identity; merged-reference identity remains controller-owned — guarded by: existing case-profile redaction tests. -- Dependency and oracle selection remain compile-time closed — guarded by: TypeScript build and existing registry tests. -- Preflight evidence is not a serializable admission/security token and cannot authorize a lane — guarded by: receipt type and operator dispatch tests. -- Brunch stops at `promotion_prepared`; no source repository receives candidate output — guarded by: existing no-landing suites. -- Minimal Petri remains the sole greenfield case — guarded by: existing case-profile tests. - -## Verification Approach - -- **Inner:** self-contained temporary Git parent/reference fixtures with injected install/oracle runners prove ordering, receipt shape, redaction, no-provider trace, and cleanup. -- **Middle:** existing D137 admission and Petrinaut oracle suites remain unchanged; the preflight invokes those public operations rather than duplicating their assertions. -- **Outer:** one explicit real preflight command against the local `hashintel/hash` checkout. It must retain a setup-valid receipt before historical provider work can be scoped. -- **Checkpoint:** focused tests, `npm run verify:full`, `npm run check`, `git diff --check`, and skipped-test delta versus `origin/next`. - -## Cross-cutting obligations - -- The source checkout, controller root, parent target, reference calibration root, and receipt output root are explicit and pairwise safe. -- No env-gated skipped test may stand in for the real preflight witness. -- Install output is bounded and retained by digest; secrets, absolute paths, and full dependency logs do not enter the receipt. -- Setup invalidity remains separate from candidate assertion failure. -- The full `/processes/draft` host/iframe journey remains separately owned non-gating evidence. - -## Explicitly Out - -- Any Claude or Brunch provider turn, elicitation lane, 2×2 matrix, candidate interpretation, or promoted comparison report. -- The fully provisioned HASH `/processes/draft` host/iframe smoke. -- Arbitrary reference commits, dependency commands, oracle plugins, or manifest-selected shell behavior. -- Making the preflight receipt an admission capability or extending `ExecutionAttempt`. - -## Completion evidence - -The controller operation and CLI are built, but the scope is **not done**. The direct declared -`@hashintel/refractive` build closes the focused setup defect without generic graph preparation: the -diagnostic real no-provider run now passes both immutable installs, tracked-source checks, and all six -focused builds. Navigation readiness is no longer coupled to network idleness, and the unchanged -5-second semantic checks now expose a fixture/public-UI contract mismatch: the exact heading is absent -and the pinned historical action is `Create`, not `Create optimization`. No provider run may proceed -until the browser contract is respecified from the real public UI. - -| Leaf | Outcome | Evidence | -| ---- | ------- | -------- | -| Closed parent/reference identities | met | `petrinaut-historical-preflight.test.ts` proves fixed case selection and code-owned merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0`; the real receipt binds parent commit/tree | -| Absolute disjoint CLI roots; no arbitrary command/reference/plugin | met | `operator-cli.test.ts` — `Petrinaut preflight operator command` | -| No Claude/Brunch provider turn | met | `petrinaut-historical-preflight.test.ts` command trace contains controller preparation/calibration only | -| D137 lane-ready parent, exact identities, packet child, admission, recipe | met | focused preflight + existing `historical-replay-target.test.ts`; real parent install passed | -| Parent setup failure stops calibration | met | `retains setup_failed evidence and never calibrates after parent preparation fails` | -| Reference is disjoint/controller-only and never returned lane-ready | met | core preflight test checks forbidden-root admission, parent/reference separation, redacted receipt, and cleanup | -| Reference install/build/oracle setup failure remains setup_failed | met | install-stage, tracked-cleanliness, and oracle-preparation rivals retain distinct bounded evidence | -| Oracle assertion failure remains distinct from setup failure | met | `preserves the calibration assertion distinction from the unchanged oracle` | -| Unchanged real `petrinaut-optimization-oracles-v1` passes merged PR #9051 | **blocked** | all six focused steps pass; semantic navigation has no timeout or failed request, but the exact heading is absent and all downstream checks fail on the fixture-only `Create optimization` label | -| Write-once schema-validated redacted receipt with digests | met | receipt/evidence collision rivals plus failed-result digest round-trip; fixed relative filenames bind bounded parent/reference dependency and oracle-summary files | -| Receipt binds parent/reference identities, recipe, oracle pack/report, status | met | real invalid receipt binds both resolved identities, both passed recipes, oracle pack/report digests, setup status, evidence metadata, and cleanup | -| Parent/receipt leakage and controller-private packet exclusion | met-with-divergence | parent negative scans and receipt path-redaction pass; the receipt intentionally contains the reference commit/tree because the controlling requirement explicitly requires that binding, while containing no reference absolute path | -| Pass/failure clean only owned workspaces and retain receipt | met | parent is removed after reference-leak validation and before the reference install to bound scratch usage; real receipt records both workspaces removed | - -Diagnostic red: `npm test -- src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts -t "reference installation fails"` -failed because the receipt had no structured dependency stage or retained file. The oracle diagnostic -rival then failed because preparation command output was discarded. Green: parent/reference failures -cross D137 as structured observations; bounded path/secret-redacted files are retained after cleanup -with `wx`, and the oracle exposes only a narrow capture callback for its fixed preparation results. -Direct-fix red: `petrinaut-optimization-oracle.test.ts -t "Refractive workspace build"` failed because -the compile-time sequence omitted the declared workspace dependency. Green: the exact public sequence -contains `refractive-build` directly before UI, while the known-good slow fixture passes that sequence -and rejects omission and reordering. -Navigation red: the background-readiness slow rival failed all seven checks because `page.goto` -waited for `networkidle` under the 5-second action timeout. Green: the same DOM-ready candidate passes -all claims while a background request remains active for six seconds; `failedRequests` stays empty. - -Real evidence: - -- command: `npx tsx src/dev/execution-comparison-operator.ts petrinaut-preflight` with the real - `/Users/kostandin/Projects/hashdev/hash` checkout and explicit disjoint scratch roots -- receipt: - `/private/tmp/brunch-petrinaut-preflight-20260722T1558/evidence/petrinaut-historical-preflight-receipt.json` -- receipt SHA-256: `0ddbd4bffd4e57ce7426bc254bd8780e6d399c1eaf98f27499c261ac174a8046` -- final status: `assertion_failed`; setup status: `invalid`; exact stage: - `oracle_calibration` → historical semantic contract -- parent dependency evidence: `parent-dependency.json`, SHA-256 - `a9252a4448a87b9589efb4e4f1a7474d2540fdc87f80c28645ea3848804b1773`, 14,724 - bytes, not truncated -- reference dependency evidence: `reference-dependency.json`, SHA-256 - `813ac025c81b6b29c35bd296d41f2e98b4fb2dec13824998360713acd2d976b7`, 14,724 - bytes, not truncated -- oracle evidence: `oracle-summary.json`, SHA-256 - `39106c60d47d88699df2a92920ed46348ecb1e3efdbb61b05a7a80efe51468ae`, 22,011 - bytes, not truncated -- parent/reference: exact commit/tree matched; `corepack yarn install --immutable --mode=skip-build` - and tracked-source cleanliness passed for both -- oracle preparation: all six fixed steps passed; `refractive-build` exited 0 before - `petrinaut-ui-build` exited 0 -- browser calibration: `route-and-accessibility` reports `view: expected 1, received 0`; all six - downstream checks time out under the unchanged 5-second semantic budget waiting for the exact - `Create optimization` button -- real navigation contributes no timeout or `net::ERR_ABORTED`; `failedRequests` and `consoleErrors` - are empty -- pinned-source check: `OptimizationsView` renders `title="Optimizations"` but its button child is - `Create`; `/optimization` mounts `LocalStorageDemoApp` without a route-specific direct-view override -- no setup failure, provider turn, or browser claim pass -- cleanup: parent `removed`; reference `removed` - -Re-entry requires respecifying the browser oracle from pinned historical public semantics, including -the route's initial mode/view transition and actual accessible labels. Do not silently rename the -historical control, weaken semantic assertions, or replace this invalid receipt. - -Final verification after the semantic-contract witness: focused preflight/invariant run — 7 files / -54 tests passed; focused known-good/sensitivity/background-readiness run — 1 slow file / 4 tests -passed; `npm run verify:full` — 329 default files / 2,587 tests passed and 1 file / 2 tests skipped, -plus 9 slow files / 71 tests passed, then build passed; `npm run check`, `git diff --check`, and -touched-file lints passed. Skipped-test declarations are unchanged versus `origin/next` (1 → 1; -delta 0). - -## Expected touched paths (tentative) - -```text -memory/ -├── PLAN.md ~ -└── cards/brownfield-comparison-cases--petrinaut-provider-preflight.md -docs/praxis/comparison-runs.md ~ -src/dev/ -├── TOPOLOGY.md ~ -├── execution-comparison-operator.ts ~ -└── execution-comparison/ - ├── historical-replay-target.ts ? - ├── petrinaut-historical-preflight.ts + - ├── petrinaut-historical-preflight/ ? - ├── pinned-dependency-preparation.ts ? - └── __tests__/ - ├── operator-cli.test.ts ~ - └── petrinaut-historical-preflight.test.ts + -``` diff --git a/memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md b/memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md deleted file mode 100644 index e4b483c70..000000000 --- a/memory/cards/brownfield-comparison-cases--petrinaut-reference-fidelity.md +++ /dev/null @@ -1,203 +0,0 @@ -# Rebaseline Petrinaut reference fidelity - -Frontier: brownfield-comparison-cases -Status: active -Mode: single -Created: 2026-07-22 - -Posture: proving (inherited from `brownfield-comparison-cases`). - -## Orientation - -- **Containing seam:** D136-L's Petrinaut public packet, typed mechanical-address contract, controller-owned browser oracle, synthetic sensitivity fixture, and D137-L/D138-L real historical preflight. -- **Frontier:** FE-1241 `brownfield-comparison-cases`; this is the next slice on `ka/fe-1241-brownfield-comparison-cases`, not a new issue or branch. -- **Volatile state:** both real immutable installs, tracked-clean checks, all six focused builds, semantic route navigation, retained evidence, and cleanup pass without a provider turn; calibration stops because fixture-authored exact heading/action claims contradict merged PR #9051 behavior. -- **Main risk:** treating synthetic accessibility labels or implementation shape as source truth would either reject the known-good reference or weaken genuine scenario/request/progress/error/cancel/secrecy requirements. - -## Target Behavior - -The frozen Petrinaut comparison profile passes its no-provider merged-reference calibration under a source-faithful typed mechanical contract. - -## Cold-start reads - -- `memory/SPEC.md` — D70-L, D136-L, D137-L, D138-L; A50-L; relevant verification and historical-replay invariants -- `memory/PLAN.md` — frontier `brownfield-comparison-cases`, especially selected Petrinaut replay, acceptance, verification, cross-cutting obligations, and current execution pointer -- `memory/cards/brownfield-comparison-cases--petrinaut-provider-preflight.md` — retained invalid receipt, exact stop condition, completed setup evidence, and re-entry requirement -- `memory/cards/brownfield-comparison-cases--petrinaut-optimization-case-oracle.md` — original oracle contract, sensitivity intent, and inherited invariants -- `src/dev/TOPOLOGY.md` — Historical Replay Isolation and Brownfield Comparison Oracles -- `docs/praxis/comparison-runs.md` — controller isolation, validity, evidence retention, redaction, and no-landing rules -- `docs/praxis/manual-testing.md` — browser/accessibility observation, bounded evidence, findings ownership, and teardown discipline -- Linear `FE-1162` task body and [hashintel/hash PR #9051](https://github.com/hashintel/hash/pull/9051) title/body — source task semantics and UI-slice claims -- merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0` over parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978` — observable behavior only; never a golden diff, architecture oracle, or provider-visible source -- `testing/execution-comparisons/cases/petrinaut-optimization/{spec.md,public-contract.json,controller/oracle-manifest.json}` — current public packet and claim map -- `testing/end-to-end-comparisons/cases/petrinaut-optimization/{shared-baseline.md,study-contract.json,controller/requirement-registry.json}` — public baseline, hash chain, and requirement provenance -- `src/dev/execution-comparison/{case-contract.ts,oracle-pack.ts,petrinaut-optimization-oracle.ts}` and `petrinaut-optimization-oracle/{browser,claims,fixture,runner,types}.ts` — parser/type mismatch, compiled oracle, addresses, rivals, and evidence -- `src/dev/execution-comparison/{petrinaut-historical-preflight.ts,petrinaut-historical-preflight/evidence.ts}` — controller-only reference composition, retained receipt, redaction, and cleanup - -## Boundary Crossings - -```text -→ FE-1162 task body + PR #9051 body -→ controller-only disposable checkout of merged commit 276e17d7 -→ unchanged immutable install + closed focused builds -→ standalone /optimization route in a fresh browser -→ bounded DOM, accessibility-tree, request, stream, abort, and rendered-state evidence -→ source-fidelity map for route/view/navigation/drawer/control/progress/error/cancel semantics -→ corrected public spec, shared baseline, requirement provenance, and closed mechanical addresses -→ regenerated specification/public-packet/study/oracle content hashes -→ compiled browser oracle consuming one typed address per source-backed interaction -→ rewritten synthetic known-good plus focused omission/reordering/behavioral rivals -→ unchanged real no-provider preflight against the merged reference -→ write-once redacted receipt/evidence and cleanup of every owned workspace/process -``` - -The merged-reference workspace is controller-only and forbidden to every future lane. Audit evidence may describe observable behavior and source provenance, but no reference path, implementation file, diff, selector, or solution material may enter the public packet, synthetic fixture, provider target, or address contract. - -## Risks and Assumptions - -- **RISK:** exact labels, roles, or drawer/navigation steps are inferred from the synthetic fixture rather than the source task/reference. - - **MITIGATION:** capture a controller-only behavioral map from the task/PR body plus browser accessibility evidence first; delete unsupported stronger claims and permit exactly one source-backed role/name address per required interaction. -- **RISK:** rebaselining removes real behavior merely because the current merged UI is awkward. - - **MITIGATION:** preserve scenario-first configuration, flat fixed/optimized typed parameters, one saved/custom metric plus direction, same-origin request construction, progressive trial/best/completion, distinct error/cancel states, browser abort, and private-origin secrecy; only unsupported addressability/accessibility strength may change. -- **RISK:** the historical patch becomes a golden implementation or leaks into a provider lane. - - **MITIGATION:** inspect source only inside a disposable controller root; grade browser/API outcomes, never file names, component structure, diff shape, or architecture; retain negative leakage scans. -- **RISK:** a permissive locator ladder makes both the reference and defective rivals pass. - - **MITIGATION:** use a closed discriminated mechanical-address type with one exact role/name contract per source-backed action; prohibit locator alternatives, arbitrary CSS/XPath selectors, text fallbacks, and runtime-selected addresses. -- **RISK:** edited packet bytes leave stale transitive hashes or a receipt bound to the prior oracle pack. - - **MITIGATION:** regenerate every affected specification, public-contract, baseline, registry, study, manifest, packet, and implementation-pack digest through the existing loaders; tests must reject one-byte drift. -- **ASSUMPTION:** no Petrinaut provider attempt exists, so D138-L still permits rebaselining the frozen packet/oracle. - - **IMPACT IF FALSE:** packet and oracle bytes are immutable; this slice must stop without editing them and report the attempt evidence. - - **VALIDATE:** pre-edit attempt/evidence inventory plus preflight command trace proving zero Claude/Brunch provider launches. -- **ASSUMPTION:** merged PR #9051 satisfies its source task's observable UI/API behavior after unsupported stronger labels are removed. - - **IMPACT IF FALSE:** FE-1241 cannot use this merged commit as a known-good reference; provider work remains blocked and D136-L/the selected replay requires explicit respecification. - - **VALIDATE:** run the rebaselined compiled oracle unchanged against the merged reference; any deeper scenario, request, progress, error, cancel/abort, or secrecy mismatch is stop-the-line evidence, not permission to weaken the requirement. - -## D138-L posture check - -This slice exercises D138-L's one pre-attempt rebaseline window and scores on all proving axes: - -- **Proof of life:** the selected merged reference passes the same compiled public-behavior oracle intended for future candidates without consuming a provider turn. -- **Invariant:** source task/reference semantics become the sole authority for mechanical addresses while the synthetic fixture returns to its proper role as a sensitivity rival. -- **Uncertainty:** the slice determines whether PR #9051 is actually a valid known-good behavioral reference before FE-1241 spends provider budget. - -The rebaseline is permitted only because no attempt exists. It ends when the regenerated packet/oracle passes the merged reference and sensitivity rivals; all later provider attempts freeze those bytes again. If the reference misses a genuine source requirement, stop and report instead of converting the mismatch into a weaker contract. - -## Acceptance Criteria - -```text -Petrinaut reference-fidelity acceptance -├── source-fidelity audit -│ ├── ✓ retained source-fidelity evidence — FE-1162 and PR #9051 body claims map to observable route/view/navigation/drawer/control/progress/error/cancel/API semantics -│ ├── ✓ controller-only browser/AX capture — merged commit 276e17d7 is inspected from a disposable forbidden root with no provider launch or target-visible reference material -│ ├── ✓ source-fidelity review — every retained exact role/name and interaction step is witnessed by source behavior; unsupported fixture-authored strength is absent -│ └── ✓ stop-the-line review — any mismatch in scenario, bindings, metric/direction, request, progress/best/completion, error, cancel/abort, or private-origin behavior blocks the slice -├── typed contract and hash regeneration -│ ├── ✓ case-contract.test.ts — Petrinaut's interface and parser expose the same closed typed mechanical-address shape -│ ├── ✓ case-contract.test.ts — each required address has one exact source-backed role/name; alternatives, arbitrary selectors, extra keys, and missing keys fail closed -│ ├── ✓ case-profile.test.ts + loadPublicCasePacket — corrected spec/public baseline/registry/contract bytes retain the behavioral requirements and reject stale specification or packet hashes -│ └── ✓ study-contract/profile hash checks + loadControllerOraclePack — every changed transitive content hash and oracle-pack identity is regenerated; one-byte drift fails -├── oracle sensitivity -│ ├── ✓ petrinaut-optimization-oracle.slow.test.ts — the rewritten synthetic candidate implements the rebaselined public semantics and passes every compiled claim -│ ├── ✓ petrinaut-optimization-oracle.test.ts — source-backed mechanical addresses are consumed from the typed contract rather than duplicated as fixture-owned truth -│ ├── ✓ petrinaut-optimization-oracle.test.ts — focused omission rivals fail scenario, saved/custom metric, progress/best, error, cancel/abort, and private-origin claims -│ └── ✓ petrinaut-optimization-oracle.slow.test.ts — focused ordering/behavioral rivals fail when scenario gating/reset, preparation order, progressive state order, abort, or same-origin behavior is wrong -├── real-reference pass -│ ├── ✓ real petrinaut-preflight command — unchanged regenerated compiled oracle passes merged commit 276e17d7 after the fixed immutable install and six focused builds -│ ├── ✓ real oracle summary — all declared checks pass with no setup failure, assertion failure, console error, or unexpected failed request -│ └── ✓ petrinaut-historical-preflight.test.ts — any genuine source-behavior mismatch remains assertion_failed and cannot be relabeled setup-valid -├── cleanup and redaction -│ ├── ✓ petrinaut-historical-preflight.test.ts + real receipt — source, parent, reference, browser, server, and fake-optimizer roots/processes are clean or removed on pass and failure -│ ├── ✓ leakage scans — target, public packet, synthetic fixture, receipt, and retained evidence contain no reference path, source diff, implementation selector, secret, or controller-private packet -│ └── ✓ receipt/evidence contract — final write-once redacted receipt binds regenerated packet/oracle identities and bounded evidence hashes while retaining the successful reference verdict -└── no-provider proof - ├── ✓ petrinaut-historical-preflight.test.ts — command trace contains zero Claude/Brunch provider, elicitation, execution-lane, or matrix launches - ├── ✓ attempt inventory check — no Petrinaut ExecutionAttempt exists before or after calibration - └── ✓ repository gate — focused suites, npm run verify:full, npm run check, git diff --check, touched-file lints, and skipped-test delta pass with every owned root clean -``` - -## Invariants preserved - -- The merged reference is behavioral calibration evidence, never a golden diff or architecture oracle — guarded by: source-dependency scan, browser/API-only assertions, and retained source-fidelity review. -- No provider attempt may start before the merged reference passes the frozen no-provider calibration — guarded by: preflight status/receipt and command-trace tests. -- After the first provider attempt, packet/oracle bytes cannot be rebaselined — guarded by: pre-edit attempt inventory and D138-L stop condition. -- Public requirements still cover scenario-first/reset behavior, flat typed fixed/optimized bindings, one saved/custom metric with direction, same-origin request construction, progressive trials/best/completion, distinct error/cancel, upstream abort, and private-origin secrecy — guarded by: requirement registry, browser checks, captured request/abort evidence, and focused rivals. -- Mechanical addresses remain exact and typed without alternatives or arbitrary selectors — guarded by: contract parser/type tests and oracle source checks. -- Synthetic fixtures prove sensitivity only and contain no historical implementation knowledge or invented ground truth — guarded by: fixture/oracle source scan and contrastive rivals. -- Historical reference roots, controller packets, private optimizer origins, and retained raw evidence never become provider-visible — guarded by: D137-L forbidden-root admission, redaction tests, and negative leakage scans. -- Setup invalidity remains distinct from assertion failure; a deeper real-reference mismatch is a respec signal — guarded by: preflight stage/result tests. -- Minimal Petri remains the sole greenfield case; Brunch/Petri packets, oracles, attempts, and promoted witnesses remain unchanged — guarded by: predecessor profile/oracle regression suites. -- Neither source repository receives candidate output or a landed change — guarded by: existing `promotion_prepared` and no-landing suites. - -## Verification Approach - -- **Inner:** parser/type parity, exact-key mechanical-address validation, content/hash-chain regeneration, oracle source isolation, claim mapping, preflight failure classification, receipt/redaction, no-provider trace, and focused omission rivals. -- **Middle:** rewritten synthetic candidate plus focused ordering/behavioral rivals exercise the complete compiled browser oracle with deterministic same-origin optimizer responses; unchanged Petri/Brunch and D137-L admission suites remain green. -- **Outer:** a controller-only disposable merged-reference inspection captures bounded browser/accessibility/request evidence, then the real no-provider `petrinaut-preflight` command must pass the unchanged regenerated oracle and retain its final receipt/evidence. D138-L owns this outer gate in this card; no later provider matrix or host/iframe smoke substitutes for it. -- **Checkpoint:** focused tests while iterating; then `npm run verify:full`, `npm run check`, `git diff --check`, touched-file lints, skipped-test delta versus `origin/next`, process/workspace cleanup inspection, and source/target Git cleanliness. - -## Cross-cutting obligations - -- Preserve FE-1239 exact-byte handoff and FE-1230 immutable-attempt contracts without schema widening. -- Preserve D137-L's two-commit target admission, explicit forbidden roots, compiled dependency recipe, runtime network denial, and absence of arbitrary commands/plugins/selectors. -- Keep source issue/PR/reference material controller-only; public artifacts may state behavior but never historical identity, expected hidden values, implementation paths, or solution wording. -- Public baseline controls only source-backed common addressability and is never credited as elicitation gain. -- Regenerate hashes only for bytes changed by source contradiction; do not opportunistically rewrite unrelated frozen content. -- Retain invalid and successful preflight evidence separately; never replace the earlier assertion-failed receipt. -- Bound and redact browser/AX/request evidence; retain hashes and safe summaries rather than absolute paths, secrets, or full dependency logs. -- Reconcile D136-L, `src/dev/TOPOLOGY.md`, `memory/PLAN.md`, and this card's completion evidence to actual final truth. -- Leave the working repository, source checkout, disposable targets, browser/server processes, and all owned scratch roots clean. - -## Explicitly Out - -- Any Claude or Brunch provider turn, elicitation run, execution lane, 2×2 matrix, comparison report, scoring, winner claim, or reliability repetition. -- Weakening a genuine source behavior so the merged reference passes. -- Golden-diff, source-layout, component-name, module-boundary, CSS/XPath, screenshot-pixel, or architecture matching. -- Permissive locator alternatives, fallback label ladders, arbitrary selectors, runtime-selected addresses, or fixture-authored accessibility requirements. -- Changes to merged HASH/Petrinaut implementation, optimizer backend/API behavior, Brunch product runtime, or `ExecutionAttempt`. -- The fully provisioned HASH `/processes/draft` host/iframe smoke, masked visual/code review, optional real-optimizer smoke, or provider-campaign evidence. -- Replacing or mutating the retained failed preflight receipt. -- New cases, a second greenfield case, arbitrary oracle plugins, broad HASH build preparation, source landing, or PR publication. - -## Expected touched paths (tentative) - -```text -testing/ -├── execution-comparisons/cases/petrinaut-optimization/ -│ ├── spec.md ~ -│ ├── public-contract.json ~ -│ └── controller/oracle-manifest.json ? -└── end-to-end-comparisons/cases/petrinaut-optimization/ - ├── shared-baseline.md ~ - ├── study-contract.json ~ - └── controller/requirement-registry.json ~ -src/dev/ -├── TOPOLOGY.md ~ -├── end-to-end-comparison/__tests__/case-profile.test.ts ~ -└── execution-comparison/ - ├── case-contract.ts ~ - ├── oracle-pack.ts ? - ├── petrinaut-optimization-oracle.ts ? - ├── petrinaut-optimization-oracle/ - │ ├── browser.ts ~ - │ ├── claims.ts ~ - │ ├── fixture.ts ~ - │ ├── runner.ts ? - │ └── types.ts ? - ├── petrinaut-historical-preflight.ts ? - ├── petrinaut-historical-preflight/evidence.ts ? - └── __tests__/ - ├── case-contract.test.ts ~ - ├── operator-oracle-dispatch.test.ts ? - ├── petrinaut-optimization-oracle.test.ts ~ - ├── petrinaut-optimization-oracle.slow.test.ts ~ - └── petrinaut-historical-preflight.test.ts ~ -memory/ -├── PLAN.md ~ -├── SPEC.md ~ -└── cards/ - ├── brownfield-comparison-cases--petrinaut-reference-fidelity.md ~ - └── brownfield-comparison-cases--petrinaut-provider-preflight.md ~ -docs/praxis/comparison-runs.md ? -``` - -Controller-only disposable workspaces and final preflight evidence live under explicit external scratch/output roots. They are runtime evidence, not additional repository planning files or target-visible fixtures. diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index deaf70d95..45cddc9f6 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -29,7 +29,7 @@ It does not own published CLI behavior, public RPC contracts, database imports f `execution-comparison/host-landing-oracle.ts` is the public Brunch controller entry point; its same-named private subtree owns disposable Git fixtures, settled-session public-TUI actuation, the independent Git outcome model, and report types. The oracle resumes only controller-supplied settled sessions under `PI_OFFLINE=1`, invokes `/brunch:land` through the built candidate, and distinguishes setup invalidity from claim failure. -Petrinaut preparation selected by D136-L is split across the historical-replay operation and oracle. `execution-comparison/operator-cli.ts` resolves every `pinned_git` contract into the deep operation; that operation materializes the frozen full HASH tree and exact handoff per lane, then runs the one compiled immutable install (`corepack yarn install --immutable --mode=skip-build`) before the network-denied lane. Install failure or tracked-source mutation removes only the newly owned target. The pinned Brunch branch then seeds the exact specification as one opaque settled requirement, establishes brownfield posture, and returns the `specId` consumed by `execution-comparison-brunch.ts`; Claude receives the same source and handoff without Brunch graph state. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against the standalone `/optimization` route. The compiled sequence includes the directly declared `@hashintel/refractive` workspace build after design-system/core/optimizer prerequisites and immediately before Petrinaut UI; the synthetic candidate rejects omission or reordering rather than discovering a generic workspace graph. Controller HTTP reachability remains a separate setup check. Fresh browser pages use bounded `domcontentloaded` navigation rather than network-idle coupling, then retain the 5-second semantic locator/action budget; a synthetic route with more than five seconds of background traffic proves readiness without spurious aborted-request evidence. The real merged reference now passes every focused build and reaches browser semantics, where the fixture-authored heading/control contract diverges from the pinned historical public UI. The browser sees only the candidate origin and same-origin proxy, while controller fixtures and the private optimizer origin remain outside target-visible roots. The fully provisioned `/processes/draft` host/iframe journey is non-gating outer evidence and does not alter this mechanical hard gate. +Petrinaut preparation selected by D136-L is split across the historical-replay operation and oracle. `execution-comparison/operator-cli.ts` resolves every `pinned_git` contract into the deep operation; that operation materializes the frozen full HASH tree and exact handoff per lane, then runs the one compiled immutable install (`corepack yarn install --immutable --mode=skip-build`) before the network-denied lane. Install failure or tracked-source mutation removes only the newly owned target. The pinned Brunch branch then seeds the exact specification as one opaque settled requirement, establishes brownfield posture, and returns the `specId` consumed by `execution-comparison-brunch.ts`; Claude receives the same source and handoff without Brunch graph state. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against the standalone `/optimization` route. The compiled sequence includes the directly declared `@hashintel/refractive` workspace build after design-system/core/optimizer prerequisites and immediately before Petrinaut UI; the synthetic candidate rejects omission or reordering rather than discovering a generic workspace graph. Controller HTTP reachability remains a separate setup check. Fresh browser pages use bounded `domcontentloaded` navigation rather than network-idle coupling, then retain the 5-second semantic locator/action budget. The typed mechanical addresses and synthetic rivals are rebaselined to source-backed PR #9051 semantics; they no longer treat fixture-authored labels or keyboard-focus strength as reference truth. No post-rebaseline real-reference pass is retained, so D138-L still blocks a first provider attempt until the controller preflight succeeds. The browser sees only the candidate origin and same-origin proxy, while controller fixtures and the private optimizer origin remain outside target-visible roots. The fully provisioned `/processes/draft` host/iframe journey is non-gating outer evidence and does not alter this mechanical hard gate. `execution-comparison/petrinaut-historical-preflight.ts` is the controller-only, no-provider-turn calibration composition for that seam. It prepares and admits a disposable parent target through @@ -41,7 +41,9 @@ The schema-validated receipt names only those safe relative files plus their has truncation state. The admitted parent is removed after reference-leak validation and before reference installation so two large dependency trees do not coexist. The receipt is not lane admission authority and no reference workspace or launch descriptor is returned. A failed real calibration remains setup -evidence rather than permission to weaken the recipe or oracle. +evidence rather than permission to weaken the recipe or oracle. The implementation includes this +composition and deterministic source-fidelity contract without retaining a post-rebaseline real run; +that explicit waiver closes no D138-L gate and authorizes no provider lane. `execution-comparison-operator.ts` dispatches both brownfield oracles and the retained Petri browser oracle through a closed compile-time registry and includes every implementation module in the immutable oracle-pack hash; manifests cannot select paths, commands, or plugins at runtime. From 650b35482a871b3679ab8094ea91a49a18e0dc0f Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 23:14:41 +0200 Subject: [PATCH 12/15] FE-1241: Archive completed brownfield frontier Move the waived provider evidence to an explicit Later gate and keep only current D136-D138 topology in live canonical memory. Co-authored-by: Cursor --- docs/archive/PLAN_HISTORY.md | 10 ++++++++++ memory/PLAN.md | 29 +++-------------------------- memory/SPEC.md | 10 +++++----- src/dev/TOPOLOGY.md | 4 ++-- 4 files changed, 20 insertions(+), 33 deletions(-) diff --git a/docs/archive/PLAN_HISTORY.md b/docs/archive/PLAN_HISTORY.md index 240b6e120..b55201737 100644 --- a/docs/archive/PLAN_HISTORY.md +++ b/docs/archive/PLAN_HISTORY.md @@ -3,6 +3,16 @@ This file is the active POC-line plan archive for `memory/PLAN.md`. Legacy pre-`next` history was moved out of the live docs tree with the old archived implementation. +## 2026-07-22 FE-1241 brownfield comparison closeout + +`brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-isolated-brownfield-comparison-cases), [PR #362](https://github.com/hashintel/brunch/pull/362)) completed the mechanical expansion from the sole greenfield Petri case to frozen Brunch backend and Petrinaut frontend replays. The branch is `ka/fe-1241-brownfield-comparison-cases`, based on `next` after FE-1240 landed. + +The frontier added content-addressed mission/baseline/registry/execution/oracle packets, exact packet handoff, a two-commit historical target (source-materialization root plus packet-only child), exhaustive code-owned dependency recipes, strict Claude/Brunch admission, pinned Brunch graph seeding, and compile-time oracle dispatch without widening `ExecutionAttempt`. Brunch replays FE-1201/PR #336 host landing at parent `f5a423b19f76cf345d88053456870a126e451618`; Petrinaut replays PR #9051 optimization UI in the full HASH tree at parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978`. Historical patches bootstrap claims and rivals but are never golden diffs or target-visible material. + +D137-L is the sole pinned preparation/admission path. D136-L permits only the compiled immutable Petrinaut install before runtime denial and the closed focused builds after lane termination. Brunch's public `/brunch:land` journey is judged by an independent full-range Git model; Petrinaut's standalone `/optimization` journey is judged through a same-origin deterministic optimizer, typed source-backed mechanical addresses, and contrastive rivals. Petri remains the only greenfield case; Clay, arbitrary commands/plugins, source landing, aggregate winners, repetitions, and `ExecutionAttempt` schema changes remain out. + +The real post-rebaseline Petrinaut merged-reference preflight and historical 2×2 provider matrix were explicitly waived for FE-1241. No passing reference receipt or provider-campaign evidence is claimed. Before any future first Petrinaut provider attempt, PLAN Later item `petrinaut-reference-fidelity-campaign` must satisfy D138-L with a passing controller-only merged-reference receipt, then receive separate authorization for provider work. Current implementation state lives in [`src/dev/TOPOLOGY.md`](../../src/dev/TOPOLOGY.md); operational preflight guidance lives in [`docs/praxis/comparison-runs.md`](../praxis/comparison-runs.md). + ## 2026-07-22 Executor Petri sequence reconciliation `executor-slice-attempt-lifecycle` (FE-1192) merged in #324 on 2026-07-13. `petri-execution-parity` (FE-1195) then absorbed and completed the planned isolation/fan-in, durable parallel authority, Petrinaut-visible attempt topology, and epic integration sequence in #325. Their stale active/future entries and dependency chain were retired from `memory/PLAN.md`; current execution ownership and lifecycle semantics live in `src/executor/TOPOLOGY.md` under D127-L–D129-L. diff --git a/memory/PLAN.md b/memory/PLAN.md index d53e9db25..a80cb106e 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -84,7 +84,7 @@ Close the entire first batch of walkthrough-related findings: remediation, the o ### Recently Completed -- 2026-07-22 `brownfield-comparison-cases` (FE-1241) — **✓ implementation complete:** frozen Brunch and Petrinaut brownfield packets, the two-commit lane-ready replay boundary, compile-time preparation/oracle registries, deterministic sensitivity rivals, and portable CI are built. The real post-rebaseline Petrinaut preflight/provider campaign was explicitly not run; D138-L remains its re-entry gate. +- 2026-07-22 `brownfield-comparison-cases` (FE-1241) — **✓ complete (provider campaign waived):** frozen Brunch and Petrinaut brownfield packets, the two-commit lane-ready replay boundary, compile-time preparation/oracle registries, deterministic sensitivity rivals, and portable CI are built. The real post-rebaseline Petrinaut preflight/provider campaign was explicitly not run; D138-L remains a Later re-entry gate. - 2026-07-21 `executor-slice-admission-parity` (FE-1240) — **✓ complete:** candidate admission now rejects scoped slices without executable criterion, design, or verification-machinery context; exact findings enter bounded repair, every admitted repaired slice survives preview/worker-context parsing, and the populated-plan execution guard remains fail-closed. - 2026-07-20 `comparison-reporting-skills` (FE-1232) — **✓ implementation complete, building on landed FE-1230:** added separate project-shared Notion publication and comparison-evidence reporting skills; elicitation, execution, end-to-end, and frozen campaign-strategy references; and executable guardrails for active-procedure precedence, safe mutation, validity-first interpretation, reproducible judging, failure retention, and audience-safe controller-only redaction. Older completion history (including FE-1192/FE-1195 executor topology closure): [`docs/archive/PLAN_HISTORY.md`](../docs/archive/PLAN_HISTORY.md). @@ -119,7 +119,6 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand - `executor-slice-admission-parity` ([FE-1240](https://linear.app/hash/issue/FE-1240/prevent-invalid-scoped-slices-from-reaching-execution)) — **complete on `ka/fe-1240-slice-admission-parity`, restacked on `next` after FE-1239 landed:** incomplete scoped-slice worker context is rejected during deterministic plan admission, exact findings enter bounded repair before `slice_execute`, and the execution boundary remains fail-closed. Definition below. - `executor-plan-coherence` ([FE-1250](https://linear.app/hash/issue/FE-1250/build-coherent-execution-plans)) — **implementation complete on `ka/fe-1250-coherent-execution-plans`, based on FE-1240 via `next`:** frontier-verified multi-slice epics now require one ordinary terminal member over every sibling, planner conduct establishes shared foundations, and workers preserve cumulative public contracts for the canonical harness. Fast verification passes; next is the separately authorized unchanged Petri comparison rerun. No new browser gate, durable plan kind, or executor lifecycle phase. Definition below. - `comparison-publication-workflow` ([FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports)) — **active on `ka/fe-1251-comparison-publication`, stacked on FE-1250:** capture immutable controller and release provenance before comparison lanes start, then explicitly publish retained validity-first reports into the canonical Notion database through a guarded idempotent skill. Definition below. -- `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases)) — **implementation complete on `ka/fe-1241-brownfield-comparison-cases`, based on `next` after FE-1240 landed:** both frozen brownfield profiles/oracles and D137-L's admitted two-commit replay path are built and contract-hardened; pinned recipes are exhaustive and the deterministic suite is green. No active scope remains. Any future Petrinaut provider attempt re-enters through D138-L's real merged-reference preflight. - **Carved from FE-1167 (2026-07-13):** the Execute-mode evidence sub-list — Execute entry beats on thin vs rich seeds (assessment honesty: Ask on thin, Proceed on rich) and the FE-1107/KA residue (close-or-narrow, demo/walkthrough session via `TESTING_PLAN.md`, post-KA plan pass). The former sticky-posture question is no longer KA residue: FE-1187 `remediation-4` owns the persistent Specify elicitation-style audit/SPEC revision, and its Continue lexical audit owns the old `continue` ambiguity. Full context in the archived FE-1167 definition (`docs/archive/PLAN_HISTORY.md`). - `planning-process-model` — **moved to the KA stream 2026-07-13; reshaped by D126-L**: the durable scope handoff is settled, so this item now owns only plan projection and epistemic-horizon questions beyond committed scopes. Definition below. - **[1.x data-model handoff owed by FE-1187](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md):** a concise colleague-facing note must distinguish current canon from directional vocabulary before the consolidated outer checkpoint. Current: `{milestone, frontier, scope}` with executor-derived slices; basis (`explicit | implicit`) orthogonal to settlement (`advisory | settled`); no persisted readiness grade or spec-global elicitation-gap table; active-branch Pi JSONL reads; one CommandExecutor mutation authority with spec-local LSN/change log; no new projected `vv_obligation` (legacy rows remain readable). Directional only: explain stored `thesis` as pitch/concept without renaming it yet, and avoid new coupling to spec-local `term` while its possible workspace lift remains future work. Persisted judgment-shaped reconciliation needs remain current, but are YAGNI-suspect: add no kinds/consumers/orchestration dependency without fresh evidence; re-evaluate derivation/removal when KA work first needs that table. Delivered note: [`Brunch 1.x data-model handoff`](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md); it does not block the deterministic TUI queue. @@ -128,6 +127,7 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand ### Later Instrumentation experiments and far-horizon items. Each re-enters only via re-qualification with a named trigger. +- `petrinaut-reference-fidelity-campaign` — **Later, evidence-gated by D138-L:** before any first Petrinaut historical provider attempt, rerun the controller-only merged-reference preflight against PR #9051 and require a passing receipt from the frozen source-backed packet/oracle. Only after that pass may a separately authorized 2×2 provider matrix be scoped. The real post-rebaseline run and provider campaign were waived for FE-1241; this row does not reopen that completed frontier. - `tier-2-regression-probes` — **Later, trigger-gated**: runnable probes over the tier-2 real-boot faux-provider harness that track improvement/regression of Brunch's own conduct on rich seed scenarios (the intra-product lane's mechanical oracle). Re-enter when the seed/fixture library demonstrates rich, relevant, challenging scenarios worth pinning; artifacts follow the normal probe contract (`docs/architecture/probes-and-transcripts.md`). - `mechanism-trace` — **Later**: post-hoc `wiring` / `nudge` / `conduct` transcript timeline plus static wiring inventory. Re-enter when instrumentation is prioritized; FE-1187 already owns the extracted sweep-debt tripwire. Archived snapshot: [`docs/archive/PLAN_HISTORY.md`](../docs/archive/PLAN_HISTORY.md#2026-07-14-rolling-frontier-reduction). - `agent-tracing` — passive trace instrumentation over Pi lifecycle events for debugging plus conduct/quality evaluation: NDJSON emitter extension (introspection-tap discipline), subagent span joining via SDK `session.subscribe`, and a mechanical-trace × semantic-JSONL join for deterministic conduct checks and judged passes. Entry move is an `ln-spike` (dev-gated `nikiforovall/pi-otel` import: do span trees beat `.brunch/debug/` + JSONL projections?) before any port of `JoshMock/the-agency` observability as the in-product base. Traces are dev/eval artifacts, never product truth (no event-spine backdoor). Design: `docs/design/AGENT_TRACING.md`; sibling idea note `docs/design/RLM_INVESTIGATION_PATTERN.md`. Relation: Later `mechanism-trace` is the transcript-native sibling (carrier classification, no event plane); if both land they may join on a shared trace vocabulary. Absorbs Pi-native P5 (provider/cache observability — latency, cache behavior, whole-run spans), spike-led. @@ -457,29 +457,6 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Boundary:** no hidden-oracle exposure, inferred browser command, new plan-plane node, candidate command surface, execution-time plan repair, or FE-1241 comparison-framework change. - **Current execution pointer:** none; the two-card scope is consumed. Re-enter only for the unchanged frozen Petri comparison rerun. -### brownfield-comparison-cases - -- **Name:** Add Brunch and Petrinaut brownfield end-to-end comparison cases -- **Linear / branch:** [FE-1241](https://linear.app/hash/issue/FE-1241/add-brunch-and-petrinaut-brownfield-end-to-end-comparison-cases), child of FE-1211; `ka/fe-1241-brownfield-comparison-cases`, based on `next` after FE-1240 landed. -- **Kind:** structural evaluation expansion — parameterized case/substrate/oracle composition plus two repository-backed end-to-end cases; one frontier with several scoped slices, not one frontier per case. -- **Certainty:** proving. -- **Status:** implementation complete 2026-07-22; PR #362 is conflict-free and its full CI, Bugbot, and Semgrep gates pass. D137-L is the sole pinned preparation path, both frozen case/oracle stacks are built, and Petrinaut's public contract was rebaselined from fixture-authored labels to source-backed semantics before any provider attempt. The post-rebaseline real merged-reference preflight was explicitly waived, so no passing reference receipt or provider-campaign evidence is claimed. D138-L remains the mandatory re-entry gate before a first Petrinaut provider attempt. -- **Objective:** prove that FE-1239's exact elicitation → immutable handoff → crossed Brunch/Claude execution → requirement-ledger path works against real pinned brownfield repositories without weakening controller isolation or turning the Petri tracer into generic campaign machinery. -- **Lights up:** pinned repository + private feature mission + shared public baseline → two codebase-aware elicitation lanes → two exact handoffs applied to the same base tree → four isolated execution cells → repository-specific controller oracle → validity-first requirement ledger, once for Brunch and once for Petrinaut. -- **Stabilizes:** one data-driven case profile across `greenfield` and `brownfield`; immutable repository/base-tree identity beside immutable spec identity; clean same-base materialization for every lane; an exhaustive code-owned dependency recipe per pinned case, including the closed controller-only immutable install before runtime denial; pinned Brunch graph seeding and launch identity without tracked-source mutation; compile-time oracle dispatch; and diff/evidence collection that excludes harness seed files without widening FE-1230's `ExecutionAttempt`. -- **Selected replay — Brunch backend:** source FE-1201 / [PR #336](https://github.com/hashintel/brunch/pull/336), base `f5a423b19f76cf345d88053456870a126e451618`, merged reference `0092a5498d22603eeb22529e8823b365ff59b505`. Derive only the host-landing backend behavior and its temp-repository safety contract; TUI/RPC/web presentation changes in the historical PR are outside the mission, so the whole reference diff is not a golden patch. -- **Selected replay — Petrinaut frontend:** source FE-1162 UI slice / [PR #9051](https://github.com/hashintel/hash/pull/9051), full `hashintel/hash` base `5c7a2d9db5caa851c38938f4b1bac19005b0e978`, merged reference `276e17d7b0f80c8a80d5abe01849bbb67c6169d0`. Replay only the optimization UI; optimizer backend/API capabilities already present at the base commit are shared public baseline, not implementation work or elicitation gain. -- **A49-L retired (2026-07-21), clarified by D137-L (2026-07-22):** the live Claude probes established provider continuity while arbitrary target network and forbidden-root reads failed. Historical isolation excludes source-history reachability; it does not require a literal one-commit execution base. The declared synthetic prefix is one source-materialization root plus one packet-only exact-handoff child, with no tracked worktree mutation, extra commits, remotes, or refs; D136-L dependency artifacts may remain untracked after controller preparation. Admission rejects target/symlink escape, packet drift, missing/weakened policy, reachable required network probe, unbranded verifier, or unsupported host. Claude uses strict empty MCP, explicit non-web tools, `dontAsk` plus an empty network allowlist, canonical forbidden-root denial, empty ambient setting sources/plugin settings, and `failIfUnavailable`; Brunch removes web/research/Specify-subagent surfaces, retains planner/worker, rejects lexical and symlink escape from foreground/child file tools, and injects a verifier that denies runtime network plus sealed-root reads/writes. -- **Retired:** A49-L; later historical case launches must consume this admission path and may not fall back to prompt-only conduct. -- **Why now / unlocks:** FE-1239 proved the full chain only on an empty greenfield browser app, and FE-1240 closes the slice-admission defect that stopped both Brunch execution cells. The next load-bearing unknown is whether the same contracts survive repository inspection, pinned pre-existing code, and feature-delta verification. -- **Boundary:** retain `minimal-petri-net-editor` unchanged as the sole `greenfield / whole_application / frontend` case; add exactly the derived FE-1201 host-landing slice as `brunch / brownfield / single_feature / backend` and PR #9051 as `petrinaut / brownfield / single_feature / frontend`. Petrinaut receives the full pinned HASH monorepo, not an extracted library fixture. The framework may parameterize case id, frozen axes, repository substrate, delivery contract, and sealed oracle id, but it must not accept arbitrary manifest shell commands or make controller material or historical solution refs target-visible. -- **Acceptance:** the existing Petri case, hashes, browser-oracle behavior, and promoted witness remain valid; both brownfield cases freeze content-addressed mission/baseline/registry/execution/oracle packets; every lane-ready target starts from a separate clean checkout of the declared base tree with no later refs/remotes or reachable historical-solution services; exact approved spec and contract bytes cross unchanged; pinned cases require an explicit safe controller source and record only their compiled dependency recipe without tracked-source mutation before runtime network denial; pinned Brunch preparation returns an Execute-plan-ready `specId`; harness seed commits remain distinct from implementation diffs; controller roots remain unreachable; compile-time registries dispatch every recipe and oracle; known-good fixtures and focused rivals prove deterministic sensitivity. A future provider campaign, if authorized, owns the two elicitation handoffs/four execution attempts, complete cleanup, and requirement evidence; Petrinaut cannot enter it until D138-L passes. -- **Verification:** per `memory/SPEC.md` §Verification Design. Inner — case-profile, base/tree identity, materialization, strict solution isolation, lexical/canonical/symlink path isolation, oracle dispatch, and handoff/matrix contracts. Middle — unchanged Petri regression; Brunch black-box `/brunch:land` TUI journeys over disposable repositories plus an independent full-range Git model; Petrinaut deterministic fake-provider browser/accessibility journeys and focused behavioral rivals. Outer — no FE-1241 provider campaign was run. A future authorized campaign must first produce a passing D138-L real-reference receipt; optional real-optimizer, fully provisioned host/iframe, and masked visual/code review remain non-gating. -- **Cross-cutting obligations:** public baseline controls only addressability needed by the common oracle and is never credited as elicitation gain; existing repository knowledge may be inspected equally by both elicitation lanes; private mission/reveal/oracle files never enter target workspaces; failed and invalid attempts remain retained; Brunch stops at `promotion_prepared` and never lands into either source repository; provider/model/budget/intervention rules remain frozen per study. -- **Explicitly out:** Clay/Pi Campaign Machine; any second greenfield case; whole-repository rewrites; evolving-plan studies; reliability repetitions; automatic scoring or cross-case winner; live production deployment; Cursor/Codex lanes; arbitrary oracle plugins; production `/compare-end-to-end`; landing provider outputs into either source repository; and FE-1230 `ExecutionAttempt` schema widening. -- **Traceability:** D70-L fixture taxonomy; D134-L/I67-L controller/mission isolation; D40-L/D120-L/I62-L execution and no-landing boundaries; FE-1210/FE-1230/FE-1232/FE-1239/FE-1240; [`docs/praxis/comparison-runs.md`](../docs/praxis/comparison-runs.md); Notion `brunch Test Plan` axes. -- **Current execution pointer:** none. All tracked scope files are consumed. Re-enter only if a Petrinaut provider attempt is authorized; D138-L then requires the real merged-reference preflight to pass before any lane starts. - ### comparison-reporting-skills - **Name:** Report comparison evidence @@ -669,7 +646,7 @@ KA stream: reuses: canonical harness | isolated integration | epic verification excludes: browser-specific executor gate | durable plan kind | new lifecycle phase -[hard]-> brownfield-comparison-cases - status: implementation complete FE-1241; no provider campaign run; future Petrinaut attempts gate on D138-L + status: complete FE-1241; real provider campaign waived; D138-L residual moved to Later lights_up: same-base brownfield elicitation -> exact handoff -> crossed execution -> case oracle cases: minimal-petri-net-editor = sole greenfield reference | FE-1201-derived brunch backend | PR #9051 petrinaut frontend excludes: Clay | more greenfields | repetitions | aggregate winner | ExecutionAttempt widening diff --git a/memory/SPEC.md b/memory/SPEC.md index 887ad64ab..63da61083 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -127,7 +127,7 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | A46-L | A closed Zod schema can represent every `SessionPresentationDelta`, including `OpenAsk` questions with questionnaire questions present or exactly absent. **Validated 2026-07-15:** the contract composes the owned question schemas as exact alternatives and round-trips all delta variants without a cast or restated owner type. | medium | validated | D133-L; `src/rpc/__tests__/standalone-web-session-host.contract.test.ts` | | A47-L | Pi's valuable `InteractiveMode` TUI behavior can be preserved while one independent cwd-scoped Brunch session host owns the sole writable sealed Pi runtime, JSONL session manager, graph command authority, and semantic live-event fan-out used by both TUI and React clients. The unknown is the TUI attachment seam: Pi exports `InteractiveMode` over an in-process `AgentSessionRuntime`, not a remote TUI client. Validation must prove a real TUI + browser target without a second writable runtime, raw-Pi browser contract, or permanent second relay. Frontier: `shared-session-host-tracer`; retirement/cutover: `shared-session-host-cutover`. | medium | open | D39-L, D132-L, D133-L; I64-L, I65-L | | A48-L | A read-only semantic preflight can improve orientation-menu availability over deterministic graph-fact heuristics while returning within a ≤3-second interaction budget often enough to justify a model-backed path. Admission is limited to the configured soft recommended evaluator model; other foreground selections, missing evaluator auth, timeout, malformed output, or failure use the deterministic safe subset without restricting Pi-native `/model`. The path must consume no foreground turn, write no transcript/graph/session truth, and cache only in process by `{specId, lsn, operationalMode}`. A Brunch-owned reconciliation-blocker reader participates in gating now with an explicit empty implementation; injected non-empty blockers veto availability so future derived/persisted blocker wiring cannot be forgotten. Validation: a tracked human-approved contrastive catalog plus structured scratch tracer report; first run three uncached feasibility calls, then only if plausible run ten uncached labeled cases, requiring ≥8/10 within 3 seconds, exact flags on every completed response, at least one named deterministic-fallback miss corrected, and zero false-positive moves. If the budget or quality gate fails, retire only the model path, not the deterministic fallback. | low | open | D74-L, D109-L, D123-L; `walkthrough-remediation-2` | - + ### Active Decisions @@ -306,9 +306,9 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D132-L | Standalone interactive web uses one cwd-scoped combined Brunch host with a target-addressed inventory of sealed, in-process Pi `AgentSession`s (2026-07-14). The same process serves React assets/WebSocket Brunch RPC and owns coordinator/graph authority; it does not construct `InteractiveMode`, expose raw Pi RPC, spawn one Pi child per session, host multiple projects, or promise in-flight survival across host restart. One durable session target has one driver/many observers and cannot be opened as duplicate writable runtimes; write leases wait for real same-session contention. Hosted-session mutations return the complete `LiveSessionHostResult` discriminated `{status}` union as JSON-RPC success payloads, including domain refusals; only malformed boundary input and thrown host failures use JSON-RPC errors. FE-1200 materialized the one-target path and validated simultaneous target isolation (A43-L). Depends on: D5-L, D10-L, D33-L, D39-L, D84-L; req 4, req 31. Supersedes: D10-L/D72-L read-only-sidecar posture and D84-L singleton/TUI-owned target topology. | [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md), [`src/session/TOPOLOGY.md`](../src/session/TOPOLOGY.md), [`src/rpc/TOPOLOGY.md`](../src/rpc/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — target-addressed host and concurrent-session isolation materialized 2026-07-14 | | D133-L | Web and TUI share transport-neutral presentation semantics, not platform components or separately-authored domain decoders (2026-07-14). Validated Brunch `toolResult.details` project to a shared semantic presentation model; LLM-context, TUI, and React adapters render that meaning for their audiences. Web hydrates from a named JSONL-derived product projection and overlays target-addressed live events, then refetches canonical truth at settlement/reconnect. The host emits neither ANSI/TUI strings nor ready-made HTML, and no chat mirror/event store is introduced. FE-1200 materialized the full required persisted family inventory: ordinary text; free-text, choice, choices, and bounded-questionnaire terminal read-back; candidate, review-set, and digest offers/continuations; and receipt-bearing review settlement. Live React controls answer free text and listed single/multi choices; bounded questionnaires remain answerable headlessly through D38-L's schema-tagged string/JSON envelope, without a dedicated React questionnaire form. Depends on: A44-L, D17-L, D19-L, D104-L; req 12, req 17, req 32. | [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — full required-family coverage materialized 2026-07-15 | | D134-L | The approachable `/compare-specs` control topology is one top-level project Pi session acting as the simulated user and driving exactly one comparison-harness interactive subshell at a time. It does not spawn a Pi actor that then opens a nested interactive shell. The top-level agent alone receives the private mission; each harness receives only its approved minimal framing and the user's natural messages. Shared top-level context and lane order are acceptable and disclosed for this exploratory workflow; isolation-sensitive studies continue to use FE-1210's separate rigorous campaign recipe. Ordinary text interaction is the stock-Pi baseline for choices and approvals; a custom structured-question tool may enhance presentation but is never required. Setup checks are bounded to actual selected-harness prerequisites—no throwaway Pi/Claude provider turns or synthetic actor launches on every run. Supersedes: FE-1215's unmaterialized fresh-nested-actor design note. | [`.pi/prompts/compare-specs.md`](../.pi/prompts/compare-specs.md), [`testing/comparisons/missions/README.md`](../testing/comparisons/missions/README.md); PLAN `operator-comparison-workflow` | active — remediation materialized and focused Brunch smoke witnessed 2026-07-17; full comparison witness deferred | -| D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller boundary. Before a network-denied candidate lane, the controller materializes the parent tree and exact handoff, runs only `corepack yarn install --immutable --mode=skip-build`, and requires tracked source to remain clean. After the lane terminates at `promotion_prepared`, closed focused builds launch the standalone `/optimization` route; a deterministic loopback optimizer grades scenario-first configuration, fixed/optimized bindings, saved/custom objective direction, request construction, progressive/completion/error/cancellation behavior, upstream abort, same-origin secrecy, and source-backed accessibility semantics. D138-L owns merged-reference calibration; synthetic fixtures prove sensitivity but cannot invent labels, roles, or interaction shapes. The broad `/processes/draft` host/iframe shell remains non-gating outer evidence because it requires unrelated platform preparation. Depends on: D134-L, D138-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases` | active — implementation and deterministic source-fidelity rebaseline materialized 2026-07-22; real merged-reference pass not retained | -| D137-L | Historical replay preparation has one deep public operation that owns the security-sensitive order from pinned-source materialization through exact handoff, closed case-owned dependency preparation, strict admission, and lane finalization (2026-07-22). Its final Git authority is the declared two-commit synthetic prefix clarified under A49-L; admission validates the root source identity, packet-only child, strict Claude/Brunch policy pair, network/path probes, and absence of any third history, remote, ref, residue, or escaping symlink. Public callers receive only a lane-discriminated ready descriptor (`Brunch` includes the seeded brownfield `specId`; Claude does not); private stage types may enforce internal ordering, but no public phase machine, arbitrary command/plugin surface, or serializable security token is introduced. A compile-time case registry selects either no dependency preparation for Brunch host landing or D136-L's sole immutable Petrinaut recipe. Depends on: A49-L retirement, D136-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases` | active — materialized 2026-07-22 | -| D138-L | A historical replay profile must pass a no-provider-turn behavioral calibration against its declared merged reference before its first provider attempt (2026-07-22). The source issue/PR and merged behavior—not a synthetic fixture—own observable semantics; the patch is calibration evidence, never a golden diff or architecture oracle. Before any attempt exists, a contradicted packet/oracle may be rebaselined with hashes and rivals regenerated. Synthetic fixtures remain sensitivity rivals and may not author stronger requirements absent from the source. Any reference mismatch blocks provider work; after the first provider attempt, frozen bytes may not be rewritten. Depends on: D70-L, D136-L, D137-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `brownfield-comparison-cases` | active gate — source-fidelity rebaseline materialized 2026-07-22; post-rebaseline real calibration explicitly not run, so no first provider attempt is authorized | +| D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller boundary. Before a network-denied candidate lane, the controller materializes the parent tree and exact handoff, runs only `corepack yarn install --immutable --mode=skip-build`, and requires tracked source to remain clean. After the lane terminates at `promotion_prepared`, closed focused builds launch the standalone `/optimization` route; a deterministic loopback optimizer grades scenario-first configuration, fixed/optimized bindings, saved/custom objective direction, request construction, progressive/completion/error/cancellation behavior, upstream abort, same-origin secrecy, and source-backed accessibility semantics. D138-L owns merged-reference calibration; synthetic fixtures prove sensitivity but cannot invent labels, roles, or interaction shapes. The broad `/processes/draft` host/iframe shell remains non-gating outer evidence because it requires unrelated platform preparation. Depends on: D134-L, D138-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md) | active — materialized 2026-07-22; real merged-reference pass not retained | +| D137-L | Historical replay preparation has one deep public operation that owns the security-sensitive order from pinned-source materialization through exact handoff, closed case-owned dependency preparation, strict admission, and lane finalization (2026-07-22). Its final Git authority is the declared two-commit synthetic prefix clarified under A49-L; admission validates the root source identity, packet-only child, strict Claude/Brunch policy pair, network/path probes, and absence of any third history, remote, ref, residue, or escaping symlink. Public callers receive only a lane-discriminated ready descriptor (`Brunch` includes the seeded brownfield `specId`; Claude does not); private stage types may enforce internal ordering, but no public phase machine, arbitrary command/plugin surface, or serializable security token is introduced. A compile-time case registry selects either no dependency preparation for Brunch host landing or D136-L's sole immutable Petrinaut recipe. Depends on: A49-L retirement, D136-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md) | active — materialized 2026-07-22 | +| D138-L | A historical replay profile must pass a no-provider-turn behavioral calibration against its declared merged reference before its first provider attempt (2026-07-22). The source issue/PR and merged behavior—not a synthetic fixture—own observable semantics; the patch is calibration evidence, never a golden diff or architecture oracle. Before any attempt exists, a contradicted packet/oracle may be rebaselined with hashes and rivals regenerated. Synthetic fixtures remain sensitivity rivals and may not author stronger requirements absent from the source. Any reference mismatch blocks provider work; after the first provider attempt, frozen bytes may not be rewritten. Depends on: D70-L, D136-L, D137-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN Later `petrinaut-reference-fidelity-campaign` | active gate — source-fidelity rebaseline materialized 2026-07-22; post-rebaseline real calibration explicitly not run, so no first provider attempt is authorized | | D60-L | Agent context splits into pull / projection / render / surface, distinguishes graph-truth from active-context reads, and keeps `workspace.state` separate. Agent context (what the agent reasons over) spans `cwd` (filesystem kickoff heuristic — `.brunch?`, session count/length, README/markdown sizes, file counts), `graph` (overview/list/query), and `node` (variable-hop neighborhood). ... | [`src/graph/TOPOLOGY.md`](../src/graph/TOPOLOGY.md), [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md) | active | | D83-L | Context-render house style: a markdown frame (md-pen) with TOON for uniform data and a fenced ASCII tree for hierarchy, wrapped in `
` tags; agent context clusters into `` / `` / `` scopes. Refines D60-L's RENDER stage. LLM-facing agent-context renders adopt one consistent dialect instead of ad-hoc `[bracket]` + bullet lists: - **Audience scope.** `src/agents/contexts/` owns the LLM agent-context dialect (the `
` scope-clustering plus TOON data blocks and t ... | See archive snapshot for full rationale. | active | | D85-L | Suspended prompt-resource axis model: strategy/lens/method are no longer runtime state. A 2026-06-18 grill consolidation of the `agents/skills/` topology and the D58-L manifest axes, implemented across FE-893, FE-861, and FE-898, produced useful prompt-resource content and path topology. ... | [`src/agents/TOPOLOGY.md`](../src/agents/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md) | retired/suspended | @@ -888,7 +888,7 @@ The first required probe is M0: after manual TUI interaction, a checker proves ` | FE-1230 incomparable private process signals | Token accounting, permission prompts, hidden reasoning, and private tool traces differ by product or may be unavailable. | Compare only target-visible/common evidence; retain Brunch-only diagnostics outside judgment packets; mark unavailable cost/permission/tool metrics `not assessable` rather than fabricating parity. | | FE-1230 three-run reliability ceiling | Three Brunch repetitions can reveal catastrophic instability but cannot estimate tail reliability or generalize beyond one browser-app case. | State the ceiling in every report; make no broad speed/cost/reliability claim; add cases or repetitions only after the tracer proves the artifact/oracle contract and a named decision requires stronger evidence. | | FE-1230 same-model non-equivalence | Pinning Opus 4.8 does not equalize product-owned system prompts, tool policy, context management, thinking controls, or provider wrappers. | Frame the result as a same-base-model workflow/product comparison, record exact product/provider/model/configuration versions, and keep process interpretation separate from mechanical outcome correctness. | -| Brownfield historical-solution leakage | A target with later Git refs, remotes, GitHub/Linear/Notion access, or exact historical wording could retrieve the reference implementation and invalidate the replay. Package/dependency preparation may still need controlled network access. | The retired A49-L route archive/materializes base trees without Git history, records source commit/tree identity, sanitizes missions, prepares selected dependencies controller-side, disables MCP/web/research surfaces, bounds filesystem tools, and denies target-command network. Admission adversarially rejects extra refs/remotes, path/symlink escape, weakened policy, reachable loopback/GitHub/Linear/Notion, unavailable isolation, or failing declared local checks; reachable solution sources invalidate the lane rather than reducing confidence. | +| Brownfield historical-solution leakage | A target with later Git refs, remotes, GitHub/Linear/Notion access, or exact historical wording could retrieve the reference implementation and invalidate the replay. Package/dependency preparation may still need controlled network access. | D137-L materializes only the declared two-commit synthetic prefix (source identity root plus packet-only child), sanitizes missions, prepares selected dependencies controller-side, disables MCP/web/research surfaces, bounds filesystem tools, and denies target-command network. Admission adversarially rejects any third history, extra refs/remotes, path/symlink escape, weakened policy, reachable loopback/GitHub/Linear/Notion, unavailable isolation, or failing declared local checks; reachable solution sources invalidate the lane rather than reducing confidence. | | Brunch derived-slice reference mismatch | The selected backend-only FE-1201 mission intentionally excludes TUI/RPC/web changes from the historical PR, so no historical whole-diff equality or completeness claim is valid. | Freeze a fresh requirement registry and independent temp-Git model/property oracle for the derived backend contract; use the historical implementation only to bootstrap fixtures and wrong rivals. | | Brunch public-command oracle ceiling | `/brunch:land` is architecture-neutral and outcome-observable, but fresh sessions auto-kick a provider, settled-session/TUI setup can fail independently of candidate behavior, and a fixed temporary-Git fixture set cannot cover every repository topology. | Resume a controller-supplied settled session under `PI_OFFLINE=1`; separate `setup_failed` from `assertion_failed`; pair the black-box command journey with an independent full-range Git model and focused final-commit-only/bookkeeping rivals; make no reliability claim and add repository topologies only when a named failure or later campaign requires them. | | Petrinaut fake-provider, host-integration, and visual ceiling | The focused `/optimization` hard gate proves the UI/provider contract but bypasses HASH's authenticated host/iframe bridge; the real `/processes/draft` shell requires unrelated GraphQL/codegen and broad platform/toolchain preparation. A deterministic fake also cannot prove every live optimizer response, while functional browser/accessibility assertions cannot judge ordinary visual hierarchy or progress feel. | Keep the standalone fake-provider oracle as the common mechanical gate. Run one fully provisioned HASH host/iframe smoke, one optional real-backend smoke, and masked human visual/code review as non-gating outer evidence; mark unavailable integration evidence `not_assessable`. Promote recurring live-shape or bridge failures into focused controller fixtures only when they can remain architecture-neutral; never let qualitative review override a mechanical failure. | diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index 45cddc9f6..9559b9661 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -11,8 +11,8 @@ This directory owns Brunch-only development loops and curation seams. Nothing he - faux/introspection/tier-2 harnesses used by tests and probes - dev-only witnesses such as `generate-fan-out-witness.ts` - controller-owned execution-comparison packets, lane adapters, immutable evidence contracts, and independently executable black-box oracle journeys (`execution-comparison/`) -- controller-owned end-to-end comparison composition (`end-to-end-comparison.ts` and `end-to-end-comparison/`): content-addressed study and exact-byte handoff contracts, isolated Brunch/Claude adapters, a closed four-cell matrix over unchanged `ExecutionAttempt` leaves, requirement traceability, audience-safe redaction, and the A49-L historical-replay admission boundary -- the shell-callable execution-comparison operator wrapper (`execution-comparison-operator.ts`), which lists/resolves frozen cases, prepares only Brunch/Claude targets, invokes the existing oracle after lane termination, and validates immutable attempt records for the project-local `/compare-execution` prompt +- controller-owned end-to-end comparison composition (`end-to-end-comparison.ts` and `end-to-end-comparison/`): content-addressed study and exact-byte handoff contracts, isolated Brunch/Claude adapters, a closed four-cell matrix over unchanged `ExecutionAttempt` leaves, requirement traceability, audience-safe redaction, and D137-L historical-replay admission +- the shell-callable execution-comparison operator wrapper (`execution-comparison-operator.ts`), which lists/resolves frozen cases, prepares Brunch/Claude targets, runs the controller-only Petrinaut reference preflight, invokes the existing oracle after lane termination, and validates immutable attempt records for the project-local `/compare-execution` prompt - the shell-callable comparison provenance writer (`comparison-provenance.ts`), which captures one write-once release/controller snapshot after setup approval and before any comparison lane - the standalone component preview harness (`scripts/dev-components.ts` → `src/dev/component-preview.ts`) for previewing `.pi/components` in isolation on a real terminal, with no workspace/session/DB - the agent-drivable PTY walkthrough fallback (`npm run tui-driver` → `src/dev/tui-driver.ts`): named `expect`-pumped PTY sessions with guarded fifo control, headless-xterm screen rendering, and wait-for-text; use it when the canonical project-local `pi-interactive-shell` overlay cannot bind in a sandbox/headless host; sessions live under gitignored `.fixtures/scratch/tui-driver/` From 0e300212cbc5273be2a00510d91e6662f2d52ce7 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 23 Jul 2026 00:32:29 +0200 Subject: [PATCH 13/15] FE-1241: Harden brownfield oracle contracts Bind retained evidence to every runtime oracle input and make Petrinaut interactions fail closed on declared addresses. Co-authored-by: Cursor --- docs/archive/PLAN_HISTORY.md | 4 +- memory/PLAN.md | 4 +- src/dev/execution-comparison-operator.ts | 9 +- .../__tests__/case-contract.test.ts | 3 + .../__tests__/operator-cli.test.ts | 25 ++- .../operator-oracle-dispatch.test.ts | 34 +++- ...petrinaut-optimization-oracle.slow.test.ts | 43 ++++ .../petrinaut-optimization-oracle.test.ts | 12 +- src/dev/execution-comparison/case-contract.ts | 24 +-- .../petrinaut-optimization-oracle/browser.ts | 192 +++++++++++------- .../controller/requirement-registry.json | 2 +- .../study-contract.json | 4 +- .../public-contract.json | 15 ++ 13 files changed, 268 insertions(+), 103 deletions(-) diff --git a/docs/archive/PLAN_HISTORY.md b/docs/archive/PLAN_HISTORY.md index b55201737..dad1e461a 100644 --- a/docs/archive/PLAN_HISTORY.md +++ b/docs/archive/PLAN_HISTORY.md @@ -9,10 +9,12 @@ Legacy pre-`next` history was moved out of the live docs tree with the old archi The frontier added content-addressed mission/baseline/registry/execution/oracle packets, exact packet handoff, a two-commit historical target (source-materialization root plus packet-only child), exhaustive code-owned dependency recipes, strict Claude/Brunch admission, pinned Brunch graph seeding, and compile-time oracle dispatch without widening `ExecutionAttempt`. Brunch replays FE-1201/PR #336 host landing at parent `f5a423b19f76cf345d88053456870a126e451618`; Petrinaut replays PR #9051 optimization UI in the full HASH tree at parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978`. Historical patches bootstrap claims and rivals but are never golden diffs or target-visible material. -D137-L is the sole pinned preparation/admission path. D136-L permits only the compiled immutable Petrinaut install before runtime denial and the closed focused builds after lane termination. Brunch's public `/brunch:land` journey is judged by an independent full-range Git model; Petrinaut's standalone `/optimization` journey is judged through a same-origin deterministic optimizer, typed source-backed mechanical addresses, and contrastive rivals. Petri remains the only greenfield case; Clay, arbitrary commands/plugins, source landing, aggregate winners, repetitions, and `ExecutionAttempt` schema changes remain out. +D137-L is the sole pinned preparation/admission path. D136-L permits only the compiled immutable Petrinaut install before runtime denial and the closed focused builds after lane termination. Brunch's public `/brunch:land` journey is judged by an independent full-range Git model; Petrinaut's standalone `/optimization` journey is judged through a same-origin deterministic optimizer, declared mechanical addresses, calibration-derived inputs, and contrastive rivals. Petri remains the only greenfield case; Clay, arbitrary commands/plugins, source landing, aggregate winners, repetitions, and `ExecutionAttempt` schema changes remain out. The real post-rebaseline Petrinaut merged-reference preflight and historical 2×2 provider matrix were explicitly waived for FE-1241. No passing reference receipt or provider-campaign evidence is claimed. Before any future first Petrinaut provider attempt, PLAN Later item `petrinaut-reference-fidelity-campaign` must satisfy D138-L with a passing controller-only merged-reference receipt, then receive separate authorization for provider work. Current implementation state lives in [`src/dev/TOPOLOGY.md`](../../src/dev/TOPOLOGY.md); operational preflight guidance lives in [`docs/praxis/comparison-runs.md`](../praxis/comparison-runs.md). +The same-day `ln-review` reopened FE-1241 before merge and closed four bounded corrections: calibration-seed oracle hashing, typed/calibration-derived Petrinaut address enforcement without fixture fallback, Brunch public-type parity, and absolute oracle evidence output. The proposed hardcoded Brunch source identity was rejected during full verification: the tracked profile hashes pin production identity, while generic parsers must accept generated Git object ids for disposable replay fixtures and materialization enforces the declared source. No D138-L gate was closed. + ## 2026-07-22 Executor Petri sequence reconciliation `executor-slice-attempt-lifecycle` (FE-1192) merged in #324 on 2026-07-13. `petri-execution-parity` (FE-1195) then absorbed and completed the planned isolation/fan-in, durable parallel authority, Petrinaut-visible attempt topology, and epic integration sequence in #325. Their stale active/future entries and dependency chain were retired from `memory/PLAN.md`; current execution ownership and lifecycle semantics live in `src/executor/TOPOLOGY.md` under D127-L–D129-L. diff --git a/memory/PLAN.md b/memory/PLAN.md index a80cb106e..09d6daa71 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -84,7 +84,7 @@ Close the entire first batch of walkthrough-related findings: remediation, the o ### Recently Completed -- 2026-07-22 `brownfield-comparison-cases` (FE-1241) — **✓ complete (provider campaign waived):** frozen Brunch and Petrinaut brownfield packets, the two-commit lane-ready replay boundary, compile-time preparation/oracle registries, deterministic sensitivity rivals, and portable CI are built. The real post-rebaseline Petrinaut preflight/provider campaign was explicitly not run; D138-L remains a Later re-entry gate. +- 2026-07-22 `brownfield-comparison-cases` (FE-1241) — **✓ complete (provider campaign waived):** frozen Brunch and Petrinaut packets, the D137-L replay boundary, deterministic oracles, and portable CI are built. Post-review hardening now binds the Petrinaut calibration seed into oracle identity, removes fixture-owned locator fallbacks and source-reference overclaims, aligns the Brunch public type with its backend runtime shape, and requires an absolute oracle evidence root. The real provider campaign remains waived; D138-L is still a Later re-entry gate. - 2026-07-21 `executor-slice-admission-parity` (FE-1240) — **✓ complete:** candidate admission now rejects scoped slices without executable criterion, design, or verification-machinery context; exact findings enter bounded repair, every admitted repaired slice survives preview/worker-context parsing, and the populated-plan execution guard remains fail-closed. - 2026-07-20 `comparison-reporting-skills` (FE-1232) — **✓ implementation complete, building on landed FE-1230:** added separate project-shared Notion publication and comparison-evidence reporting skills; elicitation, execution, end-to-end, and frozen campaign-strategy references; and executable guardrails for active-procedure precedence, safe mutation, validity-first interpretation, reproducible judging, failure retention, and audience-safe controller-only redaction. Older completion history (including FE-1192/FE-1195 executor topology closure): [`docs/archive/PLAN_HISTORY.md`](../docs/archive/PLAN_HISTORY.md). @@ -646,7 +646,7 @@ KA stream: reuses: canonical harness | isolated integration | epic verification excludes: browser-specific executor gate | durable plan kind | new lifecycle phase -[hard]-> brownfield-comparison-cases - status: complete FE-1241; real provider campaign waived; D138-L residual moved to Later + status: complete FE-1241 including post-review contract hardening; real provider campaign waived; D138-L residual remains Later lights_up: same-base brownfield elicitation -> exact handoff -> crossed execution -> case oracle cases: minimal-petri-net-editor = sole greenfield reference | FE-1201-derived brunch backend | PR #9051 petrinaut frontend excludes: Clay | more greenfields | repetitions | aggregate winner | ExecutionAttempt widening diff --git a/src/dev/execution-comparison-operator.ts b/src/dev/execution-comparison-operator.ts index 3d68f9826..152ed1bee 100644 --- a/src/dev/execution-comparison-operator.ts +++ b/src/dev/execution-comparison-operator.ts @@ -80,6 +80,12 @@ const COMPILED_ORACLES: Readonly> = { fileURLToPath( new URL('./execution-comparison/petrinaut-optimization-oracle/browser.ts', import.meta.url), ), + fileURLToPath( + new URL( + './execution-comparison/petrinaut-optimization-oracle/calibration-seed.json', + import.meta.url, + ), + ), fileURLToPath( new URL('./execution-comparison/petrinaut-optimization-oracle/claims.ts', import.meta.url), ), @@ -205,6 +211,7 @@ export async function runExecutionComparisonOperatorCli( } case 'oracle': { assertOnlyOptions(options, ['case', 'app', 'out']); + const out = requiredAbsolute(options, 'out'); const selected = await resolveExecutionCase(required(options, 'case'), casesRoot); const manifest = await loadControllerOracleManifest(selected.caseDir); const oracle = resolveCompiledExecutionOracle(manifest.id); @@ -217,7 +224,7 @@ export async function runExecutionComparisonOperatorCli( caseDir: selected.caseDir, }); const retained = await retainCompiledOracleReport({ - out: required(options, 'out'), + out, oracleId: oraclePack.manifest.id, oraclePackSha256: oraclePack.packSha256, report, diff --git a/src/dev/execution-comparison/__tests__/case-contract.test.ts b/src/dev/execution-comparison/__tests__/case-contract.test.ts index c54a2d552..f49809310 100644 --- a/src/dev/execution-comparison/__tests__/case-contract.test.ts +++ b/src/dev/execution-comparison/__tests__/case-contract.test.ts @@ -91,8 +91,11 @@ describe('execution comparison public case contract', () => { create: { kind: 'roleName', role: 'button', name: 'Create' }, createDrawer: { kind: 'roleName', role: 'dialog', name: 'Create an optimization' }, scenario: { kind: 'roleContents', role: 'combobox', contents: 'Select a scenario' }, + scenarioSelected: { kind: 'roleContents', role: 'combobox', contents: 'Seasonal Flu' }, metric: { kind: 'roleContents', role: 'combobox', contents: 'Select a metric' }, + metricCustomOption: { kind: 'roleName', role: 'option', name: 'Custom code' }, metricCode: { kind: 'roleName', role: 'textbox', name: 'Editor content' }, + optimizationName: { kind: 'roleName', role: 'textbox', name: 'Name' }, directionMaximize: { kind: 'roleName', role: 'radio', name: 'Maximize' }, directionMinimize: { kind: 'roleName', role: 'radio', name: 'Minimize' }, run: { kind: 'roleName', role: 'button', name: 'Run' }, diff --git a/src/dev/execution-comparison/__tests__/operator-cli.test.ts b/src/dev/execution-comparison/__tests__/operator-cli.test.ts index c401dfc05..5bfe45d57 100644 --- a/src/dev/execution-comparison/__tests__/operator-cli.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-cli.test.ts @@ -1,6 +1,6 @@ import { cp, mkdir, mkdtemp, readFile, readdir, rm, symlink } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { join, relative } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it, vi } from 'vitest'; @@ -220,6 +220,29 @@ describe('execution comparison target preparation', () => { }); }); +describe('execution comparison oracle CLI', () => { + it('rejects a relative evidence path before oracle launch', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-relative-oracle-out-')); + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + await expect( + runExecutionComparisonOperatorCli([ + 'oracle', + '--case', + 'minimal-petri-net-editor-v1', + '--app', + root, + '--out', + relative(process.cwd(), join(root, 'report.json')), + ]), + ).rejects.toThrow('--out must be an absolute path'); + } finally { + stdout.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + }); +}); + describe('Petrinaut preflight operator command', () => { it('requires absolute closed roots and dispatches no provider or arbitrary command input', async () => { const root = await mkdtemp(join(tmpdir(), 'brunch-preflight-cli-')); diff --git a/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts b/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts index ebb1fe330..58f1e1fe9 100644 --- a/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts @@ -1,6 +1,7 @@ -import { mkdtemp, readFile } from 'node:fs/promises'; +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; @@ -9,6 +10,11 @@ import { resolveCompiledExecutionOracle, retainCompiledOracleReport, } from '../../execution-comparison-operator.js'; +import { loadControllerOraclePack } from '../oracle-pack.js'; + +const petrinautCaseDir = fileURLToPath( + new URL('../../../../testing/execution-comparisons/cases/petrinaut-optimization/', import.meta.url), +); describe('execution comparison compiled oracle dispatch', () => { it('keeps shared framing neutral across browser and backend delivery contracts', () => { @@ -32,6 +38,7 @@ describe('execution comparison compiled oracle dispatch', () => { expect.arrayContaining([ expect.stringContaining('petrinaut-optimization-oracle.ts'), expect.stringContaining('petrinaut-optimization-oracle/browser.ts'), + expect.stringContaining('petrinaut-optimization-oracle/calibration-seed.json'), expect.stringContaining('petrinaut-optimization-oracle/claims.ts'), expect.stringContaining('petrinaut-optimization-oracle/fake-optimizer.ts'), ]), @@ -93,6 +100,31 @@ describe('execution comparison compiled oracle dispatch', () => { }), ).rejects.toMatchObject({ code: 'EEXIST' }); }); + + it('changes the Petrinaut oracle-pack identity when its calibration seed changes', async () => { + const oracle = resolveCompiledExecutionOracle('petrinaut-optimization-oracles-v1'); + const seedPath = oracle.implementationFiles.find((path) => path.endsWith('calibration-seed.json')); + expect(seedPath).toBeDefined(); + if (seedPath === undefined) return; + + const root = await mkdtemp(join(tmpdir(), 'brunch-oracle-seed-rival-')); + const rivalSeedPath = join(root, 'calibration-seed.json'); + await writeFile(rivalSeedPath, `${await readFile(seedPath, 'utf8')}\n`); + const [knownPack, rivalPack] = await Promise.all([ + loadControllerOraclePack({ + caseDir: petrinautCaseDir, + implementationFiles: oracle.implementationFiles, + }), + loadControllerOraclePack({ + caseDir: petrinautCaseDir, + implementationFiles: oracle.implementationFiles.map((path) => + path === seedPath ? rivalSeedPath : path, + ), + }), + ]); + + expect(rivalPack.packSha256).not.toBe(knownPack.packSha256); + }); }); function snapshot() { diff --git a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts index cf8d7035e..709cb12a3 100644 --- a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts +++ b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts @@ -6,10 +6,13 @@ import { fileURLToPath } from 'node:url'; import { afterAll, describe, expect, it } from 'vitest'; import { runCommand } from '../../../app/command-runner.js'; +import { isPetrinautOptimizationExecutionCaseContract, loadPublicCasePacket } from '../case-contract.js'; +import { loadControllerOracleManifest } from '../oracle-pack.js'; import { PETRINAUT_FOCUSED_PREPARATION, runPetrinautOptimizationOracle, } from '../petrinaut-optimization-oracle.js'; +import { runPetrinautBrowserChecks } from '../petrinaut-optimization-oracle/browser.js'; import { createInventedLabelsPetrinautCandidate, createKnownGoodPetrinautCandidate, @@ -93,6 +96,46 @@ describe('standalone Petrinaut optimization browser oracle', () => { expect(report.checks.find(({ id }) => id === 'route-and-accessibility')?.status).toBe('failed'); }, 120_000); + it('does not reuse the empty-state scenario address after selection', async () => { + const candidateRoot = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-wrong-scenario-')); + roots.push(candidateRoot); + await createKnownGoodPetrinautCandidate(candidateRoot); + for (const step of PETRINAUT_FOCUSED_PREPARATION) { + const result = await runCommand(step.command, step.args, { + cwd: candidateRoot, + timeoutMs: 30_000, + maxOutputBytes: 16 * 1024, + }); + expect(result.exitCode, result.stderr).toBe(0); + } + const [packet, manifest] = await Promise.all([ + loadPublicCasePacket(caseDir), + loadControllerOracleManifest(caseDir), + ]); + if (!isPetrinautOptimizationExecutionCaseContract(packet.contract)) { + throw new Error('expected Petrinaut packet'); + } + if (manifest.id !== 'petrinaut-optimization-oracles-v1') { + throw new Error('expected Petrinaut manifest'); + } + const contract = { + ...packet.contract, + mechanicalAddresses: { + ...packet.contract.mechanicalAddresses, + scenarioSelected: { + kind: 'roleName', + role: 'combobox', + name: 'Missing selected scenario selector', + } as const, + }, + }; + + const result = await runPetrinautBrowserChecks({ candidateRoot, contract, manifest }); + + expect(result.checks.find(({ id }) => id === 'route-and-accessibility')?.status).toBe('passed'); + expect(result.checks.find(({ id }) => id === 'scenario-configuration')?.status).toBe('failed'); + }, 120_000); + it.each([ [ 'omission', diff --git a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts index 404254513..035b8f110 100644 --- a/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts +++ b/src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.test.ts @@ -86,14 +86,24 @@ describe('controller-owned Petrinaut optimization oracle boundary', () => { expect(source).not.toMatch(/implementationPath|oraclePath|pluginPath|manifest\.(?:command|path)/u); }); - it('consumes source-backed mechanical addresses from the typed public contract', async () => { + it('keeps D138 mechanical interactions on declared or calibration-derived addresses', async () => { const browserSource = await readFile(join(oracleRoot, 'browser.ts'), 'utf8'); expect(browserSource).toMatch(/mechanicalAddresses/u); expect(browserSource).toMatch(/locate\(page, addresses\.create\)/u); expect(browserSource).toMatch(/locate\(page, addresses\.metricCode\)/u); + expect(browserSource).toMatch(/addresses\.scenarioSelected/u); + expect(browserSource).toMatch(/addresses\.metricCustomOption/u); + expect(browserSource).toMatch(/addresses\.optimizationName/u); + expect(browserSource).toMatch(/parseCalibrationInputs/u); expect(browserSource).not.toMatch(/Create optimization/u); expect(browserSource).not.toMatch(/getByRole\('heading', \{ name: 'Optimizations'/u); expect(browserSource).not.toMatch(/getByRole\('tab', \{ name: 'Optimizations'/u); + expect(browserSource).not.toMatch(/getByRole\('checkbox'/u); + expect(browserSource).not.toMatch(/getByRole\('combobox'\)\.first/u); + expect(browserSource).not.toMatch( + /Seasonal Flu|High Virulence Outbreak|Optimize infected_ratio|Infected Fraction/u, + ); + expect(browserSource).not.toMatch(/OPTIMIZATION_NAME_ADDRESS|CUSTOM_METRIC_OPTION/u); }); it.each([ diff --git a/src/dev/execution-comparison/case-contract.ts b/src/dev/execution-comparison/case-contract.ts index 7fbe81175..3d2369bc2 100644 --- a/src/dev/execution-comparison/case-contract.ts +++ b/src/dev/execution-comparison/case-contract.ts @@ -70,21 +70,6 @@ export interface BrunchHostLandingExecutionCasePublicContract { readonly publicCommand: '/brunch:land'; readonly executionTerminal: 'promotion_prepared'; }; - readonly accessibility: Readonly< - Record< - | 'view' - | 'tab' - | 'create' - | 'scenario' - | 'metric' - | 'direction' - | 'run' - | 'cancel' - | 'status' - | 'results', - AccessibleNameContract - > - >; readonly rules: readonly string[]; } @@ -103,8 +88,11 @@ export type PetrinautMechanicalAddressKey = | 'create' | 'createDrawer' | 'scenario' + | 'scenarioSelected' | 'metric' + | 'metricCustomOption' | 'metricCode' + | 'optimizationName' | 'directionMaximize' | 'directionMinimize' | 'run' @@ -369,8 +357,11 @@ const PETRINAUT_MECHANICAL_ADDRESS_KEYS = [ 'create', 'createDrawer', 'scenario', + 'scenarioSelected', 'metric', + 'metricCustomOption', 'metricCode', + 'optimizationName', 'directionMaximize', 'directionMinimize', 'run', @@ -389,8 +380,11 @@ const PETRINAUT_MECHANICAL_ADDRESSES = { create: { kind: 'roleName', role: 'button', name: 'Create' }, createDrawer: { kind: 'roleName', role: 'dialog', name: 'Create an optimization' }, scenario: { kind: 'roleContents', role: 'combobox', contents: 'Select a scenario' }, + scenarioSelected: { kind: 'roleContents', role: 'combobox', contents: 'Seasonal Flu' }, metric: { kind: 'roleContents', role: 'combobox', contents: 'Select a metric' }, + metricCustomOption: { kind: 'roleName', role: 'option', name: 'Custom code' }, metricCode: { kind: 'roleName', role: 'textbox', name: 'Editor content' }, + optimizationName: { kind: 'roleName', role: 'textbox', name: 'Name' }, directionMaximize: { kind: 'roleName', role: 'radio', name: 'Maximize' }, directionMinimize: { kind: 'roleName', role: 'radio', name: 'Minimize' }, run: { kind: 'roleName', role: 'button', name: 'Run' }, diff --git a/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts b/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts index 074db59f3..c6d285666 100644 --- a/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts +++ b/src/dev/execution-comparison/petrinaut-optimization-oracle/browser.ts @@ -51,6 +51,7 @@ export async function runPetrinautBrowserChecks(input: { const consoleErrors: string[] = []; const failedRequests: string[] = []; const calibrationSeed = JSON.parse(await readFile(CALIBRATION_SEED_PATH, 'utf8')) as unknown; + const calibration = parseCalibrationInputs(calibrationSeed); try { await waitForRoute( `${candidateOrigin}${input.contract.acceptance.publicRoute}`, @@ -60,6 +61,7 @@ export async function runPetrinautBrowserChecks(input: { browser = await chromium.launch({ executablePath: await resolveChromeExecutable(), headless: true }); const definitions = checkDefinitions({ contract: input.contract, + calibration, candidateOrigin, fakeOrigin: fake.origin, requests: fake.requests, @@ -113,8 +115,16 @@ export async function runPetrinautBrowserChecks(input: { type CheckDefinition = (page: Page) => Promise; +interface CalibrationInputs { + readonly primaryScenarioName: string; + readonly resetScenarioName: string; + readonly optimizeParameterAddress: PetrinautMechanicalAddress; + readonly savedMetricName: string; +} + function checkDefinitions(input: { readonly contract: PetrinautOptimizationExecutionCasePublicContract; + readonly calibration: CalibrationInputs; readonly candidateOrigin: string; readonly fakeOrigin: string; readonly requests: { readonly body: unknown; readonly aborted: boolean }[]; @@ -130,7 +140,7 @@ function checkDefinitions(input: { await openCreateDrawer(page, addresses); await requireCount(locate(page, addresses.createDrawer), 1, 'createDrawer'); await requireCount(locate(page, addresses.scenario), 1, 'scenario'); - await selectScenarioOption(page, addresses, 'Seasonal Flu'); + await selectScenarioOption(page, addresses, input.calibration.primaryScenarioName); await requireCount(locate(page, addresses.metric), 1, 'metric'); await requireCount(locate(page, addresses.directionMaximize), 1, 'directionMaximize'); await requireCount(locate(page, addresses.run), 1, 'run'); @@ -141,21 +151,25 @@ function checkDefinitions(input: { }); return [ 'public /optimization route ready', - 'required controls expose stable source-backed mechanical addresses', + 'required controls resolve through declared mechanical addresses', ]; }, ], [ 'scenario-configuration', async (page) => { - await openConfiguration(page, addresses); - const optimize = page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }); + await openConfiguration(page, addresses, input.calibration); + const optimize = locate(page, input.calibration.optimizeParameterAddress); await optimize.click({ force: true }); assert(await optimize.isChecked(), 'optimize toggle did not enable'); await locate(page, addresses.directionMinimize).click({ force: true }); - await selectScenarioOption(page, addresses, 'High Virulence Outbreak'); + await selectComboboxOption( + page, + addresses.scenarioSelected, + optionAddress(input.calibration.resetScenarioName), + ); assert( - !(await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).isChecked()), + !(await locate(page, input.calibration.optimizeParameterAddress).isChecked()), 'scenario change retained optimized binding', ); assert( @@ -171,14 +185,12 @@ function checkDefinitions(input: { [ 'request-contract', async (page) => { - await openConfiguration(page, addresses); + await openConfiguration(page, addresses, input.calibration); const savedRequestIndex = input.requests.length; - await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ - force: true, - }); - await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); + await locate(page, input.calibration.optimizeParameterAddress).click({ force: true }); + await selectComboboxOption(page, addresses.metric, optionAddress(input.calibration.savedMetricName)); await locate(page, addresses.directionMaximize).click({ force: true }); - await setOptimizationName(page, 'saved metric proof'); + await setOptimizationName(page, addresses, 'saved metric proof'); await locate(page, addresses.run).click(); await waitForAddress(page, addresses.statusComplete); const savedBody = input.requests[savedRequestIndex]?.body; @@ -193,15 +205,13 @@ function checkDefinitions(input: { await dismissOverlayDrawers(page); await openCreateDrawer(page, addresses); - await selectScenarioOption(page, addresses, 'Seasonal Flu'); + await selectScenarioOption(page, addresses, input.calibration.primaryScenarioName); const customRequestIndex = input.requests.length; - await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ - force: true, - }); - await selectComboboxOption(page, addresses.metric, 'Custom code'); + await locate(page, input.calibration.optimizeParameterAddress).click({ force: true }); + await selectComboboxOption(page, addresses.metric, addresses.metricCustomOption); await locate(page, addresses.metricCode).fill('return 42;'); await locate(page, addresses.directionMinimize).click({ force: true }); - await setOptimizationName(page, 'custom metric proof'); + await setOptimizationName(page, addresses, 'custom metric proof'); await locate(page, addresses.run).click(); await waitForAddress(page, addresses.statusComplete); const body = input.requests[customRequestIndex]?.body; @@ -242,13 +252,11 @@ function checkDefinitions(input: { [ 'progress-and-completion', async (page) => { - await openConfiguration(page, addresses); - await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ - force: true, - }); - await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); + await openConfiguration(page, addresses, input.calibration); + await locate(page, input.calibration.optimizeParameterAddress).click({ force: true }); + await selectComboboxOption(page, addresses.metric, optionAddress(input.calibration.savedMetricName)); await locate(page, addresses.directionMaximize).click({ force: true }); - await setOptimizationName(page, 'progress proof'); + await setOptimizationName(page, addresses, 'progress proof'); await locate(page, addresses.run).click(); await waitForAddress(page, addresses.statusComplete); const progressiveTrialCount = await page.getByText(/^\d+$/u).count(); @@ -266,13 +274,11 @@ function checkDefinitions(input: { [ 'service-error', async (page) => { - await openConfiguration(page, addresses); - await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ - force: true, - }); - await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); + await openConfiguration(page, addresses, input.calibration); + await locate(page, input.calibration.optimizeParameterAddress).click({ force: true }); + await selectComboboxOption(page, addresses.metric, optionAddress(input.calibration.savedMetricName)); await locate(page, addresses.directionMaximize).click({ force: true }); - await setOptimizationName(page, 'service failure'); + await setOptimizationName(page, addresses, 'service failure'); await locate(page, addresses.run).click(); await waitForAddress(page, addresses.statusError); return ['service error rendered distinctly']; @@ -282,13 +288,11 @@ function checkDefinitions(input: { 'cancel-and-abort', async (page) => { const requestIndex = input.requests.length; - await openConfiguration(page, addresses); - await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ - force: true, - }); - await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); + await openConfiguration(page, addresses, input.calibration); + await locate(page, input.calibration.optimizeParameterAddress).click({ force: true }); + await selectComboboxOption(page, addresses.metric, optionAddress(input.calibration.savedMetricName)); await locate(page, addresses.directionMaximize).click({ force: true }); - await setOptimizationName(page, 'cancel proof'); + await setOptimizationName(page, addresses, 'cancel proof'); await locate(page, addresses.run).click(); // Create closes and the view drawer opens on the active record (initializing/running). await page @@ -327,13 +331,11 @@ function checkDefinitions(input: { async (page) => { const browserRequests: string[] = []; page.on('request', (request) => browserRequests.push(request.url())); - await openConfiguration(page, addresses); - await page.getByRole('checkbox', { name: 'Optimize infected_ratio', exact: true }).click({ - force: true, - }); - await selectComboboxOption(page, addresses.metric, 'Infected Fraction'); + await openConfiguration(page, addresses, input.calibration); + await locate(page, input.calibration.optimizeParameterAddress).click({ force: true }); + await selectComboboxOption(page, addresses.metric, optionAddress(input.calibration.savedMetricName)); await locate(page, addresses.directionMaximize).click({ force: true }); - await setOptimizationName(page, 'secrecy proof'); + await setOptimizationName(page, addresses, 'secrecy proof'); await locate(page, addresses.run).click(); await waitForAddress(page, addresses.statusComplete); requirePetrinautFocusedObservation({ @@ -365,14 +367,15 @@ async function navigateToOptimizations( async function openConfiguration( page: Page, addresses: PetrinautOptimizationExecutionCasePublicContract['mechanicalAddresses'], + calibration: CalibrationInputs, ): Promise { await navigateToOptimizations(page, addresses); await openCreateDrawer(page, addresses); assert( - (await page.getByRole('checkbox', { name: /^Optimize /u }).count()) === 0, + (await locate(page, calibration.optimizeParameterAddress).count()) === 0, 'configuration appeared before scenario selection', ); - await selectScenarioOption(page, addresses, 'Seasonal Flu'); + await selectScenarioOption(page, addresses, calibration.primaryScenarioName); } async function openCreateDrawer( @@ -427,26 +430,27 @@ async function selectScenarioOption( addresses: PetrinautOptimizationExecutionCasePublicContract['mechanicalAddresses'], optionText: string, ): Promise { - const drawer = locate(page, addresses.createDrawer); - await drawer.waitFor(); - const empty = locate(page, addresses.scenario); - // Empty-state address when present; after selection the same dialog combobox - // keeps the scenario control (placeholder text no longer matches). - const combobox = (await empty.count()) > 0 ? empty : drawer.getByRole('combobox').first(); + const combobox = locate(page, addresses.scenario); + await requireCount(combobox, 1, 'scenario'); + await selectComboboxLocatorOption(page, combobox, optionAddress(optionText)); +} + +async function selectComboboxLocatorOption( + page: Page, + combobox: Locator, + option: PetrinautMechanicalAddress, +): Promise { const tagName = await combobox.evaluate((element) => element.tagName); if (tagName === 'SELECT') { - const value = await combobox.evaluate((element, expectedText) => { - for (const option of Array.from((element as HTMLSelectElement).options)) { - if ((option.textContent ?? '').includes(expectedText)) return option.value; - } - return null; - }, optionText); - assert(value !== null && value.length > 0, `no select option matched ${optionText}`); - await combobox.selectOption(value); + assert( + option.kind === 'roleName' && option.role === 'option', + 'native select options require a roleName option address', + ); + await combobox.selectOption({ label: option.name }); return; } await combobox.click({ force: true }); - await page.getByRole('option').filter({ hasText: optionText }).first().click(); + await locate(page, option).click(); } function cssRoleSelector(role: string): string { @@ -461,28 +465,22 @@ function cssEscape(value: string): string { async function selectComboboxOption( page: Page, address: PetrinautMechanicalAddress, - optionText: string, + option: PetrinautMechanicalAddress, ): Promise { - const combobox = locate(page, address); - const tagName = await combobox.evaluate((element) => element.tagName); - if (tagName === 'SELECT') { - const value = await combobox.evaluate((element, expectedText) => { - for (const option of Array.from((element as HTMLSelectElement).options)) { - if ((option.textContent ?? '').includes(expectedText)) return option.value; - } - return null; - }, optionText); - assert(value !== null && value.length > 0, `no select option matched ${optionText}`); - await combobox.selectOption(value); - return; - } - await combobox.click({ force: true }); - await page.getByRole('option').filter({ hasText: optionText }).first().click(); + await selectComboboxLocatorOption(page, locate(page, address), option); } -async function setOptimizationName(page: Page, name: string): Promise { - const dialog = page.getByRole('dialog', { name: 'Create an optimization', exact: true }); - const nameField = dialog.getByRole('textbox').first(); +function optionAddress(name: string): PetrinautMechanicalAddress { + return { kind: 'roleName', role: 'option', name }; +} + +async function setOptimizationName( + page: Page, + addresses: PetrinautOptimizationExecutionCasePublicContract['mechanicalAddresses'], + name: string, +): Promise { + const nameField = locate(page, addresses.optimizationName); + await requireCount(nameField, 1, 'optimizationName'); await nameField.fill(name); } @@ -495,6 +493,44 @@ async function requireCount(locator: Locator, expected: number, label: string): assert(actual === expected, `${label}: expected ${expected}, received ${actual}`); } +function parseCalibrationInputs(value: unknown): CalibrationInputs { + assert(record(value), 'invalid Petrinaut calibration seed'); + const sdcpn = value['sdcpn']; + assert(record(sdcpn), 'calibration seed is missing sdcpn'); + const primary = recordById(sdcpn['scenarios'], 'scenario__seasonal_flu', 'scenario'); + const reset = recordById(sdcpn['scenarios'], 'scenario__high_virulence', 'scenario'); + const metric = recordById(sdcpn['metrics'], 'metric__infected_fraction', 'metric'); + assert(typeof primary['name'] === 'string', 'primary calibration scenario is missing its name'); + assert(typeof reset['name'] === 'string', 'reset calibration scenario is missing its name'); + assert(typeof metric['name'] === 'string', 'calibration metric is missing its name'); + assert(Array.isArray(primary['scenarioParameters']), 'primary scenario is missing parameters'); + const parameter = primary['scenarioParameters'].find( + (candidate) => record(candidate) && candidate['identifier'] === 'infected_ratio', + ); + assert(record(parameter), 'primary scenario is missing the optimized calibration parameter'); + assert( + typeof parameter['identifier'] === 'string', + 'optimized calibration parameter is missing its identifier', + ); + return { + primaryScenarioName: primary['name'], + resetScenarioName: reset['name'], + optimizeParameterAddress: { + kind: 'roleName', + role: 'checkbox', + name: `Optimize ${parameter['identifier']}`, + }, + savedMetricName: metric['name'], + }; +} + +function recordById(value: unknown, id: string, label: string): Record { + assert(Array.isArray(value), `calibration seed is missing ${label} rows`); + const found = value.find((candidate) => record(candidate) && candidate['id'] === id); + assert(record(found), `calibration seed is missing ${label} ${id}`); + return found; +} + async function availablePort(): Promise { const server = createServer(); await new Promise((resolve, reject) => { diff --git a/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json b/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json index c4bd00245..3dc19e428 100644 --- a/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json +++ b/testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json @@ -83,7 +83,7 @@ "publicWording": "Expose the Optimizations view, Create action, and Create an optimization drawer controls on /optimization.", "controller": { "wording": "Closed mechanical addresses for Simulate navigation, Create, scenario/metric selectors and metric editor, Maximize/Minimize, Run/Cancel, and Complete/Error/Cancelled remain exact.", - "expectedState": "Every required source-backed mechanical address resolves exactly once." + "expectedState": "Every required declared mechanical address resolves exactly once." } } ] diff --git a/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json b/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json index 421adc4da..a9d4e0a8d 100644 --- a/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json +++ b/testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json @@ -12,11 +12,11 @@ }, "requirementRegistry": { "path": "testing/end-to-end-comparisons/cases/petrinaut-optimization/controller/requirement-registry.json", - "sha256": "sha256:e2b08e6079b69bd1fbcbd8fb3590a5d0fa487b220f9441bccc648989d174455e" + "sha256": "sha256:9e8866d4a2b96a2e43c261f8514c349582547db958978cf13eb3a4654b73cd02" }, "executionContractTemplate": { "path": "testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json", - "sha256": "sha256:5d73dc3c9c76c350a22b9e5d2a06c9a9741a2474523feffe9bb4ba4e5076b321" + "sha256": "sha256:a8dedb7d6be4dfb706e47194de3bb32caf7724e2eb34f808c8a9ab139c235312" }, "oracle": { "id": "petrinaut-optimization-oracles-v1", diff --git a/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json b/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json index 584ad0fb1..59c3aeed5 100644 --- a/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json +++ b/testing/execution-comparisons/cases/petrinaut-optimization/public-contract.json @@ -70,16 +70,31 @@ "role": "combobox", "contents": "Select a scenario" }, + "scenarioSelected": { + "kind": "roleContents", + "role": "combobox", + "contents": "Seasonal Flu" + }, "metric": { "kind": "roleContents", "role": "combobox", "contents": "Select a metric" }, + "metricCustomOption": { + "kind": "roleName", + "role": "option", + "name": "Custom code" + }, "metricCode": { "kind": "roleName", "role": "textbox", "name": "Editor content" }, + "optimizationName": { + "kind": "roleName", + "role": "textbox", + "name": "Name" + }, "directionMaximize": { "kind": "roleName", "role": "radio", From ec0d321768f41a995da3ec6ed5a092ef0bca2551 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 23 Jul 2026 00:41:11 +0200 Subject: [PATCH 14/15] FE-1241: Fix Claude launch flag encoding Encode tool policy lists unambiguously and provision ripgrep so the Linux full gate can exercise bounded grep reads. Co-authored-by: Cursor --- .github/workflows/test.yml | 4 ++-- .../__tests__/execution-adapters.test.ts | 15 +++++++++++++++ src/dev/end-to-end-comparison/claude-adapter.ts | 5 ++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e0d6a0c6..f85e81e26 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,10 +35,10 @@ jobs: git config --global user.name "Brunch CI" git config --global init.defaultBranch main - - name: Install PTY test dependency + - name: Install native test dependencies run: | sudo apt-get update - sudo apt-get install --yes expect zsh + sudo apt-get install --yes expect ripgrep zsh - name: Install run: npm ci diff --git a/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts b/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts index 95302d5ea..0b28fc6a4 100644 --- a/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts @@ -8,10 +8,12 @@ import { afterEach, describe, expect, it } from 'vitest'; import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; import { prepareBrunchExecutionCell } from '../brunch-adapter.js'; import { + createClaudeExecutionLaunch, finalizeClaudeExecutionWorkspace, prepareClaudeExecutionWorkspace, runClaudeExecutionWorkspace, } from '../claude-adapter.js'; +import { createClaudeSolutionIsolationPolicy } from '../solution-isolation.js'; const roots: string[] = []; const contractTemplatePath = fileURLToPath( @@ -35,6 +37,19 @@ async function handoff(root: string): Promise<{ specificationPath: string; speci } describe('end-to-end execution adapters', () => { + it('encodes brownfield Claude tool lists as single CLI option values', () => { + const workspaceDir = '/tmp/brunch-brownfield-claude'; + const policy = createClaudeSolutionIsolationPolicy(workspaceDir, ['/tmp/controller']); + const launch = createClaudeExecutionLaunch({ workspaceDir, isolationPolicy: policy }); + const allowedIndex = launch.args.indexOf('--allowedTools'); + const disallowedIndex = launch.args.indexOf('--disallowedTools'); + + expect(launch.args[allowedIndex + 1]).toBe(policy.allowedTools.join(',')); + expect(launch.args[allowedIndex + 2]).toBe('--disallowedTools'); + expect(launch.args[disallowedIndex + 1]).toBe('WebFetch,WebSearch'); + expect(launch.args[disallowedIndex + 2]).toBe('--settings'); + }); + it('prepares Brunch from exact free-form bytes while preserving the legacy public packet', async () => { const root = await mkdtemp(join(tmpdir(), 'brunch-e2e-adapter-')); roots.push(root); diff --git a/src/dev/end-to-end-comparison/claude-adapter.ts b/src/dev/end-to-end-comparison/claude-adapter.ts index 3a11e61a3..98082c55c 100644 --- a/src/dev/end-to-end-comparison/claude-adapter.ts +++ b/src/dev/end-to-end-comparison/claude-adapter.ts @@ -74,10 +74,9 @@ export function createClaudeExecutionLaunch(input: { '--tools', input.isolationPolicy.allowedTools.join(','), '--allowedTools', - ...input.isolationPolicy.allowedTools, + input.isolationPolicy.allowedTools.join(','), '--disallowedTools', - 'WebFetch', - 'WebSearch', + 'WebFetch,WebSearch', '--settings', JSON.stringify({ enabledPlugins: {}, From bc323a3120cbdd565c7d87b1284aeb3b5fe4d6f5 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 23 Jul 2026 02:46:27 +0200 Subject: [PATCH 15/15] FE-1241: Simplify comparisons for fast learning --- .pi/prompts/compare-execution.md | 27 +- docs/archive/PLAN_HISTORY.md | 8 +- docs/praxis/comparison-guide.md | 4 +- docs/praxis/comparison-runs.md | 45 +- memory/PLAN.md | 7 +- memory/SPEC.md | 11 +- src/app/brunch-tui.ts | 6 - src/dev/TOPOLOGY.md | 26 +- src/dev/end-to-end-comparison.ts | 9 +- .../__tests__/execution-adapters.test.ts | 6 - .../__tests__/solution-isolation.test.ts | 400 +++------- .../end-to-end-comparison/brunch-adapter.ts | 8 - .../end-to-end-comparison/claude-adapter.ts | 2 +- .../solution-isolation.ts | 481 +----------- src/dev/execution-comparison-brunch.ts | 34 +- src/dev/execution-comparison-operator.ts | 67 +- .../compare-execution-prompt.test.ts | 6 +- .../execution-comparison-brunch.test.ts | 53 +- .../historical-replay-target.test.ts | 674 +---------------- .../__tests__/operator-cli.test.ts | 141 ---- .../petrinaut-historical-preflight.test.ts | 691 ----------------- .../historical-replay-target.ts | 19 +- src/dev/execution-comparison/operator-cli.ts | 3 +- .../petrinaut-historical-preflight.ts | 707 ------------------ .../evidence.ts | 255 ------- testing/comparisons/missions/README.md | 4 + 26 files changed, 230 insertions(+), 3464 deletions(-) delete mode 100644 src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts delete mode 100644 src/dev/execution-comparison/petrinaut-historical-preflight.ts delete mode 100644 src/dev/execution-comparison/petrinaut-historical-preflight/evidence.ts diff --git a/.pi/prompts/compare-execution.md b/.pi/prompts/compare-execution.md index 23863aa3b..5b22aff0a 100644 --- a/.pi/prompts/compare-execution.md +++ b/.pi/prompts/compare-execution.md @@ -39,7 +39,9 @@ For a supplied or selected id, run: npx tsx src/dev/execution-comparison-operator.ts inspect --case ``` -This is the only pre-approval case read. It validates the existing FE-1230 public case loader and returns only the exact frozen specification, exact public contract, hashes, and shared framing. If resolution or validation fails, report the error and stop; never guess, repair, normalize, or fall back to another case. +This is the only pre-approval case read. It validates the public case loader and returns only the exact frozen specification, exact public contract, packet hash, repository requirement, compiled oracle identity, and shared framing. If resolution or validation fails, report the error and stop; never guess, repair, normalize, or fall back to another case. + +For a case whose inspection reports `requiresSourceRepository: true`, ask for the absolute path to a trusted local checkout containing the pinned commit. This path is controller setup and is never target-visible. Greenfield cases require no source checkout. Ask which executors to run: **Brunch**, **Claude Code**, or both, and in which disclosed order. The default recommendation is both in the operator's chosen order. This approachable procedure does not claim order blinding, matched private reasoning, or statistical reliability. @@ -56,8 +58,9 @@ Display all of the following together before any target preparation or launch: 5. the run identity, case directory id, public case id, and public-packet hash; 6. exact scratch, attempt-record, per-lane target, process-log, final-tree, final-diff, browser-report, and visible-interaction output paths; 7. Brunch's pinned TUI entrypoint and Claude Code's structured adapter; -8. the fixed post-lane oracle identity `petri-editor-browser-v2`; and -9. the limits: sequential lanes, unchanged inputs, no substantive intervention, no landing, no score, no winner, and no reliability or parity claim. +8. the case's compiled post-lane oracle identity returned by `inspect`; +9. whether preparation is greenfield or a pinned remote-free brownfield snapshot; and +10. the limits: sequential lanes, unchanged inputs, no substantive intervention, no landing, no score, no winner, and no reliability or parity claim. Ask through ordinary typed text for explicit **approve**, **revise**, or **reject**. Ambiguity, questions, qualifications, partial approval, or silence are not approval. Revise and redisplay the complete setup, or reject and stop. Do not prepare a target, start a shell, invoke an executor, or invoke the oracle before explicit approval. @@ -82,13 +85,14 @@ Give both executors the exact same frozen specification and public contract with ### Brunch -Prepare through the existing FE-1230 workspace adapter: +Prepare through the case-aware workspace adapter. For greenfield cases omit `--source-repository`; for pinned brownfield cases include the approved absolute source checkout: ```sh npx tsx src/dev/execution-comparison-operator.ts prepare \ --case \ --lane brunch \ - --target + --target \ + [--source-repository ] ``` From the Brunch repository root, open one direct `interactive_shell` using the pinned project-local entrypoint reported by preparation: @@ -107,13 +111,14 @@ Keep Brunch run metadata, Petri journal/projection, JSONL, and `.brunch/debug/` ### Claude Code -Prepare through the execution wrapper: +Prepare through the same case-aware wrapper, using the same source checkout rule: ```sh npx tsx src/dev/execution-comparison-operator.ts prepare \ --case \ --lane claude_code \ - --target + --target \ + [--source-repository ] ``` Open one direct `interactive_shell` from the fresh target cwd through Claude Code's normal structured adapter: @@ -128,7 +133,7 @@ Send only the approved shared framing. Do not substitute a raw binary, another m After every lane terminates, first capture its final process status and all target-visible interaction evidence. Kill any still-running executor process, query the shell to a final status, dismiss the completed shell record, and verify that no executor process or interactive session remains. -Then run the unchanged controller-owned `petri-editor-browser-v2` oracle against the retained output: +Then run the unchanged controller-owned oracle identity returned by `inspect` against the retained output: ```sh npx tsx src/dev/execution-comparison-operator.ts oracle \ @@ -137,7 +142,7 @@ npx tsx src/dev/execution-comparison-operator.ts oracle \ --out /browser/report.json ``` -Do this after every lane terminates, including failed, exhausted, and invalid lanes. Let the existing wrapper run the target's declared `npm test` and `npm run build`, start the fixed static server, run the existing independent browser journeys, and close browser/server resources. Do not edit the frozen case, controller files, oracle manifest, fixtures, expected states, browser journey implementation, or report to make a lane pass. If the unchanged oracle cannot start, retain that setup failure and do not invent a product verdict. +Do this after every lane terminates, including failed, exhausted, and invalid lanes. Let the case-owned oracle run its fixed command, build, Git, TUI, or browser checks and close all resources. Do not edit the frozen case, controller files, oracle manifest, fixtures, expected states, journey implementation, or report to make a lane pass. If the unchanged oracle cannot start, retain that setup failure and do not invent a product verdict. After the oracle exits, verify cleanup again. Retain the target cwd; cleanup means no live process or session, not deletion of evidence. Do not launch the next lane until the prior executor shell and oracle resources are both fully clean. @@ -149,7 +154,7 @@ For every successful, failed, exhausted, and invalid attempt, retain immutably: - elapsed and intervention ledgers; - cleanup status and any residue; - the final tree and complete base-to-tip diff, plus honest notes for uncommitted or unavailable material; -- common `npm test`, `npm run build`, and browser results; +- common case-owned command and oracle results; - normalized visible interaction evidence; and - the unchanged retained output itself. @@ -165,7 +170,7 @@ The immutable writer rejects an existing attempt id. Never overwrite, delete, re ## Report -After all selected lanes terminate and cleanup is proven, write `report.md` beside the unchanged run-start `provenance.json` under the run scratch root. Present validity before outcomes for each lane. Then report factual terminal state, cleanup, produced output, common command results, and unchanged browser-oracle results. +After all selected lanes terminate and cleanup is proven, write `report.md` beside the unchanged run-start `provenance.json` under the run scratch root. Present study design and run identity, then validity before outcomes for each lane, factual terminal state, cleanup, produced output, common command results, unchanged oracle results, requirement findings, limitations, and recommendations. Keep every referenced path repository-relative so `/comparison-publish` can validate the retained bundle. Keep common evidence and Brunch-only diagnostics visibly separate. Use `not_assessable` for unavailable common evidence. Do not score, rank, choose a winner, claim reliability, infer parity from unavailable or product-private evidence, or let Brunch-only run/Petri/debug data improve its common result. Do not turn ordinary visual hierarchy, clarity, or drag feel into an automatic mechanical verdict. diff --git a/docs/archive/PLAN_HISTORY.md b/docs/archive/PLAN_HISTORY.md index dad1e461a..f08eede5d 100644 --- a/docs/archive/PLAN_HISTORY.md +++ b/docs/archive/PLAN_HISTORY.md @@ -7,13 +7,13 @@ Legacy pre-`next` history was moved out of the live docs tree with the old archi `brownfield-comparison-cases` ([FE-1241](https://linear.app/hash/issue/FE-1241/add-isolated-brownfield-comparison-cases), [PR #362](https://github.com/hashintel/brunch/pull/362)) completed the mechanical expansion from the sole greenfield Petri case to frozen Brunch backend and Petrinaut frontend replays. The branch is `ka/fe-1241-brownfield-comparison-cases`, based on `next` after FE-1240 landed. -The frontier added content-addressed mission/baseline/registry/execution/oracle packets, exact packet handoff, a two-commit historical target (source-materialization root plus packet-only child), exhaustive code-owned dependency recipes, strict Claude/Brunch admission, pinned Brunch graph seeding, and compile-time oracle dispatch without widening `ExecutionAttempt`. Brunch replays FE-1201/PR #336 host landing at parent `f5a423b19f76cf345d88053456870a126e451618`; Petrinaut replays PR #9051 optimization UI in the full HASH tree at parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978`. Historical patches bootstrap claims and rivals but are never golden diffs or target-visible material. +The frontier added content-addressed mission/baseline/registry/execution/oracle packets, exact packet handoff, lightweight pinned-source snapshots, closed case-owned dependency recipes, target-bounded Brunch/Claude launch policy, pinned Brunch graph seeding, and compile-time oracle dispatch without widening `ExecutionAttempt`. Brunch replays FE-1201/PR #336 host landing at parent `f5a423b19f76cf345d88053456870a126e451618`; Petrinaut replays PR #9051 optimization UI in the full HASH tree at parent `5c7a2d9db5caa851c38938f4b1bac19005b0e978`. Historical patches bootstrap claims and rivals but are never golden diffs or target-visible material. -D137-L is the sole pinned preparation/admission path. D136-L permits only the compiled immutable Petrinaut install before runtime denial and the closed focused builds after lane termination. Brunch's public `/brunch:land` journey is judged by an independent full-range Git model; Petrinaut's standalone `/optimization` journey is judged through a same-origin deterministic optimizer, declared mechanical addresses, calibration-derived inputs, and contrastive rivals. Petri remains the only greenfield case; Clay, arbitrary commands/plugins, source landing, aggregate winners, repetitions, and `ExecutionAttempt` schema changes remain out. +D137-L is the shared pinned preparation path. D136-L permits only the compiled immutable Petrinaut install before execution and the closed focused builds after lane termination. Brunch's public `/brunch:land` journey is judged by an independent full-range Git model; Petrinaut's standalone `/optimization` journey is judged through a same-origin deterministic optimizer, declared mechanical addresses, calibration-derived inputs, and contrastive rivals. Petri remains the greenfield case; Clay, arbitrary commands/plugins, source landing, aggregate winners, repetitions, and `ExecutionAttempt` schema changes remain out. -The real post-rebaseline Petrinaut merged-reference preflight and historical 2×2 provider matrix were explicitly waived for FE-1241. No passing reference receipt or provider-campaign evidence is claimed. Before any future first Petrinaut provider attempt, PLAN Later item `petrinaut-reference-fidelity-campaign` must satisfy D138-L with a passing controller-only merged-reference receipt, then receive separate authorization for provider work. Current implementation state lives in [`src/dev/TOPOLOGY.md`](../../src/dev/TOPOLOGY.md); operational preflight guidance lives in [`docs/praxis/comparison-runs.md`](../praxis/comparison-runs.md). +Before merge, the implementation was reoriented from publication-grade benchmark proof to fast product learning. The automated merged-reference preflight, duplicate HASH installations, macOS network verifier, external-service probes, and receipt machinery were retired. Routine runs now use remote-free pinned snapshots and bounded tools as explicit experimental hygiene; source/reference calibration is repeated only when the source, public contract, or oracle changes. Current implementation state lives in [`src/dev/TOPOLOGY.md`](../../src/dev/TOPOLOGY.md); operational guidance lives in [`docs/praxis/comparison-runs.md`](../praxis/comparison-runs.md). -The same-day `ln-review` reopened FE-1241 before merge and closed four bounded corrections: calibration-seed oracle hashing, typed/calibration-derived Petrinaut address enforcement without fixture fallback, Brunch public-type parity, and absolute oracle evidence output. The proposed hardcoded Brunch source identity was rejected during full verification: the tracked profile hashes pin production identity, while generic parsers must accept generated Git object ids for disposable replay fixtures and materialization enforces the declared source. No D138-L gate was closed. +The retained hardening binds the calibration seed into oracle identity, uses typed source-backed Petrinaut addresses without fixture fallback, aligns the Brunch public type with its backend runtime shape, and requires absolute oracle evidence output. The shared `/compare-execution` workflow now derives repository preparation and oracle identity from every selected case and retains publication-compatible provenance, attempts, cleanup, and validity-first reports for Petri, Brunch host landing, and Petrinaut. ## 2026-07-22 Executor Petri sequence reconciliation diff --git a/docs/praxis/comparison-guide.md b/docs/praxis/comparison-guide.md index 629d7783c..d6a72b7ba 100644 --- a/docs/praxis/comparison-guide.md +++ b/docs/praxis/comparison-guide.md @@ -24,9 +24,11 @@ From a trusted top-level project Pi session, run: ```text /compare-execution minimal-petri-net-editor +/compare-execution brunch-host-landing +/compare-execution petrinaut-optimization ``` -The workflow gives Brunch and Claude Code the same frozen specification and applies the same independent browser checks to both outputs. Alpha 10 records the Brunch version and commit automatically. +The workflow gives Brunch and Claude Code the same frozen specification and applies the selected case's independent checks to both outputs. Brownfield cases require a local checkout containing their pinned parent commit; the operator creates a fresh remote-free target from it. Alpha 10 records the Brunch version and commit automatically. ## End-to-end comparison diff --git a/docs/praxis/comparison-runs.md b/docs/praxis/comparison-runs.md index e9d69bba0..4bf71e99e 100644 --- a/docs/praxis/comparison-runs.md +++ b/docs/praxis/comparison-runs.md @@ -72,10 +72,12 @@ The publication skill validates `report.md`, `provenance.json`, and the availabl ## Quick start: compare execution -From a trusted top-level project Pi session, run: +From a trusted top-level project Pi session, run any frozen execution case: ```text /compare-execution minimal-petri-net-editor +/compare-execution brunch-host-landing +/compare-execution petrinaut-optimization ``` The project-local operator displays the frozen specification, public contract, selected Brunch/Claude @@ -84,40 +86,23 @@ fresh isolated lane at a time, cleans up each live shell before continuing, and controller-owned browser oracle to every retained outcome. With no case id, `/compare-execution` lists eligible cases under `testing/execution-comparisons/cases/`. +Greenfield cases prepare an empty repository. Brownfield cases ask for a trusted local source checkout, +materialize the pinned commit into a fresh remote-free repository, add the exact packet, and disable +web/MCP surfaces while bounding file access to the target. This prevents accidental contamination; it +does not claim adversarial isolation from every host capability. + This is developer/evaluation tooling under `.pi/prompts/`, not a shipped Brunch product command. Brunch stops at `promotion_prepared`; the operator never invokes `/brunch:land`, scores a lane, chooses a winner, or treats Brunch-only run/Petri/debug evidence as common comparison evidence. -### Petrinaut historical setup preflight - -Before spending a provider turn on the frozen Petrinaut case, run the controller-only preflight with -an explicit real HASH source checkout and three disjoint scratch roots: - -```sh -npx tsx src/dev/execution-comparison-operator.ts petrinaut-preflight \ - --source-repository /absolute/path/to/hash \ - --parent-target /absolute/disposable-work/parent \ - --reference-target /absolute/disposable-work/reference \ - --output-root /absolute/retained-evidence -``` +### Petrinaut calibration -The command accepts no provider, reference commit, dependency command, manifest command, or oracle -plugin. It prepares the frozen parent through D137-L, calibrates only the code-owned merged PR #9051 -reference with the same compiled immutable install and unchanged -`petrinaut-optimization-oracles-v1`, writes one path-redacted receipt plus bounded write-once -`parent-dependency.json`, `reference-dependency.json`, and `oracle-summary.json` files for each reached -phase, and removes both owned workspaces. Receipt evidence entries contain only fixed relative -filenames, SHA-256, byte count, and truncation state. The receipt is evidence, not an admission token. -`setup_failed` or `assertion_failed` blocks historical provider work; never bypass the failed phase or -replace an invalid receipt. - -The 2026-07-22 diagnostic run proved both immutable installs, tracked-source checks, and all six -focused builds, then exposed that fixture-authored labels were stronger than PR #9051's public UI. -The frozen packet, typed addresses, and synthetic rivals were rebaselined to source-backed semantics -before any provider attempt. PR #362 deliberately shipped without rerunning the expensive real -post-rebaseline preflight, so no passing merged-reference receipt exists. Before a future first -Petrinaut provider attempt, rerun the command above and require a passing receipt under D138-L; the -waiver used to close the implementation PR is not reusable as campaign evidence. +The Petrinaut oracle uses the standalone `/optimization` route, closed focused builds, and a +deterministic loopback optimizer. Its source-backed addresses were calibrated against PR #9051 during +case construction. Recheck the pinned parent fails and the merged reference passes only when the +source commit, public contract, or oracle changes; record that bounded calibration in the change that +updates the case. Routine comparisons do not duplicate the HASH install or materialize a reference +checkout before every provider run. ## End-to-end comparison tracer diff --git a/memory/PLAN.md b/memory/PLAN.md index 09d6daa71..9a5472079 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -21,7 +21,7 @@ Brunch-next has delivered the original composition spine: the host, sealed Pi pr **Parallel lanes.** Group 3 agent-layer work is pickup-ready. `agent-control-plane-closure` is one earned coverage frontier over foreground and background prompt ingress: it absorbs the former `graph-assurance-conduct` and `subagent-skill-access` items, reconciles the stale eager-context fossils left by D58-L's load-on-demand cutover, and closes prompt replacement/resource-invocation contradictions before capture experiments amplify them. FE-1187 retains only the concern-grouped review renderer. FE-1208's reshaped `automation-observability-dx` closure landed on the restacked base; its invalid warrant pilots remain historical evidence, while the useful evaluator/report primitives are available to `capture-ledger-tracer`. Two evaluation use cases stay deliberately distinct: **seed-based intra-product testing** (Brunch on its own terms — seeds encode Brunch graph/spec state only Brunch understands) and **mission-driven agent-as-user cross-product comparison** (`agent-as-user-comparison` — competitor CLIs share no Brunch state, so the comparable artifact is the "ready" spec/plan document a mission produces). FE-1210 closed the rigorous technical tracer on 2026-07-17; the separately planned operator workflow (`operator-comparison-workflow`, FE-1215) makes saved missions, visible comparison-harness framing, conversational launch, and a readable free-form report approachable without weakening the retained regression/evaluation machinery. Its first operator-led run falsified the nested fresh-actor topology: the top-level project Pi session must act as the simulated user and drive one direct harness subshell at a time, with stock-Pi text interaction as the portable baseline; the corrected real witness is deferred to `saved-mission-comparison-witness`. The KA stream owns executor/orchestrator/Execute-mode work. FE-1192 and FE-1195 completed the attempt, isolated fan-in, durable parallel authority, and epic integration sequence; `executor-plan-coherence` now uses those settled seams to test whether multi-slice plans converge on one working result. Broader instrumentation remains trigger-gated under Later. -**End-to-end comparison lane.** FE-1239 materializes the handoff that the completed reporting procedure names but does not run: two fresh rigorous elicitation lanes produce exact approved specifications, each specification crosses an immutable content-addressed handoff into both Brunch and Claude execution, and the unchanged Petri-editor oracle joins all four outputs into requirement-level evidence. Shared interface requirements are disclosed before elicitation and remain controlled baseline, while private/reveal requirements stay controller-only. The first tracer supports bounded within-case contrasts only — no winner, reliability claim, or cross-case causal conclusion. FE-1241 keeps that Petri editor as the sole greenfield case and adds frozen Brunch/Petrinaut brownfield packets, admitted historical-replay targets, and controller-owned oracles; Clay remains outside the campaign. No Petrinaut provider attempt was run. D138-L still requires an explicit passing merged-reference preflight before any future first attempt; that real run was waived for PR #362 and is not claimed as evidence. +**End-to-end comparison lane.** FE-1239 materializes the handoff that the completed reporting procedure names but does not run: two fresh rigorous elicitation lanes produce exact approved specifications, each specification crosses an immutable content-addressed handoff into both Brunch and Claude execution, and the unchanged Petri-editor oracle joins all four outputs into requirement-level evidence. Shared interface requirements are disclosed before elicitation and remain controlled baseline, while private/reveal requirements stay controller-only. The first tracer supports bounded within-case contrasts only — no winner, reliability claim, or cross-case causal conclusion. FE-1241 keeps that Petri editor as the greenfield case and adds frozen Brunch/Petrinaut brownfield packets plus controller-owned oracles. All three missions can now enter the same execution-only workflow and publication contract. Brownfield preparation uses pinned remote-free snapshots and target-bounded tools as experimental hygiene, not adversarial benchmark proof; Clay remains outside the campaign. **Current seams.** Brunch ships on the `1.0.0-alpha.x` line. One-shot `ask` is the only interactive structured-exchange terminal; D125-L's live ask registry provides headless discovery/answering, while the transcript-backed pending projection remains a compatibility surface for live offer tools after the legacy `present_question` pending branch retired. Sweep classification remains fail-closed and compile-time anchored to the exchange-schema terminal names (D117-L), while the larger capture-conditional watermark question remains A40-L. @@ -84,7 +84,7 @@ Close the entire first batch of walkthrough-related findings: remediation, the o ### Recently Completed -- 2026-07-22 `brownfield-comparison-cases` (FE-1241) — **✓ complete (provider campaign waived):** frozen Brunch and Petrinaut packets, the D137-L replay boundary, deterministic oracles, and portable CI are built. Post-review hardening now binds the Petrinaut calibration seed into oracle identity, removes fixture-owned locator fallbacks and source-reference overclaims, aligns the Brunch public type with its backend runtime shape, and requires an absolute oracle evidence root. The real provider campaign remains waived; D138-L is still a Later re-entry gate. +- 2026-07-22 `brownfield-comparison-cases` (FE-1241) — **✓ complete, learning-first:** frozen Brunch and Petrinaut packets, lightweight pinned-source preparation, deterministic oracles, publication-compatible attempt evidence, and portable CI are built. The Petrinaut calibration seed remains part of oracle identity and its source-backed addresses retain contrastive coverage. Expensive merged-reference preflight and adversarial admission were retired before a provider campaign because the current decision needs fast case-level product evidence, not an externally defensible benchmark. - 2026-07-21 `executor-slice-admission-parity` (FE-1240) — **✓ complete:** candidate admission now rejects scoped slices without executable criterion, design, or verification-machinery context; exact findings enter bounded repair, every admitted repaired slice survives preview/worker-context parsing, and the populated-plan execution guard remains fail-closed. - 2026-07-20 `comparison-reporting-skills` (FE-1232) — **✓ implementation complete, building on landed FE-1230:** added separate project-shared Notion publication and comparison-evidence reporting skills; elicitation, execution, end-to-end, and frozen campaign-strategy references; and executable guardrails for active-procedure precedence, safe mutation, validity-first interpretation, reproducible judging, failure retention, and audience-safe controller-only redaction. Older completion history (including FE-1192/FE-1195 executor topology closure): [`docs/archive/PLAN_HISTORY.md`](../docs/archive/PLAN_HISTORY.md). @@ -127,7 +127,6 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand ### Later Instrumentation experiments and far-horizon items. Each re-enters only via re-qualification with a named trigger. -- `petrinaut-reference-fidelity-campaign` — **Later, evidence-gated by D138-L:** before any first Petrinaut historical provider attempt, rerun the controller-only merged-reference preflight against PR #9051 and require a passing receipt from the frozen source-backed packet/oracle. Only after that pass may a separately authorized 2×2 provider matrix be scoped. The real post-rebaseline run and provider campaign were waived for FE-1241; this row does not reopen that completed frontier. - `tier-2-regression-probes` — **Later, trigger-gated**: runnable probes over the tier-2 real-boot faux-provider harness that track improvement/regression of Brunch's own conduct on rich seed scenarios (the intra-product lane's mechanical oracle). Re-enter when the seed/fixture library demonstrates rich, relevant, challenging scenarios worth pinning; artifacts follow the normal probe contract (`docs/architecture/probes-and-transcripts.md`). - `mechanism-trace` — **Later**: post-hoc `wiring` / `nudge` / `conduct` transcript timeline plus static wiring inventory. Re-enter when instrumentation is prioritized; FE-1187 already owns the extracted sweep-debt tripwire. Archived snapshot: [`docs/archive/PLAN_HISTORY.md`](../docs/archive/PLAN_HISTORY.md#2026-07-14-rolling-frontier-reduction). - `agent-tracing` — passive trace instrumentation over Pi lifecycle events for debugging plus conduct/quality evaluation: NDJSON emitter extension (introspection-tap discipline), subagent span joining via SDK `session.subscribe`, and a mechanical-trace × semantic-JSONL join for deterministic conduct checks and judged passes. Entry move is an `ln-spike` (dev-gated `nikiforovall/pi-otel` import: do span trees beat `.brunch/debug/` + JSONL projections?) before any port of `JoshMock/the-agency` observability as the in-product base. Traces are dev/eval artifacts, never product truth (no event-spine backdoor). Design: `docs/design/AGENT_TRACING.md`; sibling idea note `docs/design/RLM_INVESTIGATION_PATTERN.md`. Relation: Later `mechanism-trace` is the transcript-native sibling (carrier classification, no event plane); if both land they may join on a shared trace vocabulary. Absorbs Pi-native P5 (provider/cache observability — latency, cache behavior, whole-run spans), spike-led. @@ -646,7 +645,7 @@ KA stream: reuses: canonical harness | isolated integration | epic verification excludes: browser-specific executor gate | durable plan kind | new lifecycle phase -[hard]-> brownfield-comparison-cases - status: complete FE-1241 including post-review contract hardening; real provider campaign waived; D138-L residual remains Later + status: complete FE-1241 with learning-first pinned snapshots, deterministic case oracles, and publication-compatible execution runs lights_up: same-base brownfield elicitation -> exact handoff -> crossed execution -> case oracle cases: minimal-petri-net-editor = sole greenfield reference | FE-1201-derived brunch backend | PR #9051 petrinaut frontend excludes: Clay | more greenfields | repetitions | aggregate winner | ExecutionAttempt widening diff --git a/memory/SPEC.md b/memory/SPEC.md index 63da61083..99b33e056 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -306,9 +306,8 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D132-L | Standalone interactive web uses one cwd-scoped combined Brunch host with a target-addressed inventory of sealed, in-process Pi `AgentSession`s (2026-07-14). The same process serves React assets/WebSocket Brunch RPC and owns coordinator/graph authority; it does not construct `InteractiveMode`, expose raw Pi RPC, spawn one Pi child per session, host multiple projects, or promise in-flight survival across host restart. One durable session target has one driver/many observers and cannot be opened as duplicate writable runtimes; write leases wait for real same-session contention. Hosted-session mutations return the complete `LiveSessionHostResult` discriminated `{status}` union as JSON-RPC success payloads, including domain refusals; only malformed boundary input and thrown host failures use JSON-RPC errors. FE-1200 materialized the one-target path and validated simultaneous target isolation (A43-L). Depends on: D5-L, D10-L, D33-L, D39-L, D84-L; req 4, req 31. Supersedes: D10-L/D72-L read-only-sidecar posture and D84-L singleton/TUI-owned target topology. | [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md), [`src/session/TOPOLOGY.md`](../src/session/TOPOLOGY.md), [`src/rpc/TOPOLOGY.md`](../src/rpc/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — target-addressed host and concurrent-session isolation materialized 2026-07-14 | | D133-L | Web and TUI share transport-neutral presentation semantics, not platform components or separately-authored domain decoders (2026-07-14). Validated Brunch `toolResult.details` project to a shared semantic presentation model; LLM-context, TUI, and React adapters render that meaning for their audiences. Web hydrates from a named JSONL-derived product projection and overlays target-addressed live events, then refetches canonical truth at settlement/reconnect. The host emits neither ANSI/TUI strings nor ready-made HTML, and no chat mirror/event store is introduced. FE-1200 materialized the full required persisted family inventory: ordinary text; free-text, choice, choices, and bounded-questionnaire terminal read-back; candidate, review-set, and digest offers/continuations; and receipt-bearing review settlement. Live React controls answer free text and listed single/multi choices; bounded questionnaires remain answerable headlessly through D38-L's schema-tagged string/JSON envelope, without a dedicated React questionnaire form. Depends on: A44-L, D17-L, D19-L, D104-L; req 12, req 17, req 32. | [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/web/TOPOLOGY.md`](../src/web/TOPOLOGY.md) | active — full required-family coverage materialized 2026-07-15 | | D134-L | The approachable `/compare-specs` control topology is one top-level project Pi session acting as the simulated user and driving exactly one comparison-harness interactive subshell at a time. It does not spawn a Pi actor that then opens a nested interactive shell. The top-level agent alone receives the private mission; each harness receives only its approved minimal framing and the user's natural messages. Shared top-level context and lane order are acceptable and disclosed for this exploratory workflow; isolation-sensitive studies continue to use FE-1210's separate rigorous campaign recipe. Ordinary text interaction is the stock-Pi baseline for choices and approvals; a custom structured-question tool may enhance presentation but is never required. Setup checks are bounded to actual selected-harness prerequisites—no throwaway Pi/Claude provider turns or synthetic actor launches on every run. Supersedes: FE-1215's unmaterialized fresh-nested-actor design note. | [`.pi/prompts/compare-specs.md`](../.pi/prompts/compare-specs.md), [`testing/comparisons/missions/README.md`](../testing/comparisons/missions/README.md); PLAN `operator-comparison-workflow` | active — remediation materialized and focused Brunch smoke witnessed 2026-07-17; full comparison witness deferred | -| D136-L | Petrinaut's common mechanical hard gate uses the full pinned HASH checkout and a two-phase controller boundary. Before a network-denied candidate lane, the controller materializes the parent tree and exact handoff, runs only `corepack yarn install --immutable --mode=skip-build`, and requires tracked source to remain clean. After the lane terminates at `promotion_prepared`, closed focused builds launch the standalone `/optimization` route; a deterministic loopback optimizer grades scenario-first configuration, fixed/optimized bindings, saved/custom objective direction, request construction, progressive/completion/error/cancellation behavior, upstream abort, same-origin secrecy, and source-backed accessibility semantics. D138-L owns merged-reference calibration; synthetic fixtures prove sensitivity but cannot invent labels, roles, or interaction shapes. The broad `/processes/draft` host/iframe shell remains non-gating outer evidence because it requires unrelated platform preparation. Depends on: D134-L, D138-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md) | active — materialized 2026-07-22; real merged-reference pass not retained | -| D137-L | Historical replay preparation has one deep public operation that owns the security-sensitive order from pinned-source materialization through exact handoff, closed case-owned dependency preparation, strict admission, and lane finalization (2026-07-22). Its final Git authority is the declared two-commit synthetic prefix clarified under A49-L; admission validates the root source identity, packet-only child, strict Claude/Brunch policy pair, network/path probes, and absence of any third history, remote, ref, residue, or escaping symlink. Public callers receive only a lane-discriminated ready descriptor (`Brunch` includes the seeded brownfield `specId`; Claude does not); private stage types may enforce internal ordering, but no public phase machine, arbitrary command/plugin surface, or serializable security token is introduced. A compile-time case registry selects either no dependency preparation for Brunch host landing or D136-L's sole immutable Petrinaut recipe. Depends on: A49-L retirement, D136-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md) | active — materialized 2026-07-22 | -| D138-L | A historical replay profile must pass a no-provider-turn behavioral calibration against its declared merged reference before its first provider attempt (2026-07-22). The source issue/PR and merged behavior—not a synthetic fixture—own observable semantics; the patch is calibration evidence, never a golden diff or architecture oracle. Before any attempt exists, a contradicted packet/oracle may be rebaselined with hashes and rivals regenerated. Synthetic fixtures remain sensitivity rivals and may not author stronger requirements absent from the source. Any reference mismatch blocks provider work; after the first provider attempt, frozen bytes may not be rewritten. Depends on: D70-L, D136-L, D137-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN Later `petrinaut-reference-fidelity-campaign` | active gate — source-fidelity rebaseline materialized 2026-07-22; post-rebaseline real calibration explicitly not run, so no first provider attempt is authorized | +| D136-L | Petrinaut's common mechanical gate uses the full pinned HASH checkout. Before a candidate lane, the controller materializes the parent tree and exact handoff, runs only `corepack yarn install --immutable --mode=skip-build`, and requires tracked source to remain clean. After the lane terminates at `promotion_prepared`, closed focused builds launch the standalone `/optimization` route; a deterministic loopback optimizer grades scenario-first configuration, fixed/optimized bindings, objective direction, request construction, progress/completion/error/cancellation, upstream abort, same-origin secrecy, and source-backed accessibility semantics. Synthetic fixtures prove sensitivity but do not author stronger semantics than the calibrated source behavior. The broad `/processes/draft` host/iframe shell remains non-gating outer evidence. Depends on: D134-L, A49-L retirement. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md) | active — learning-first gate materialized 2026-07-22 | +| D137-L | Historical replay preparation is experimental hygiene for fast product learning, not an adversarial security boundary (revised 2026-07-23). The controller materializes the pinned source tree into a fresh repository with no remote, adds the exact content-addressed packet, runs the one case-owned dependency recipe when required, checks source identity, packet bytes, and tracked cleanliness, then returns a lane-ready Brunch or Claude descriptor. Both lanes disable web/MCP surfaces where supported and bound file tools to the target, but the study reports this limitation rather than probing external services or proving host isolation. Depends on: A49-L retirement, D136-L. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md) | active — simplified for learning throughput | | D60-L | Agent context splits into pull / projection / render / surface, distinguishes graph-truth from active-context reads, and keeps `workspace.state` separate. Agent context (what the agent reasons over) spans `cwd` (filesystem kickoff heuristic — `.brunch?`, session count/length, README/markdown sizes, file counts), `graph` (overview/list/query), and `node` (variable-hop neighborhood). ... | [`src/graph/TOPOLOGY.md`](../src/graph/TOPOLOGY.md), [`src/projections/TOPOLOGY.md`](../src/projections/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/app/TOPOLOGY.md`](../src/app/TOPOLOGY.md) | active | | D83-L | Context-render house style: a markdown frame (md-pen) with TOON for uniform data and a fenced ASCII tree for hierarchy, wrapped in `
` tags; agent context clusters into `` / `` / `` scopes. Refines D60-L's RENDER stage. LLM-facing agent-context renders adopt one consistent dialect instead of ad-hoc `[bracket]` + bullet lists: - **Audience scope.** `src/agents/contexts/` owns the LLM agent-context dialect (the `
` scope-clustering plus TOON data blocks and t ... | See archive snapshot for full rationale. | active | | D85-L | Suspended prompt-resource axis model: strategy/lens/method are no longer runtime state. A 2026-06-18 grill consolidation of the `agents/skills/` topology and the D58-L manifest axes, implemented across FE-893, FE-861, and FE-898, produced useful prompt-resource content and path topology. ... | [`src/agents/TOPOLOGY.md`](../src/agents/TOPOLOGY.md), [`src/agents/contexts/TOPOLOGY.md`](../src/agents/contexts/TOPOLOGY.md), [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md) | retired/suspended | @@ -693,7 +692,7 @@ For agent-as-user evaluation, the primary behavioral claim is **consequential-fa **FE-1230 greenfield execution-comparison assessment (2026-07-20).** Observability is **partial**: build/test output, git trees, browser DOM/accessibility state, console errors, JSON downloads, Brunch run/Petri artifacts, and target-visible interactions are text-native, while ordinary visual hierarchy and drag feel remain human judgments; product-private reasoning and diagnostics are excluded from common evidence. Reproducibility is **partial**: the approved Petri-editor specification, empty-repository base, public automation contract, Opus 4.8 model, budgets, and hidden oracle bytes freeze before the first valid lane, but generated implementation shape and model conduct vary; three Brunch repetitions detect gross instability rather than reliability tails. Controllability is **partial**: the controller can install and run unchanged build, browser, reference-model, round-trip, and negative-space checks against fresh lane outputs, while provider availability and product-native planning remain external. A minimal public accessibility contract gives both lanes stable roles/names without revealing hidden scenarios or expected results. -**Brownfield end-to-end comparison assessment (updated 2026-07-22).** Observability is **partial**: pinned trees, diffs, build/test output, controller verdicts, temporary-host Git state, browser DOM/accessibility state, request/abort evidence, and cleanup are text-native; visual hierarchy and provider conduct remain external. Reproducibility is **high for deterministic mechanics and unproven for campaigns**: frozen Brunch/Petrinaut packets, the two-commit historical target, compile-time recipes/oracles, fake optimizer, and contrastive rivals are stable, but no Petrinaut provider campaign or post-rebaseline real-reference pass was retained. Controllability is **partial with fail-closed isolation**: Brunch host landing runs under `PI_OFFLINE=1` against disposable repositories and an independent Git model; Petrinaut runs focused builds and same-origin browser checks only after lane termination. D138-L blocks a future first Petrinaut provider attempt until an explicit real merged-reference preflight passes. +**Brownfield execution comparison assessment (updated 2026-07-23).** Observability is **partial**: pinned trees, diffs, build/test output, controller verdicts, temporary-host Git state, browser DOM/accessibility state, request/abort evidence, interventions, cleanup, and retained attempts are text-native; visual hierarchy and provider conduct remain external. Reproducibility is **high for deterministic mechanics and unproven for campaigns**: frozen Brunch/Petrinaut packets, remote-free pinned snapshots, case-owned recipes/oracles, the fake optimizer, and contrastive rivals are stable, while generated implementation and model conduct vary. Controllability is **partial**: Brunch host landing uses disposable repositories and an independent Git model; Petrinaut uses focused builds and same-origin browser checks after lane termination. Lightweight tool restrictions prevent accidental contamination but do not establish adversarial isolation; reports must preserve that limitation. **Standalone-web tracer assessment (2026-07-14).** Observability is **partial**: JSONL, target-addressed RPC, semantic event frames, and accessible DOM/text states make structural session behavior observable; stream cadence, visual hierarchy, and error feel remain manual. Reproducibility is **high** for this tracer: paired temporary production boots use the deterministic faux provider and an existing JSONL session, not a live model or a static transcript golden. Controllability is **partial**: the middle loop controls the browser journey and cache/overlay loss, while a workbench walkthrough owns the bounded UX verdict. The normalizer may remove only declared nondeterministic ids/timestamps; it must not mask Brunch binding/runtime/exchange differences. @@ -774,7 +773,7 @@ Dev-loop artifacts route to gitignored `.fixtures/scratch///`, res | Middle | **FE-1230 reference Petri model differential + metamorphic checks** | A tiny controller-owned P/T reference model independently computes enablement and weighted firing; the rendered marking/enablement agrees before and after firing, reset is idempotent, reload returns to the initial marking, export/import preserves structure, and disabled firing plus invalid operations leave state unchanged. | | Middle | **FE-1230 lane/process evidence contract** | Fresh repository identity, pinned model/product versions, budgets, start/end state, final git range, build/test/browser output, retries, interventions, terminal status, and cleanup are retained for every valid, failed, and invalid attempt. Only signals shared across lanes enter comparison packets; unavailable permission/cost/private-tool evidence is `not assessable`. | | Outer | **FE-1230 split execution judgment** | Identity-masked final trees/diffs plus common mechanical results receive criterion-level code/outcome review; a separate unblinded packet judges visible planning/execution conduct. Visual review mechanically rejects only catastrophic unusability (app absent, controls unreachable, interactions impossible); ordinary hierarchy, clarity, and feel remain qualitative and cannot override a mechanical failure. | -| Inner | **Brownfield case, repository-identity, and solution-isolation contracts** | The minimal Petri editor remains the sole greenfield profile; Brunch and Petrinaut profiles name frozen axes, parent commit/tree identity, sanitized mission/baseline/registry hashes, sealed compile-time oracle ids, and exact handoffs. Separate checkouts start from the same tree, contain no later refs/remotes or controller paths, and fail preflight if GitHub/Linear/Notion solution access is possible under the lane policy. | +| Inner | **Brownfield case and repository-identity contracts** | The Petri editor remains the greenfield profile; Brunch and Petrinaut profiles name frozen axes, parent commit/tree identity, sanitized mission/baseline/registry hashes, compiled oracle ids, and exact handoffs. Separate targets start from the same remote-free pinned snapshot, keep controller material outside the cwd, and retain source/packet identity as experimental hygiene without claiming adversarial isolation. | | Middle | **Brunch black-box `/brunch:land` journey plus independent Git model** | The controller creates disposable host/run/review repositories, resumes a settled candidate session under `PI_OFFLINE=1`, drives only the public TUI command, snapshots Git and run metadata before confirmation, then compares post-apply state with a controller-owned full-range reference model. This proves no pre-confirm mutation, complete multi-slice landing rather than final-commit-only application, mode-aware brownfield integration and greenfield materialization, bookkeeping exclusion, honest failure, and terminal `landed` without importing candidate host-landing modules or permitting a provider turn. | | Middle | **Petrinaut standalone fake-provider browser and accessibility oracle** | In the full pinned HASH checkout, controller-prepared focused Petrinaut packages launch the public `/optimization` website route. A deterministic loopback fake optimizer proves capability-present Optimizations visibility, scenario-first configuration, fixed/optimized parameter selection, one metric plus direction, request construction, streamed trials/best-so-far/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and reachable accessible controls without requiring the real optimizer service. | | Outer | **Brownfield live/qualitative evidence** | One fully provisioned HASH `/processes/draft` host/iframe smoke checks capability relay, optimizer proxying, iframe isolation, and adapter reality; one optional real-optimizer Petrinaut smoke checks live response compatibility. Identity-masked visual review judges Petrinaut hierarchy, clarity, and progress legibility; identity-masked code review judges both outputs. These are non-gating findings, are `not_assessable` when unavailable, and cannot override or repair common mechanical failures. | @@ -853,7 +852,7 @@ The first required probe is M0: after manual TUI interaction, a checker proves ` - **Operator-led cross-product comparisons (FE-1215; D134-L remediation before later `saved-mission-comparison-witness` evidence).** The PM-facing comparison door is deliberately distinct from rigorous frozen-packet evaluation. One project Pi prompt conversationally creates or revises a rich private **agent-as-user mission** for a simulated user: their objective, context, priorities, preferences, constraints, knowledge, uncertainty, decision latitude, and conversational posture. The invoking top-level Pi agent receives that mission and directly performs the user's side of each interaction while driving exactly one comparison-harness subshell at a time; it must not spawn a Pi actor that opens another interactive shell. Each comparison harness receives only minimal visible framing plus the opening user message and subsequent mission-grounded answers. Harness selection and framing are run setup, not mission content. Ordinary conversational text is the portable baseline for operator choices and approvals; environment-specific structured-question tools are optional presentation only. Setup checks cover actual selected-harness prerequisites without synthetic actor/provider turns on every run. Editable missions live outside `.fixtures/` under `testing/comparisons/missions/`; each run snapshots the private mission, separately identified target-visible setup/interactions, and outputs under `.fixtures/runs/agent-as-user-comparison/` while temporary lane work stays in scratch. The readable operator report may expose the full private mission so elicitation can be compared against what each harness actually learned, but it keeps that baseline separate from target-visible evidence and declares no automatic winner or prescribed rubric. Because the top-level actor context spans sequential harnesses, this approachable workflow discloses order and does not claim the per-lane actor-process isolation required by rigorous frozen-packet studies; frozen reveal policies, matched budgets, blinding, fresh-per-lane actor sessions, structured adjudication, multi-run statistics, and scripted judges remain separate tools for focused improvement/regression claims. - **FE-1230 execution-comparison oracle boundary.** Execution cases are distinct from private elicitation missions: the human-approved specification and a minimal public runtime/accessibility contract are visible to every lane, while exact browser journeys, reference-model states, expected results, claim mapping, and adversarial fixtures remain controller-only and outside every lane cwd. The public contract requires a static production build at `dist/`, `npm run build`, `npm test`, and stable accessible roles/names for the canvas and named controls; it does not prescribe framework, source topology, implementation decomposition, test library, or internal state model. The versioned `petri-editor-browser-v2` suite tests/builds once, then runs every declared journey from a fresh browser context with public-only setup, per-journey runtime evidence, and non-blocking claim-linked verdicts that distinguish harness/setup failure from product assertion failure. Brunch stops at `promotion_prepared` and never lands. The retained first pair remains immutable; replay/promotion waits for its exact artifact paths. Mutants, masked/process judging, repetitions, and generalized campaign machinery are deferred until one valid end-to-end path exists. The tracer pins `anthropic/claude-opus-4-8` in both products, but same model does not imply equivalent hidden prompting or thinking controls. -- **Brownfield historical-replay oracle boundary (`brownfield-comparison-cases`).** Historical implementations bootstrap independent claims and focused rivals; matching code, decomposition, diff shape, or architecture is never a gate. Brunch replays only FE-1201's backend host-landing behavior through public `/brunch:land`, a controller-supplied settled session under `PI_OFFLINE=1`, and an independent temporary-repository Git model. Petrinaut replays only PR #9051's optimization UI in the full pinned HASH checkout; pre-existing optimizer backend/API capability is disclosed baseline. D136-L makes standalone `/optimization` the deterministic fake-provider hard gate after the one compiled immutable install and closed focused builds; broad `/processes/draft` integration is non-gating outer evidence. D137-L admits only the declared source-root plus packet-child prefix with exhaustive recipes and strict Claude/Brunch isolation. D138-L makes source behavior—not synthetic labels—the semantic authority and requires a passing controller-only merged-reference calibration before a first provider attempt. The deterministic source-fidelity rebaseline is built, but the real post-rebaseline calibration and provider campaign were explicitly not run. Real optimizer, fully provisioned host/iframe, and masked reviews remain non-gating. +- **Brownfield historical-replay oracle boundary (`brownfield-comparison-cases`).** Historical implementations bootstrap independent claims and focused rivals; matching code, decomposition, diff shape, or architecture is never a gate. Brunch replays only FE-1201's backend host-landing behavior through public `/brunch:land`, a controller-supplied settled session under `PI_OFFLINE=1`, and an independent temporary-repository Git model. Petrinaut replays only PR #9051's optimization UI in the full pinned HASH checkout; pre-existing optimizer backend/API capability is disclosed baseline. D136-L makes standalone `/optimization` the deterministic fake-provider gate after the one immutable install and closed focused builds; broad `/processes/draft` integration remains non-gating. D137-L creates the same pinned remote-free source plus exact packet for both executors and records its lightweight contamination ceiling. A merged-reference calibration is rerun only when the source, public contract, or oracle changes, not before every provider attempt. Real optimizer, fully provisioned host/iframe, repetitions, and masked reviews remain optional follow-up evidence. - **Shared-session-host convergence oracle (planned; `shared-session-host-tracer` → `shared-session-host-cutover`).** FE-1200's standalone proof is necessary but not sufficient for architectural replacement. The proving oracle must launch the production host as the sole owner of one writable sealed Pi runtime, attach a real TUI presentation and React observer/driver to the same durable target, exercise an ordinary turn plus one extension-owned structured ask and one TUI-only product interaction, detach/restart a client without ending the hosted runtime, and reject a duplicate writable open or conflicting driver. Durable settlement must converge through the same JSONL presentation, and browser traffic must remain Brunch semantic RPC/events rather than raw Pi RPC/events. Once A47-L retires, the cutover becomes a closed coverage sweep: every required TUI/web lifecycle, command/UI, exchange, transcript, graph-update, model/auth, and shutdown row has one host-owned path and a closure oracle; only then may `SessionEventRelay`, `brunch.sessionEvent`, `/rpc/driver`, and their sidecar harnesses be deleted. Human outer evidence must confirm that the TUI remains useful rather than becoming a thin degraded shell. - **Standalone-web compound oracle (2026-07-14; coverage completed 2026-07-15, `standalone-web-session-host`).** The initial five complementary oracles prove the host tracer: (1) inner RPC/host negative-space contracts require `(specId, sessionId)` on every lifecycle/driver/ask/event path and reject targetless fallback, duplicate writable opens, second drivers, and mismatched targets; (2) projection shape/malformed-detail tests keep JSONL and raw Pi/detail shapes behind the named semantic presentation; (3) a production-wired standalone-web browser journey, controlled only by a deterministic faux provider, asserts accessible hydration, streamed text, one `ask`, answer, and `agent_settled` states; (4) paired temporary web/TUI runs use a narrow declared normalizer to prove both live→settled→fresh JSONL hydration and equivalent Brunch binding/runtime/exchange semantics—no static JSONL golden; (5) a workbench manual checklist judges only stream cadence, busy/settled truthfulness, ask interaction, reload, and error feel. Browser cache/overlay loss is part of the middle-loop journey. The follow-on production-host concurrency differential retires A43-L with overlapping graph writes, asks, failure/recovery, target-local event sequences, reconnect, and separate JSONL readback; shared graph changes appear only through canonical `worldUpdate` continuity. The completed coverage pass adds projection no-loss/malformed tests for every required persisted ask terminal shape (including questionnaire read-back), React render/answer tests for free text and listed single/multi choices, headless schema-envelope questionnaire answering coverage (with no dedicated React questionnaire form), distinct candidate/review-set/digest production settlement/reconnect witnesses, concurrency/target isolation, and receipt-bearing review settlement. Live-provider conduct and process-restart survival remain explicitly deferred to their named later work; no second truth plane or raw Pi browser contract is introduced. - **Trace → eval → score → regression flywheel.** The first combined trajectory/evaluation proof is a controlled Brunch A/B over the existing consequential-fact claim, not a generic tracing platform or omnibus architecture score. One realistic non-inferable scenario carries a human-authored hidden-fact ledger, forbidden rivals, and a controlled reveal policy. The only intervention is an eval/dev ablation of the warrant-before-commit directive at the real prompt-composition seam; all other run conditions remain fixed. Three real-provider TUI-driven runs per arm produce a joined legibility envelope: stamped run configuration, directive inventory and content hashes, advertised/read/provider-visible directive state, ordered model/tool/exchange/TUI/graph effects, atomic evaluator judgments with evidence and rubric identity, and a replayable report. Completion requires promoted evidence that discriminates the full directive from the ablated rival; the report is an instrument, not the completion claim. This proves bounded evaluator discrimination, not competitor superiority or broad prompt quality. After the tracer, mine one real walkthrough failure into the same corpus. Keep OTel, broad subagent span joining, provider matrices, interaction-quality scoring, and competitor campaigns trigger-gated until a named claim requires them. diff --git a/src/app/brunch-tui.ts b/src/app/brunch-tui.ts index 8ab1356b8..806dc444c 100644 --- a/src/app/brunch-tui.ts +++ b/src/app/brunch-tui.ts @@ -50,7 +50,6 @@ import { type SpecSessionActivationCoordinator, type SpecSessionActivationDecision, } from '../session/workspace-session-coordinator.js'; -import type { CommandRunner } from './command-runner.js'; import { openUrlBestEffort } from './open-url.js'; import { chromeStateForWorkspace, @@ -68,7 +67,6 @@ import { resolveBrunchStartupTheme, } from './pi-settings.js'; import { loadBrunchSubagents } from './pi-subagents.js'; -import { createTestRunnerPort } from './test-runner-port.js'; export { BRUNCH_SETTINGS_AUDITED_GETTERS, BRUNCH_SETTINGS_POLICY, @@ -122,7 +120,6 @@ export interface BrunchTuiLaunchContext { */ comparisonIsolation?: { readonly targetRoot: string; - readonly verifier: CommandRunner; }; reportAsyncDiagnostic?: (diagnostic: { readonly type: 'warning'; readonly message: string }) => void; /** @@ -581,9 +578,6 @@ export function createBrunchAgentSessionRuntimeFactory( ? { allowWebTools: false, foregroundFilesystemRoot: comparisonIsolation.targetRoot, - executionPorts: { - testRunner: createTestRunnerPort({ run: comparisonIsolation.verifier }), - }, } : {}), promptContext: () => { diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index 9559b9661..207dca50f 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -11,39 +11,25 @@ This directory owns Brunch-only development loops and curation seams. Nothing he - faux/introspection/tier-2 harnesses used by tests and probes - dev-only witnesses such as `generate-fan-out-witness.ts` - controller-owned execution-comparison packets, lane adapters, immutable evidence contracts, and independently executable black-box oracle journeys (`execution-comparison/`) -- controller-owned end-to-end comparison composition (`end-to-end-comparison.ts` and `end-to-end-comparison/`): content-addressed study and exact-byte handoff contracts, isolated Brunch/Claude adapters, a closed four-cell matrix over unchanged `ExecutionAttempt` leaves, requirement traceability, audience-safe redaction, and D137-L historical-replay admission -- the shell-callable execution-comparison operator wrapper (`execution-comparison-operator.ts`), which lists/resolves frozen cases, prepares Brunch/Claude targets, runs the controller-only Petrinaut reference preflight, invokes the existing oracle after lane termination, and validates immutable attempt records for the project-local `/compare-execution` prompt +- controller-owned end-to-end comparison composition (`end-to-end-comparison.ts` and `end-to-end-comparison/`): content-addressed study and exact-byte handoff contracts, Brunch/Claude adapters, a closed four-cell matrix over unchanged `ExecutionAttempt` leaves, requirement traceability, audience-safe redaction, and D137-L pinned-source preparation +- the shell-callable execution-comparison operator wrapper (`execution-comparison-operator.ts`), which lists/resolves frozen cases, reports each case's repository and oracle identity, prepares Brunch/Claude targets, invokes the selected oracle after lane termination, and validates immutable attempt records for the project-local `/compare-execution` prompt - the shell-callable comparison provenance writer (`comparison-provenance.ts`), which captures one write-once release/controller snapshot after setup approval and before any comparison lane - the standalone component preview harness (`scripts/dev-components.ts` → `src/dev/component-preview.ts`) for previewing `.pi/components` in isolation on a real terminal, with no workspace/session/DB - the agent-drivable PTY walkthrough fallback (`npm run tui-driver` → `src/dev/tui-driver.ts`): named `expect`-pumped PTY sessions with guarded fifo control, headless-xterm screen rendering, and wait-for-text; use it when the canonical project-local `pi-interactive-shell` overlay cannot bind in a sandbox/headless host; sessions live under gitignored `.fixtures/scratch/tui-driver/` It does not own published CLI behavior, public RPC contracts, database imports from outside `graph/`, or the external `pi-interactive-shell`/`zigpty` runtime. The extension is permanent project development tooling declared under root `.pi` for host-capable manual runs; it never enters Brunch's shipped package manifest, sealed `src/.pi` profile, or runtime dependency graph. `docs/praxis/manual-testing.md` owns the measured priority order, install/health check, trust and auto-install implications, takeover/return, artifact bounds, and teardown procedure. -## Historical Replay Isolation +## Historical Replay Preparation -`execution-comparison/historical-replay-target.ts` is D137-L's one deep public operation: callers provide the frozen case, lane, source, target, controller, and forbidden roots and receive only a lane-ready descriptor after every phase succeeds. The operation owns a closed two-commit synthetic prefix—one root materializes and identifies the pinned source tree, its sole child contains only the exact approved spec and public contract and becomes the execution base—then selects from an exhaustive compile-time map with one explicit code-owned dependency recipe for every pinned case id, performs strict admission, and finalizes the lane. There is no default recipe: adding a pinned case without deciding its preparation fails the TypeScript build. Brunch readiness includes a positive brownfield `specId`; Claude readiness omits `specId`; both include adapter-produced launch metadata. No public phase machine, arbitrary command/plugin seam, or half-ready preparation export remains. +`execution-comparison/historical-replay-target.ts` is D137-L's learning-first pinned-source operation: callers provide the frozen case, lane, source, target, and controller roots and receive a lane-ready descriptor after source materialization, exact packet freeze, optional case-owned dependency preparation, lightweight verification, and lane finalization. The target is a fresh repository with no remote; its root records the pinned source commit/tree and its packet child contains the exact approved public files. Preparation checks source identity, packet hashes, tracked cleanliness, and the selected dependency result. Brunch readiness includes a positive brownfield `specId`; Claude readiness omits `specId`; both include adapter-produced launch metadata. -`end-to-end-comparison/solution-isolation.ts` owns the versioned fail-closed admission mechanics. Admission validates both declared commits and their parent/diff relationship, requires exactly one strict Claude/Brunch policy pair, rejects any third history, tracked worktree mutation, remote, later ref, target/symlink escape, forbidden-root overlap, raw verifier substitution, reachable required network probe, or unsupported host, and admits declared local checks only through the branded macOS verifier that denies runtime network plus sealed-root reads/writes. D136-L deliberately permits controller-created non-ignored untracked dependency artifacts after tracked-source cleanliness passes; exact packet bytes are still rehashed independently during final admission. Production composition creates the verifier from the real host and therefore fails closed when unavailable; test composition may inject only a factory returning the same runtime-branded verifier, so Linux CI can exercise the production deep operation without making a verifier part of its domain input or admitting raw substitutes. Claude uses native sandbox denial with no ambient MCP/settings/plugins or web tools; Brunch uses target-bounded foreground and planner/worker child file tools. Controller-owned preparation may run only the compiled Petrinaut immutable-install recipe before admission; Brunch host landing performs no dependency install. Recipe/result metadata stay controller-owned, and case-private packets, historical references, fixtures, and oracles never enter the target. +`end-to-end-comparison/solution-isolation.ts` owns this small hygiene boundary plus the lane tool-policy values. Claude receives empty MCP/settings/plugin state, disabled web tools, its native target policy, and denied reads for controller/source roots. Brunch disables foreground web tools, bounds foreground and child file tools to the target, and exposes only planner/worker execution subagents. The module does not probe GitHub/Linear/Notion, materialize a merged reference, brand a host verifier, or claim adversarial isolation. Reports name this ceiling explicitly; suspicious historical exposure invalidates an attempt rather than triggering hidden repair. ## Brownfield Comparison Oracles `execution-comparison/host-landing-oracle.ts` is the public Brunch controller entry point; its same-named private subtree owns disposable Git fixtures, settled-session public-TUI actuation, the independent Git outcome model, and report types. The oracle resumes only controller-supplied settled sessions under `PI_OFFLINE=1`, invokes `/brunch:land` through the built candidate, and distinguishes setup invalidity from claim failure. -Petrinaut preparation selected by D136-L is split across the historical-replay operation and oracle. `execution-comparison/operator-cli.ts` resolves every `pinned_git` contract into the deep operation; that operation materializes the frozen full HASH tree and exact handoff per lane, then runs the one compiled immutable install (`corepack yarn install --immutable --mode=skip-build`) before the network-denied lane. Install failure or tracked-source mutation removes only the newly owned target. The pinned Brunch branch then seeds the exact specification as one opaque settled requirement, establishes brownfield posture, and returns the `specId` consumed by `execution-comparison-brunch.ts`; Claude receives the same source and handoff without Brunch graph state. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against the standalone `/optimization` route. The compiled sequence includes the directly declared `@hashintel/refractive` workspace build after design-system/core/optimizer prerequisites and immediately before Petrinaut UI; the synthetic candidate rejects omission or reordering rather than discovering a generic workspace graph. Controller HTTP reachability remains a separate setup check. Fresh browser pages use bounded `domcontentloaded` navigation rather than network-idle coupling, then retain the 5-second semantic locator/action budget. The typed mechanical addresses and synthetic rivals are rebaselined to source-backed PR #9051 semantics; they no longer treat fixture-authored labels or keyboard-focus strength as reference truth. No post-rebaseline real-reference pass is retained, so D138-L still blocks a first provider attempt until the controller preflight succeeds. The browser sees only the candidate origin and same-origin proxy, while controller fixtures and the private optimizer origin remain outside target-visible roots. The fully provisioned `/processes/draft` host/iframe journey is non-gating outer evidence and does not alter this mechanical hard gate. - -`execution-comparison/petrinaut-historical-preflight.ts` is the controller-only, no-provider-turn -calibration composition for that seam. It prepares and admits a disposable parent target through -D137-L, materializes the one code-owned merged reference in a disjoint forbidden root, reuses the same -compiled immutable recipe, and invokes the unchanged compiled Petrinaut oracle. Structured fixed-recipe -and fixed-oracle observations remain controller-only until owned-workspace cleanup, then become bounded, -path/secret-redacted, write-once parent dependency, reference dependency, and oracle-summary files. -The schema-validated receipt names only those safe relative files plus their hashes, sizes, and -truncation state. The admitted parent is removed after reference-leak validation and before reference -installation so two large dependency trees do not coexist. The receipt is not lane admission authority -and no reference workspace or launch descriptor is returned. A failed real calibration remains setup -evidence rather than permission to weaken the recipe or oracle. The implementation includes this -composition and deterministic source-fidelity contract without retaining a post-rebaseline real run; -that explicit waiver closes no D138-L gate and authorizes no provider lane. +Petrinaut preparation selected by D136-L is split across the historical-replay operation and oracle. `execution-comparison/operator-cli.ts` resolves every `pinned_git` contract into the shared operation; Petrinaut then runs the one immutable install (`corepack yarn install --immutable --mode=skip-build`) before either lane. Install failure or tracked-source mutation removes only the newly owned target. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against `/optimization`. The compiled sequence includes `@hashintel/refractive` after design-system/core/optimizer prerequisites and before Petrinaut UI. Source-backed mechanical addresses and contrastive rivals remain frozen in the case. Recalibrate parent-fails/reference-passes only when the pinned source, public contract, or oracle changes; routine attempts do not duplicate the HASH install or reference checkout. `execution-comparison-operator.ts` dispatches both brownfield oracles and the retained Petri browser oracle through a closed compile-time registry and includes every implementation module in the immutable oracle-pack hash; manifests cannot select paths, commands, or plugins at runtime. diff --git a/src/dev/end-to-end-comparison.ts b/src/dev/end-to-end-comparison.ts index 1d425f04e..ca5e1a67f 100644 --- a/src/dev/end-to-end-comparison.ts +++ b/src/dev/end-to-end-comparison.ts @@ -53,14 +53,13 @@ export { } from './end-to-end-comparison/claude-adapter.js'; export { retainExecutionCell } from './end-to-end-comparison/execution-cell.js'; export { - assertTargetBoundedPath, createBrunchSolutionIsolationPolicy, createClaudeSolutionIsolationPolicy, - createNetworkDeniedCommandRunner, - SolutionIsolationAdmissionError, + materializePinnedSourceTree, + verifyPreparedHistoricalReplay, type BrunchSolutionIsolationPolicy, type ClaudeSolutionIsolationPolicy, - type IsolationAdmissionReason, - type NetworkDeniedCommandRunner, + type MaterializedHistoricalReplayPrefix, + type MaterializedPinnedSourceTree, type SolutionIsolationPolicy, } from './end-to-end-comparison/solution-isolation.js'; diff --git a/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts b/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts index 0b28fc6a4..dac6de493 100644 --- a/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/execution-adapters.test.ts @@ -75,12 +75,6 @@ describe('end-to-end execution adapters', () => { join(root, 'targets', 'brunch'), '--spec-id', '1', - '--solution-isolation', - 'v1', - '--forbidden-root', - controllerRoot, - '--forbidden-root', - repositoryRoot, ], cwd: repositoryRoot, }); diff --git a/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts b/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts index eb7dcaf62..07ba7412d 100644 --- a/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/solution-isolation.test.ts @@ -1,6 +1,5 @@ import { createHash } from 'node:crypto'; -import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'; -import { createServer, type Server } from 'node:http'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -8,345 +7,116 @@ import { afterEach, describe, expect, it } from 'vitest'; import { runCommand } from '../../../app/command-runner.js'; import { - admitHistoricalReplay, createBrunchSolutionIsolationPolicy, createClaudeSolutionIsolationPolicy, - createNetworkDeniedCommandRunner, materializePinnedSourceTree, - SolutionIsolationAdmissionError, - type MaterializedHistoricalReplayPrefix, - type MaterializedPinnedSourceTree, - type NetworkDeniedCommandRunner, - type SolutionIsolationPolicy, + verifyPreparedHistoricalReplay, } from '../solution-isolation.js'; const roots: string[] = []; -const servers: Server[] = []; afterEach(async () => { - await Promise.all( - servers.splice(0).map( - (server) => - new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }), - ), - ); await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); }); -async function git(cwd: string, args: readonly string[]): Promise { - const result = await runCommand('git', args, { cwd }); - if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); - return result.stdout.trim(); -} - -function createPortableTestVerifier(forbiddenReadRoots: readonly string[]): NetworkDeniedCommandRunner { - return createNetworkDeniedCommandRunner({ - platform: 'darwin', - forbiddenReadRoots, - run: async (command, args, options) => { - if (command !== 'sandbox-exec') { - throw new Error(`unexpected verifier command: ${command}`); - } - const verifiedCommand = args[2]; - const verifiedArgs = args.slice(3); - if (verifiedCommand === '/usr/bin/curl') { - return args[3] === '--version' - ? { exitCode: 0, stdout: 'curl fake\n', stderr: '' } - : { exitCode: 1, stdout: '', stderr: 'network denied by fake sandbox\n' }; - } - if (verifiedCommand === '/bin/ls') { - return { exitCode: 1, stdout: '', stderr: 'path denied by fake sandbox\n' }; - } - if (verifiedCommand === undefined) { - throw new Error('test verifier received no target command'); - } - return await runCommand(verifiedCommand, verifiedArgs, options); - }, - }); -} - -async function sourceRepository(root: string): Promise<{ - readonly repositoryDir: string; - readonly sourceCommit: string; - readonly sourceTree: string; -}> { - const repositoryDir = join(root, 'source'); - await mkdir(repositoryDir); - await git(repositoryDir, ['init']); - await writeFile(join(repositoryDir, 'package.json'), '{"scripts":{"verify":"node verify.mjs"}}\n'); - await writeFile(join(repositoryDir, 'verify.mjs'), "process.stdout.write('local-ok')\n"); - await git(repositoryDir, ['add', '.']); - await git(repositoryDir, [ - '-c', - 'user.name=Isolation Test', - '-c', - 'user.email=isolation@invalid.local', - 'commit', - '-m', - 'Pinned source', - ]); - const sourceCommit = await git(repositoryDir, ['rev-parse', 'HEAD']); - const sourceTree = await git(repositoryDir, ['rev-parse', 'HEAD^{tree}']); - return { repositoryDir, sourceCommit, sourceTree }; -} - -async function localProbeUrl(): Promise { - const server = createServer((_request, response) => { - response.writeHead(200, { 'content-type': 'text/plain' }); - response.end('reachable'); - }); - servers.push(server); - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(0, '127.0.0.1', resolve); - }); - const address = server.address(); - if (address === null || typeof address === 'string') throw new Error('expected TCP server address'); - return `http://127.0.0.1:${address.port}`; -} - -function strictPolicies( - targetDir: string, - forbiddenRoots: readonly string[], -): readonly SolutionIsolationPolicy[] { - return [ - createClaudeSolutionIsolationPolicy(targetDir, forbiddenRoots), - createBrunchSolutionIsolationPolicy(targetDir), - ]; -} - -describe('brownfield historical-solution isolation', () => { - it('materializes a pinned tree as one history-free target with retained source identity', async () => { - const root = await mkdtemp(join(tmpdir(), 'brunch-solution-isolation-')); +describe('learning-first historical snapshots', () => { + it('materializes a pinned tree into a fresh remote-free repository', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-comparison-source-')); roots.push(root); - const source = await sourceRepository(root); - const targetDir = join(root, 'target'); - await writeFile(join(source.repositoryDir, 'historical-reference.patch'), 'must not enter target\n'); + const source = join(root, 'source'); + const target = join(root, 'target'); + await run('git', ['init', '--initial-branch=main', source], root); + await writeFile(join(source, 'README.md'), 'pinned source\n'); + await run('git', ['add', '--all'], source); + await run( + 'git', + ['-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-m', 'source'], + source, + ); + const sourceCommit = await output('git', ['rev-parse', 'HEAD'], source); const materialized = await materializePinnedSourceTree({ - sourceRepositoryDir: source.repositoryDir, - sourceCommit: source.sourceCommit, - targetDir, + sourceRepositoryDir: source, + sourceCommit, + targetDir: target, }); - expect(materialized).toMatchObject({ - targetDir, - sourceCommit: source.sourceCommit, - sourceTree: source.sourceTree, - recipeVersion: 1, - }); - expect(JSON.parse(await readFile(join(targetDir, '.comparison-source.json'), 'utf8'))).toEqual({ - recipeVersion: 1, - sourceCommit: source.sourceCommit, - sourceTree: source.sourceTree, - }); - expect(await git(targetDir, ['rev-list', '--count', 'HEAD'])).toBe('1'); - expect(await git(targetDir, ['remote'])).toBe(''); - expect((await git(targetDir, ['for-each-ref', '--format=%(refname)'])).split('\n')).toEqual([ - 'refs/heads/main', - ]); - expect(await readdir(targetDir)).not.toContain('historical-reference.patch'); + expect(await readFile(join(target, 'README.md'), 'utf8')).toBe('pinned source\n'); + expect(materialized.sourceCommit).toBe(sourceCommit); + expect(await output('git', ['remote'], target)).toBe(''); + expect(await output('git', ['rev-list', '--count', 'HEAD'], target)).toBe('1'); }); - it('admits both executor policies only when paths and target commands stay sealed', async () => { - const root = await mkdtemp(join(tmpdir(), 'brunch-solution-isolation-')); + it('checks frozen packet bytes and tracked cleanliness without claiming adversarial isolation', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-comparison-prefix-')); roots.push(root); - const source = await sourceRepository(root); - const targetDir = join(root, 'target'); + const source = join(root, 'source'); + const target = join(root, 'target'); + await run('git', ['init', '--initial-branch=main', source], root); + await writeFile(join(source, 'README.md'), 'source\n'); + await run('git', ['add', '--all'], source); + await run( + 'git', + ['-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-m', 'source'], + source, + ); + const sourceCommit = await output('git', ['rev-parse', 'HEAD'], source); const materialized = await materializePinnedSourceTree({ - sourceRepositoryDir: source.repositoryDir, - sourceCommit: source.sourceCommit, - targetDir, - }); - const prefix = await freezePacketChild(materialized); - const probeUrl = await localProbeUrl(); - const forbiddenRoots = [ - source.repositoryDir, - join(root, 'controller'), - join(root, 'harness'), - join(root, 'historical-reference'), - ]; - await Promise.all(forbiddenRoots.slice(1).map(async (path) => await mkdir(path))); - - const admission = await admitHistoricalReplay({ - prefix, - policies: strictPolicies(targetDir, forbiddenRoots), - forbiddenRoots, - networkProbeUrls: [probeUrl, 'https://github.com', 'https://linear.app', 'https://www.notion.so'], - verifier: createPortableTestVerifier(forbiddenRoots), - localChecks: [{ command: process.execPath, args: ['verify.mjs'] }], - }); - - expect(admission).toEqual({ - status: 'admitted', - recipeVersion: 1, - sourceCommit: source.sourceCommit, - sourceTree: source.sourceTree, - executors: ['claude_code', 'brunch'], - localChecks: [{ command: process.execPath, args: ['verify.mjs'], exitCode: 0 }], + sourceRepositoryDir: source, + sourceCommit, + targetDir: target, }); + const packet = [ + ['public-contract.json', '{}\n'], + ['spec.md', '# Mission\n'], + ] as const; + for (const [path, bytes] of packet) await writeFile(join(target, path), bytes); + await run('git', ['add', '--all'], target); + await run( + 'git', + ['-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', 'commit', '-m', 'packet'], + target, + ); + const baseSha = await output('git', ['rev-parse', 'HEAD'], target); + const prefix = { + ...materialized, + baseSha, + packetFiles: packet.map(([path, bytes]) => ({ path, sha256: sha256(bytes) })), + }; + + await expect(verifyPreparedHistoricalReplay({ prefix })).resolves.toBeUndefined(); + await writeFile(join(target, 'spec.md'), '# Drifted\n'); + await expect(verifyPreparedHistoricalReplay({ prefix })).rejects.toThrow( + /tracked source changes|drifted/u, + ); }); - it('retains specific reasons for weakened policy, extra Git reachability, path escape, and network', async () => { - const root = await mkdtemp(join(tmpdir(), 'brunch-solution-isolation-')); - roots.push(root); - const source = await sourceRepository(root); - const targetDir = join(root, 'target'); - const materialized = await materializePinnedSourceTree({ - sourceRepositoryDir: source.repositoryDir, - sourceCommit: source.sourceCommit, - targetDir, - }); - const prefix = await freezePacketChild(materialized); - const probeUrl = await localProbeUrl(); - const forbiddenRoots = [source.repositoryDir, join(root, 'controller')]; - await mkdir(forbiddenRoots[1]!); - const baseInput = { - prefix, - forbiddenRoots, - networkProbeUrls: [probeUrl, 'https://github.com', 'https://linear.app', 'https://www.notion.so'], - verifier: createPortableTestVerifier(forbiddenRoots), - localChecks: [{ command: process.execPath, args: ['verify.mjs'] }], - } as const; - - await expect( - admitHistoricalReplay({ - ...baseInput, - policies: [ - { - ...createClaudeSolutionIsolationPolicy(targetDir, forbiddenRoots), - strictMcp: false, - } as unknown as SolutionIsolationPolicy, - createBrunchSolutionIsolationPolicy(targetDir), - ], - }), - ).rejects.toMatchObject({ - reasons: [expect.objectContaining({ code: 'policy_weakened', executor: 'claude_code' })], + it('retains lightweight target policies for both lanes', () => { + expect(createBrunchSolutionIsolationPolicy('/tmp/target')).toMatchObject({ + executor: 'brunch', + foregroundWebTools: false, + executionSubagents: ['planner', 'worker'], }); - - await expect( - admitHistoricalReplay({ - ...baseInput, - policies: [createBrunchSolutionIsolationPolicy(targetDir)], - }), - ).rejects.toMatchObject({ - reasons: [expect.objectContaining({ code: 'policy_weakened' })], - }); - - await writeFile(join(targetDir, 'package.json'), '{"scripts":{"verify":"node changed.mjs"}}\n'); - await expect( - admitHistoricalReplay({ - ...baseInput, - policies: strictPolicies(targetDir, forbiddenRoots), - }), - ).rejects.toMatchObject({ - reasons: [expect.objectContaining({ code: 'git_worktree_changes_present' })], - }); - await writeFile(join(targetDir, 'package.json'), '{"scripts":{"verify":"node verify.mjs"}}\n'); - - await git(targetDir, ['remote', 'add', 'origin', 'https://github.com/hashintel/brunch.git']); - await git(targetDir, ['branch', 'later-solution']); - await expect( - admitHistoricalReplay({ - ...baseInput, - policies: strictPolicies(targetDir, forbiddenRoots), - }), - ).rejects.toMatchObject({ - reasons: expect.arrayContaining([ - expect.objectContaining({ code: 'git_remote_present' }), - expect.objectContaining({ code: 'git_ref_present' }), - ]), - }); - await git(targetDir, ['remote', 'remove', 'origin']); - await git(targetDir, ['branch', '-D', 'later-solution']); - - await expect( - admitHistoricalReplay({ - ...baseInput, - forbiddenRoots: [join(targetDir, 'nested-controller')], - policies: strictPolicies(targetDir, [join(targetDir, 'nested-controller')]), - }), - ).rejects.toMatchObject({ - reasons: expect.arrayContaining([expect.objectContaining({ code: 'path_boundary_weakened' })]), - }); - - const controllerDir = join(root, 'controller'); - await writeFile(join(controllerDir, 'reference.txt'), 'historical solution\n'); - await symlink(join(controllerDir, 'reference.txt'), join(targetDir, 'escaped-reference.txt')); - await expect( - admitHistoricalReplay({ - ...baseInput, - policies: strictPolicies(targetDir, forbiddenRoots), - }), - ).rejects.toMatchObject({ - reasons: expect.arrayContaining([ - expect.objectContaining({ - code: 'path_boundary_weakened', - detail: 'target symlink escapes isolation root: escaped-reference.txt', - }), - ]), + expect(createClaudeSolutionIsolationPolicy('/tmp/target', ['/tmp/controller'])).toMatchObject({ + executor: 'claude_code', + strictMcp: true, + webTools: false, + nativeSandbox: { deniedReadRoots: ['/tmp/controller'] }, }); - await rm(join(targetDir, 'escaped-reference.txt')); - - const networkCapableVerifier = { - recipeVersion: 1 as const, - platform: 'darwin' as const, - forbiddenReadRoots: forbiddenRoots, - run: runCommand, - } as unknown as NetworkDeniedCommandRunner; - await expect( - admitHistoricalReplay({ - ...baseInput, - verifier: networkCapableVerifier, - policies: strictPolicies(targetDir, forbiddenRoots), - }), - ).rejects.toBeInstanceOf(SolutionIsolationAdmissionError); - await expect( - admitHistoricalReplay({ - ...baseInput, - verifier: networkCapableVerifier, - policies: strictPolicies(targetDir, forbiddenRoots), - }), - ).rejects.toMatchObject({ - reasons: [expect.objectContaining({ code: 'policy_weakened' })], - }); - }); - - it('fails closed when the host cannot provide the versioned isolation recipe', () => { - expect(() => createNetworkDeniedCommandRunner({ platform: 'linux' })).toThrow( - 'solution isolation recipe v1 is unsupported on linux', - ); }); }); -async function freezePacketChild( - materialized: MaterializedPinnedSourceTree, -): Promise { - const packetFiles = [ - { path: 'public-contract.json' as const, bytes: Buffer.from('{"schemaVersion":1}\n') }, - { path: 'spec.md' as const, bytes: Buffer.from('# Exact packet\n') }, - ]; - for (const file of packetFiles) { - await writeFile(join(materialized.targetDir, file.path), file.bytes); - } - await git(materialized.targetDir, ['add', '--', ...packetFiles.map(({ path }) => path)]); - await git(materialized.targetDir, [ - '-c', - 'user.name=Isolation Test', - '-c', - 'user.email=isolation@invalid.local', - 'commit', - '-m', - 'Freeze exact packet', - ]); - return { - ...materialized, - baseSha: await git(materialized.targetDir, ['rev-parse', 'HEAD']), - packetFiles: packetFiles.map(({ path, bytes }) => ({ - path, - sha256: `sha256:${createHash('sha256').update(bytes).digest('hex')}`, - })), - }; +function sha256(value: string): string { + return `sha256:${createHash('sha256').update(value).digest('hex')}`; +} + +async function run(command: string, args: readonly string[], cwd: string): Promise { + const result = await runCommand(command, args, { cwd }); + if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); +} + +async function output(command: string, args: readonly string[], cwd: string): Promise { + const result = await runCommand(command, args, { cwd }); + if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); + return result.stdout.trim(); } diff --git a/src/dev/end-to-end-comparison/brunch-adapter.ts b/src/dev/end-to-end-comparison/brunch-adapter.ts index f242f1e71..6bd125555 100644 --- a/src/dev/end-to-end-comparison/brunch-adapter.ts +++ b/src/dev/end-to-end-comparison/brunch-adapter.ts @@ -18,7 +18,6 @@ export interface ExecutionLaunch { export function createBrunchExecutionLaunch(input: { readonly workspaceDir: string; readonly specId: number; - readonly forbiddenRoots: readonly string[]; }): ExecutionLaunch { return { command: 'npx', @@ -29,9 +28,6 @@ export function createBrunchExecutionLaunch(input: { input.workspaceDir, '--spec-id', String(input.specId), - '--solution-isolation', - 'v1', - ...input.forbiddenRoots.flatMap((root) => ['--forbidden-root', root]), ], cwd: repositoryRoot(), }; @@ -64,15 +60,11 @@ export async function prepareBrunchExecutionCell(input: { caseDir: materialized.packetDir, specificationMode: 'opaque', }); - const forbiddenRoots = [input.controllerRoot, repositoryRoot()].filter( - (root) => !containedPath(root, input.workspaceDir) && !containedPath(input.workspaceDir, root), - ); return { prepared, launch: createBrunchExecutionLaunch({ workspaceDir: input.workspaceDir, specId: prepared.specId, - forbiddenRoots, }), }; } diff --git a/src/dev/end-to-end-comparison/claude-adapter.ts b/src/dev/end-to-end-comparison/claude-adapter.ts index 98082c55c..db4dc8a20 100644 --- a/src/dev/end-to-end-comparison/claude-adapter.ts +++ b/src/dev/end-to-end-comparison/claude-adapter.ts @@ -270,7 +270,7 @@ function implementationPrompt(): string { 'Treat public-contract.json as the only additional delivery and interoperability baseline.', 'Work only inside the current Git repository. Do not inspect parent directories or external project files.', 'Do not reinterpret, normalize, or repair the specification.', - 'Run npm test and npm run build. Leave the complete implementation in the working tree when both pass.', + 'Follow the delivery, acceptance, and terminal rules in public-contract.json. Leave the complete implementation in the working tree when finished.', 'Do not create or use browser-oracle files; those are controller-owned and unavailable.', ].join(' '); } diff --git a/src/dev/end-to-end-comparison/solution-isolation.ts b/src/dev/end-to-end-comparison/solution-isolation.ts index 415e6e60f..743b639d0 100644 --- a/src/dev/end-to-end-comparison/solution-isolation.ts +++ b/src/dev/end-to-end-comparison/solution-isolation.ts @@ -1,8 +1,7 @@ import { createHash } from 'node:crypto'; -import { realpathSync } from 'node:fs'; -import { mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { dirname, join, resolve } from 'node:path'; import { runCommand, type CommandResult, type CommandRunner } from '../../app/command-runner.js'; @@ -14,23 +13,6 @@ const COMPARISON_GIT_IDENTITY = [ '-c', 'user.email=brunch-comparison@invalid.local', ] as const; -const NETWORK_DENIED_RUNNER = Symbol('network-denied-runner'); -const REQUIRED_SOLUTION_PROBE_URLS = [ - 'https://github.com', - 'https://linear.app', - 'https://www.notion.so', -] as const; -const DENIED_SOLUTION_SOURCE_DOMAINS = [ - 'github.com', - '*.github.com', - '*.githubusercontent.com', - 'linear.app', - '*.linear.app', - 'notion.so', - '*.notion.so', - 'notion.site', - '*.notion.site', -] as const; export interface MaterializedPinnedSourceTree { readonly recipeVersion: typeof RECIPE_VERSION; @@ -62,7 +44,7 @@ export interface ClaudeSolutionIsolationPolicy { readonly enabled: true; readonly failIfUnavailable: true; readonly allowedDomains: readonly []; - readonly deniedDomains: typeof DENIED_SOLUTION_SOURCE_DOMAINS; + readonly deniedDomains: readonly []; readonly deniedReadRoots: readonly string[]; }; readonly allowedTools: readonly ['Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write']; @@ -76,44 +58,10 @@ export interface BrunchSolutionIsolationPolicy { readonly specifySubagents: false; readonly foregroundFileRoot: string; readonly executionSubagents: readonly ['planner', 'worker']; - readonly networkDeniedVerifier: true; } export type SolutionIsolationPolicy = ClaudeSolutionIsolationPolicy | BrunchSolutionIsolationPolicy; -export interface NetworkDeniedCommandRunner { - readonly recipeVersion: typeof RECIPE_VERSION; - readonly platform: 'darwin'; - readonly forbiddenReadRoots: readonly string[]; - readonly [NETWORK_DENIED_RUNNER]: true; - readonly run: CommandRunner; -} - -export interface IsolationAdmissionReason { - readonly code: - | 'identity_mismatch' - | 'git_history_present' - | 'git_ref_present' - | 'git_remote_present' - | 'git_worktree_changes_present' - | 'local_check_failed' - | 'network_probe_reachable' - | 'path_boundary_weakened' - | 'policy_weakened'; - readonly detail: string; - readonly executor?: SolutionIsolationPolicy['executor']; -} - -export class SolutionIsolationAdmissionError extends Error { - readonly reasons: readonly IsolationAdmissionReason[]; - - constructor(reasons: readonly IsolationAdmissionReason[]) { - super(`historical replay isolation admission failed: ${reasons.map(({ code }) => code).join(', ')}`); - this.name = 'SolutionIsolationAdmissionError'; - this.reasons = reasons; - } -} - export async function materializePinnedSourceTree(input: { readonly sourceRepositoryDir: string; readonly sourceCommit: string; @@ -167,9 +115,7 @@ export async function materializePinnedSourceTree(input: { syntheticCommit, }; } catch (error) { - if (targetCreated) { - await rm(input.targetDir, { recursive: true, force: true }); - } + if (targetCreated) await rm(input.targetDir, { recursive: true, force: true }); throw error; } finally { await rm(archiveDir, { recursive: true, force: true }); @@ -194,8 +140,8 @@ export function createClaudeSolutionIsolationPolicy( enabled: true, failIfUnavailable: true, allowedDomains: [], - deniedDomains: DENIED_SOLUTION_SOURCE_DOMAINS, - deniedReadRoots: [...new Set(forbiddenRoots.map(canonicalPolicyPath))], + deniedDomains: [], + deniedReadRoots: [...new Set(forbiddenRoots.map((root) => resolve(root)))], }, allowedTools: ['Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write'], }; @@ -211,419 +157,36 @@ export function createBrunchSolutionIsolationPolicy(targetRoot: string): BrunchS specifySubagents: false, foregroundFileRoot: root, executionSubagents: ['planner', 'worker'], - networkDeniedVerifier: true, }; } -export function createNetworkDeniedCommandRunner( - options: { - readonly platform?: NodeJS.Platform; - readonly run?: CommandRunner; - readonly forbiddenReadRoots?: readonly string[]; - } = {}, -): NetworkDeniedCommandRunner { - // ceiling: recipe v1 supports macOS sandbox-exec only; add an explicit equivalent host recipe before admitting another platform. - const platform = options.platform ?? process.platform; - if (platform !== 'darwin') { - throw new Error(`solution isolation recipe v1 is unsupported on ${platform}`); - } - const run = options.run ?? runCommand; - const forbiddenReadRoots = [...new Set((options.forbiddenReadRoots ?? []).map(canonicalPolicyPath))]; - const sandboxProfile = denyTargetCommandAccessProfile(forbiddenReadRoots); - return { - recipeVersion: RECIPE_VERSION, - platform, - forbiddenReadRoots, - [NETWORK_DENIED_RUNNER]: true, - run: async (command, args, commandOptions) => - await run('sandbox-exec', ['-p', sandboxProfile, command, ...args], commandOptions), - }; -} - -export function assertTargetBoundedPath(policy: SolutionIsolationPolicy, requestedPath: string): string { - const targetRoot = resolve(policy.targetRoot); - const candidate = resolve(targetRoot, requestedPath); - if (!containedPath(targetRoot, candidate)) { - throw new Error(`${policy.executor} path escapes target root: ${requestedPath}`); - } - return candidate; -} - -export async function admitHistoricalReplay(input: { +export async function verifyPreparedHistoricalReplay(input: { readonly prefix: MaterializedHistoricalReplayPrefix; - readonly policies: readonly SolutionIsolationPolicy[]; - readonly forbiddenRoots: readonly string[]; - readonly networkProbeUrls: readonly string[]; - readonly verifier: NetworkDeniedCommandRunner; - readonly localChecks: readonly { readonly command: string; readonly args: readonly string[] }[]; readonly runner?: CommandRunner; -}): Promise<{ - readonly status: 'admitted'; - readonly recipeVersion: typeof RECIPE_VERSION; - readonly sourceCommit: string; - readonly sourceTree: string; - readonly executors: readonly SolutionIsolationPolicy['executor'][]; - readonly localChecks: readonly { - readonly command: string; - readonly args: readonly string[]; - readonly exitCode: number; - }[]; -}> { +}): Promise { const runner = input.runner ?? runCommand; - const reasons: IsolationAdmissionReason[] = []; - inspectVerifier(input.verifier, input.forbiddenRoots, reasons); - inspectPolicies(input.policies, input.prefix.targetDir, input.forbiddenRoots, reasons); - inspectForbiddenRoots(input.policies, input.forbiddenRoots, reasons); - await inspectMaterializedRepository(input.prefix, runner, reasons); - await inspectTargetSymlinks(input.prefix.targetDir, reasons); - if (reasons.length > 0) throw new SolutionIsolationAdmissionError(reasons); - - for (const forbiddenRoot of input.forbiddenRoots) { - const result = await input.verifier.run('/bin/ls', ['-la', forbiddenRoot], { - cwd: input.prefix.targetDir, - timeoutMs: 15_000, - maxOutputBytes: 16 * 1024, - }); - if (result.exitCode === 0) { - throw new SolutionIsolationAdmissionError([ - { - code: 'path_boundary_weakened', - detail: `target verifier can read forbidden root: ${forbiddenRoot}`, - }, - ]); - } - } - - const curlReady = await input.verifier.run('/usr/bin/curl', ['--version'], { - cwd: input.prefix.targetDir, - timeoutMs: 15_000, - maxOutputBytes: 16 * 1024, - }); - if (curlReady.exitCode !== 0) { - throw new SolutionIsolationAdmissionError([ - { code: 'policy_weakened', detail: 'network-denied verifier cannot execute /usr/bin/curl' }, - ]); - } - - for (const url of new Set([...input.networkProbeUrls, ...REQUIRED_SOLUTION_PROBE_URLS])) { - const result = await input.verifier.run('/usr/bin/curl', ['--fail', '--silent', '--show-error', url], { - cwd: input.prefix.targetDir, - timeoutMs: 15_000, - maxOutputBytes: 16 * 1024, - }); - if (result.exitCode === 0) { - throw new SolutionIsolationAdmissionError([{ code: 'network_probe_reachable', detail: url }]); - } - } - - const localChecks = []; - for (const check of input.localChecks) { - const result = await input.verifier.run(check.command, check.args, { - cwd: input.prefix.targetDir, - timeoutMs: 10 * 60_000, - maxOutputBytes: 128 * 1024, - }); - if (result.exitCode !== 0) { - throw new SolutionIsolationAdmissionError([ - { - code: 'local_check_failed', - detail: `${[check.command, ...check.args].join(' ')} exited ${result.exitCode}`, - }, - ]); - } - localChecks.push({ ...check, exitCode: result.exitCode }); - } - - return { - status: 'admitted', - recipeVersion: RECIPE_VERSION, - sourceCommit: input.prefix.sourceCommit, - sourceTree: input.prefix.sourceTree, - executors: input.policies.map(({ executor }) => executor), - localChecks, - }; -} - -function inspectVerifier( - verifier: NetworkDeniedCommandRunner, - forbiddenRoots: readonly string[], - reasons: IsolationAdmissionReason[], -): void { - if ( - forbiddenRoots.length === 0 || - verifier.recipeVersion !== RECIPE_VERSION || - verifier.platform !== 'darwin' || - verifier[NETWORK_DENIED_RUNNER] !== true || - !forbiddenRoots.every((forbiddenRoot) => - verifier.forbiddenReadRoots.some((deniedRoot) => - containedPath(deniedRoot, canonicalPolicyPath(forbiddenRoot)), - ), - ) - ) { - reasons.push({ - code: 'policy_weakened', - detail: 'target verifier does not match isolation recipe v1', - }); - } -} - -function inspectPolicies( - policies: readonly SolutionIsolationPolicy[], - targetDir: string, - forbiddenRoots: readonly string[], - reasons: IsolationAdmissionReason[], -): void { - if (!sameStrings(policies.map(({ executor }) => executor).sort(), ['brunch', 'claude_code'])) { - reasons.push({ - code: 'policy_weakened', - detail: 'isolation recipe v1 requires exactly one Brunch and one Claude Code policy', - }); - } - for (const policy of policies) { - const deniesForbiddenRoots = - policy.executor !== 'claude_code' || - forbiddenRoots.every((forbiddenRoot) => - policy.nativeSandbox.deniedReadRoots.some((deniedRoot) => - containedPath(deniedRoot, canonicalPolicyPath(forbiddenRoot)), - ), - ); - if ( - resolve(policy.targetRoot) !== resolve(targetDir) || - !isStrictPolicy(policy) || - !deniesForbiddenRoots - ) { - reasons.push({ - code: 'policy_weakened', - detail: `${policy.executor} policy does not match isolation recipe v1`, - executor: policy.executor, - }); - } - } -} - -function isStrictPolicy(policy: SolutionIsolationPolicy): boolean { - if (policy.recipeVersion !== RECIPE_VERSION) return false; - if (policy.executor === 'claude_code') { - return ( - policy.strictMcp === true && - policy.mcpServers.length === 0 && - policy.webTools === false && - policy.ambientSettings === false && - policy.ambientPlugins === false && - policy.permissionMode === 'dontAsk' && - policy.nativeSandbox.enabled === true && - policy.nativeSandbox.failIfUnavailable === true && - policy.nativeSandbox.allowedDomains.length === 0 && - sameStrings(policy.nativeSandbox.deniedDomains, DENIED_SOLUTION_SOURCE_DOMAINS) && - policy.nativeSandbox.deniedReadRoots.every( - (deniedRoot) => !containedPath(deniedRoot, canonicalPolicyPath(policy.targetRoot)), - ) && - sameStrings(policy.allowedTools, ['Bash', 'Edit', 'Glob', 'Grep', 'Read', 'Write']) - ); - } - return ( - policy.foregroundWebTools === false && - policy.specifySubagents === false && - resolve(policy.foregroundFileRoot) === resolve(policy.targetRoot) && - sameStrings(policy.executionSubagents, ['planner', 'worker']) && - policy.networkDeniedVerifier === true - ); -} - -function inspectForbiddenRoots( - policies: readonly SolutionIsolationPolicy[], - forbiddenRoots: readonly string[], - reasons: IsolationAdmissionReason[], -): void { - for (const forbiddenRoot of forbiddenRoots) { - if ( - policies.some( - (policy) => - containedPath(policy.targetRoot, forbiddenRoot) || containedPath(forbiddenRoot, policy.targetRoot), - ) - ) { - reasons.push({ - code: 'path_boundary_weakened', - detail: `forbidden root is inside target boundary: ${forbiddenRoot}`, - }); - return; - } - for (const policy of policies) { - try { - assertTargetBoundedPath(policy, forbiddenRoot); - reasons.push({ - code: 'path_boundary_weakened', - detail: `${policy.executor} can reach forbidden root: ${forbiddenRoot}`, - executor: policy.executor, - }); - return; - } catch { - // Expected: every controller, harness, parent, and reference root is outside the target. - } - } - } -} - -async function inspectMaterializedRepository( - materialized: MaterializedHistoricalReplayPrefix, - runner: CommandRunner, - reasons: IsolationAdmissionReason[], -): Promise { const identity = JSON.parse( - await readFile(join(materialized.targetDir, SOURCE_IDENTITY_FILE), 'utf8'), + await readFile(join(input.prefix.targetDir, SOURCE_IDENTITY_FILE), 'utf8'), ) as Partial; if ( identity.recipeVersion !== RECIPE_VERSION || - identity.sourceCommit !== materialized.sourceCommit || - identity.sourceTree !== materialized.sourceTree + identity.sourceCommit !== input.prefix.sourceCommit || + identity.sourceTree !== input.prefix.sourceTree ) { - reasons.push({ - code: 'identity_mismatch', - detail: 'materialized source identity does not match declaration', - }); - } - const commitCount = ( - await gitChecked(runner, materialized.targetDir, ['rev-list', '--count', 'HEAD']) - ).stdout.trim(); - if (commitCount !== '2') { - reasons.push({ code: 'git_history_present', detail: `target has ${commitCount} reachable commits` }); - } - const head = (await gitChecked(runner, materialized.targetDir, ['rev-parse', 'HEAD'])).stdout.trim(); - if (head !== materialized.baseSha) { - reasons.push({ - code: 'identity_mismatch', - detail: 'historical replay HEAD does not match the declared packet child', - }); - } - const roots = ( - await gitChecked(runner, materialized.targetDir, ['rev-list', '--max-parents=0', materialized.baseSha]) - ).stdout - .trim() - .split('\n') - .filter(Boolean); - if (roots.length !== 1 || roots[0] !== materialized.syntheticCommit) { - reasons.push({ - code: 'identity_mismatch', - detail: 'historical replay root does not match the declared materialized source commit', - }); - } - const parent = ( - await gitChecked(runner, materialized.targetDir, ['rev-parse', `${materialized.baseSha}^`]) - ).stdout.trim(); - if (parent !== materialized.syntheticCommit) { - reasons.push({ - code: 'identity_mismatch', - detail: 'historical replay packet child has the wrong parent', - }); - } - const packetDelta = ( - await gitChecked(runner, materialized.targetDir, [ - 'diff', - '--name-only', - materialized.syntheticCommit, - materialized.baseSha, - ]) - ).stdout - .trim() - .split('\n') - .filter(Boolean) - .sort(); - const declaredPacketFiles = materialized.packetFiles.map(({ path }) => path).sort(); - if (!sameStrings(packetDelta, declaredPacketFiles)) { - reasons.push({ - code: 'identity_mismatch', - detail: `historical replay child is not packet-only: ${packetDelta.join(', ')}`, - }); - } - for (const file of materialized.packetFiles) { - const bytes = await readFile(join(materialized.targetDir, file.path)); - const digest = `sha256:${createHash('sha256').update(bytes).digest('hex')}`; - if (digest !== file.sha256) { - reasons.push({ - code: 'identity_mismatch', - detail: `historical replay packet drifted: ${file.path}`, - }); - } - } - const remotes = (await gitChecked(runner, materialized.targetDir, ['remote'])).stdout.trim(); - if (remotes.length > 0) { - reasons.push({ code: 'git_remote_present', detail: remotes }); + throw new Error('materialized source identity does not match the frozen case'); } - const refs = ( - await gitChecked(runner, materialized.targetDir, ['for-each-ref', '--format=%(refname)']) - ).stdout - .trim() - .split('\n') - .filter(Boolean); - const extraRefs = refs.filter((ref) => ref !== 'refs/heads/main'); - if (extraRefs.length > 0) { - reasons.push({ code: 'git_ref_present', detail: extraRefs.join(', ') }); - } - const worktreeChanges = ( - await gitChecked(runner, materialized.targetDir, ['status', '--porcelain', '--untracked-files=no']) + const remotes = (await gitChecked(runner, input.prefix.targetDir, ['remote'])).stdout.trim(); + if (remotes !== '') throw new Error('historical replay target must not have Git remotes'); + const trackedStatus = ( + await gitChecked(runner, input.prefix.targetDir, ['status', '--porcelain', '--untracked-files=no']) ).stdout.trim(); - if (worktreeChanges.length > 0) { - reasons.push({ code: 'git_worktree_changes_present', detail: worktreeChanges }); - } -} - -async function inspectTargetSymlinks(targetDir: string, reasons: IsolationAdmissionReason[]): Promise { - const root = await realpath(targetDir); - const inspectDirectory = async (directory: string): Promise => { - for (const entry of await readdir(directory, { withFileTypes: true })) { - if (entry.name === '.git' && directory === root) continue; - const path = join(directory, entry.name); - if (entry.isDirectory()) { - await inspectDirectory(path); - continue; - } - if (!entry.isSymbolicLink()) continue; - try { - const destination = await realpath(path); - if (!containedPath(root, destination)) { - reasons.push({ - code: 'path_boundary_weakened', - detail: `target symlink escapes isolation root: ${relative(root, path)}`, - }); - } - } catch { - reasons.push({ - code: 'path_boundary_weakened', - detail: `target symlink cannot be resolved safely: ${relative(root, path)}`, - }); - } - } - }; - await inspectDirectory(root); -} - -function containedPath(root: string, candidate: string): boolean { - const relation = relative(resolve(root), resolve(candidate)); - return relation === '' || (!relation.startsWith(`..${sep}`) && relation !== '..' && !isAbsolute(relation)); -} - -function sameStrings(actual: readonly string[], expected: readonly string[]): boolean { - return actual.length === expected.length && actual.every((value, index) => value === expected[index]); -} - -function canonicalPolicyPath(path: string): string { - const resolved = resolve(path); - try { - return realpathSync(resolved); - } catch { - const parent = dirname(resolved); - if (parent === resolved) return resolved; - return join(canonicalPolicyPath(parent), resolved.slice(parent.length + 1)); - } -} - -function denyTargetCommandAccessProfile(forbiddenReadRoots: readonly string[]): string { - const rules = ['(version 1)', '(allow default)', '(deny network*)']; - for (const root of forbiddenReadRoots) { - const literal = root.replaceAll('\\', '\\\\').replaceAll('"', '\\"'); - rules.push(`(deny file-read* file-write* (subpath "${literal}"))`); + if (trackedStatus !== '') throw new Error('historical replay target has tracked source changes'); + for (const file of input.prefix.packetFiles) { + const digest = `sha256:${createHash('sha256') + .update(await readFile(join(input.prefix.targetDir, file.path))) + .digest('hex')}`; + if (digest !== file.sha256) throw new Error(`historical replay packet drifted: ${file.path}`); } - return rules.join('\n'); } async function gitChecked( diff --git a/src/dev/execution-comparison-brunch.ts b/src/dev/execution-comparison-brunch.ts index 1d69b48c2..e43ea657b 100644 --- a/src/dev/execution-comparison-brunch.ts +++ b/src/dev/execution-comparison-brunch.ts @@ -14,12 +14,10 @@ import { type BrunchTuiLaunchContext, } from '../app/brunch-tui.js'; import { openWorkspaceCommandExecutor } from '../graph/index.js'; -import { createNetworkDeniedCommandRunner } from './end-to-end-comparison/solution-isolation.js'; export async function runPinnedBrunchExecutionTui(input: { readonly workspaceDir: string; readonly specId: number; - readonly forbiddenRoots: readonly string[]; readonly provider: 'anthropic'; readonly model: 'claude-opus-4-8'; }): Promise { @@ -35,7 +33,7 @@ export async function runPinnedBrunchExecutionTui(input: { webSidecarRunner: async () => null, runWorkspaceDialogPreflight: async () => preflight, launchInteractive: async (context) => { - await launchPinnedInteractive(context, model, input.forbiddenRoots); + await launchPinnedInteractive(context, model); }, }); } @@ -63,21 +61,13 @@ export async function resolvePinnedBrunchPreflight(input: { : { action: 'newSession', specId: input.specId }; } -async function launchPinnedInteractive( - context: BrunchTuiLaunchContext, - model: Model, - forbiddenRoots: readonly string[], -): Promise { +async function launchPinnedInteractive(context: BrunchTuiLaunchContext, model: Model): Promise { const agentDir = getAgentDir(); - const verifier = createNetworkDeniedCommandRunner({ - forbiddenReadRoots: forbiddenRoots, - }); const createRuntime = createBrunchAgentSessionRuntimeFactory({ ...context, allowSubagents: false, comparisonIsolation: { targetRoot: context.workspace.cwd, - verifier: verifier.run, }, agentServices: { model }, }); @@ -96,8 +86,6 @@ async function launchPinnedInteractive( export function parseExecutionComparisonArgs(args: readonly string[]): { readonly workspaceDir: string; readonly specId: number; - readonly forbiddenRoots: readonly string[]; - readonly solutionIsolation: 'v1'; } { const { values } = parseNodeArgs({ args: [...args], @@ -106,25 +94,14 @@ export function parseExecutionComparisonArgs(args: readonly string[]): { options: { workspace: { type: 'string' }, 'spec-id': { type: 'string' }, - 'solution-isolation': { type: 'string' }, - 'forbidden-root': { type: 'string', multiple: true }, }, }); const workspaceDir = values.workspace; const specId = Number(values['spec-id']); - const forbiddenRoots = values['forbidden-root'] ?? []; - if ( - !workspaceDir || - forbiddenRoots.length === 0 || - !Number.isSafeInteger(specId) || - specId <= 0 || - values['solution-isolation'] !== 'v1' - ) { - throw new Error( - 'Usage: execution-comparison-brunch --workspace --spec-id --solution-isolation v1 --forbidden-root [--forbidden-root ...]', - ); + if (!workspaceDir || !Number.isSafeInteger(specId) || specId <= 0) { + throw new Error('Usage: execution-comparison-brunch --workspace --spec-id '); } - return { workspaceDir, specId, forbiddenRoots, solutionIsolation: 'v1' }; + return { workspaceDir, specId }; } async function main(): Promise { @@ -132,7 +109,6 @@ async function main(): Promise { await runPinnedBrunchExecutionTui({ workspaceDir: args.workspaceDir, specId: args.specId, - forbiddenRoots: args.forbiddenRoots, provider: 'anthropic', model: 'claude-opus-4-8', }); diff --git a/src/dev/execution-comparison-operator.ts b/src/dev/execution-comparison-operator.ts index 152ed1bee..f60f643b0 100644 --- a/src/dev/execution-comparison-operator.ts +++ b/src/dev/execution-comparison-operator.ts @@ -24,10 +24,6 @@ import { loadControllerOracleManifest, loadControllerOraclePack, } from './execution-comparison/oracle-pack.js'; -import { - runPetrinautHistoricalPreflight, - type PetrinautHistoricalPreflightReceipt, -} from './execution-comparison/petrinaut-historical-preflight.js'; import { runPetrinautOptimizationOracle, type PetrinautOptimizationOracleReport, @@ -144,22 +140,7 @@ const DEFAULT_CASES_ROOT = fileURLToPath( ); const DEFAULT_CONTROLLER_ROOT = fileURLToPath(new URL('../../', import.meta.url)); -interface ExecutionComparisonOperatorDependencies { - readonly runPetrinautPreflight?: (input: { - readonly sourceRepositoryDir: string; - readonly parentTargetDir: string; - readonly referenceTargetDir: string; - readonly outputRoot: string; - }) => Promise<{ - readonly receiptFile: string; - readonly receipt: PetrinautHistoricalPreflightReceipt; - }>; -} - -export async function runExecutionComparisonOperatorCli( - args: readonly string[], - dependencies: ExecutionComparisonOperatorDependencies = {}, -): Promise { +export async function runExecutionComparisonOperatorCli(args: readonly string[]): Promise { const [command, ...rest] = args; const options = parseOptions(rest); const casesRoot = DEFAULT_CASES_ROOT; @@ -173,6 +154,8 @@ export async function runExecutionComparisonOperatorCli( case 'inspect': { assertOnlyOptions(options, ['case']); const selected = await resolveExecutionCase(required(options, 'case'), casesRoot); + const manifest = await loadControllerOracleManifest(selected.caseDir); + const repository = selected.packet.contract.case.repository; process.stdout.write( `${JSON.stringify( { @@ -180,6 +163,9 @@ export async function runExecutionComparisonOperatorCli( caseId: selected.caseId, caseDir: selected.caseDir, publicPacketSha256: selected.packet.packetSha256, + oracleId: manifest.id, + repository, + requiresSourceRepository: repository.substrate === 'pinned_git', files: selected.packet.files, sharedFraming: EXECUTION_COMPARISON_SHARED_FRAMING, specification: await readFile(join(selected.caseDir, 'spec.md'), 'utf8'), @@ -197,14 +183,16 @@ export async function runExecutionComparisonOperatorCli( if (lane !== 'brunch' && lane !== 'claude_code') { throw new Error('--lane must be brunch or claude_code'); } - const sourceRepository = options.get('source-repository'); + const sourceRepository = options.has('source-repository') + ? requiredAbsolute(options, 'source-repository') + : undefined; const prepared = await prepareExecutionTarget({ lane, caseReference: required(options, 'case'), casesRoot, targetDir: resolve(required(options, 'target')), controllerRoot: DEFAULT_CONTROLLER_ROOT, - ...(sourceRepository === undefined ? {} : { sourceRepositoryDir: resolve(sourceRepository) }), + ...(sourceRepository === undefined ? {} : { sourceRepositoryDir: sourceRepository }), }); process.stdout.write(`${JSON.stringify(prepared, null, 2)}\n`); return; @@ -232,39 +220,6 @@ export async function runExecutionComparisonOperatorCli( process.stdout.write(`${JSON.stringify(retained)}\n`); return; } - case 'petrinaut-preflight': { - assertOnlyOptions(options, ['source-repository', 'parent-target', 'reference-target', 'output-root']); - const input = { - sourceRepositoryDir: requiredAbsolute(options, 'source-repository'), - parentTargetDir: requiredAbsolute(options, 'parent-target'), - referenceTargetDir: requiredAbsolute(options, 'reference-target'), - outputRoot: requiredAbsolute(options, 'output-root'), - }; - const result = await ( - dependencies.runPetrinautPreflight ?? - (async (closedInput) => - await runPetrinautHistoricalPreflight({ - sourceRepositoryDir: closedInput.sourceRepositoryDir, - parentTargetDir: closedInput.parentTargetDir, - referenceTargetDir: closedInput.referenceTargetDir, - controllerRoot: DEFAULT_CONTROLLER_ROOT, - receiptFile: join(closedInput.outputRoot, 'petrinaut-historical-preflight-receipt.json'), - })) - )(input); - process.stdout.write( - `${JSON.stringify({ - receiptFile: result.receiptFile, - status: result.receipt.status, - setupStatus: result.receipt.setupStatus, - })}\n`, - ); - if (result.receipt.status !== 'passed') { - throw new Error( - `Petrinaut preflight ${result.receipt.status}; retained receipt at ${result.receiptFile}`, - ); - } - return; - } case 'retain-attempt': { assertOnlyOptions(options, ['attempt-file', 'attempts-root']); const value = JSON.parse(await readFile(resolve(required(options, 'attempt-file')), 'utf8')) as unknown; @@ -277,7 +232,7 @@ export async function runExecutionComparisonOperatorCli( } default: throw new Error( - 'Usage: execution-comparison-operator [options]', + 'Usage: execution-comparison-operator [options]', ); } } diff --git a/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts b/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts index 0b702bfcf..92fb97a56 100644 --- a/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts +++ b/src/dev/execution-comparison/__tests__/compare-execution-prompt.test.ts @@ -51,9 +51,11 @@ describe('/compare-execution operator prompt', () => { expect(prompt).toContain('process status'); expect(prompt).toContain('final tree and complete base-to-tip diff'); expect(prompt).toContain('visible interaction evidence'); - expect(prompt).toContain('`petri-editor-browser-v2`'); + expect(prompt).toContain('oracle identity returned by `inspect`'); + expect(prompt).toContain('[--source-repository ]'); + expect(prompt).toContain('`/comparison-publish`'); expect(prompt).toContain('after every lane terminates'); - expect(prompt).toContain('Present validity before outcomes'); + expect(prompt).toContain('validity before outcomes'); expect(prompt).toContain('Do not score'); expect(prompt).toContain('diagnostic-only'); }); diff --git a/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts b/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts index ae867ed39..6558dcbe7 100644 --- a/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts +++ b/src/dev/execution-comparison/__tests__/execution-comparison-brunch.test.ts @@ -17,67 +17,20 @@ afterEach(async () => { describe('execution comparison Brunch CLI arguments', () => { it('parses a complete workspace and positive specification id', () => { - expect( - parseExecutionComparisonArgs([ - '--workspace', - '/tmp/petri-editor', - '--spec-id', - '17', - '--solution-isolation', - 'v1', - '--forbidden-root', - '/tmp/controller', - ]), - ).toEqual({ + expect(parseExecutionComparisonArgs(['--workspace', '/tmp/petri-editor', '--spec-id', '17'])).toEqual({ workspaceDir: '/tmp/petri-editor', specId: 17, - forbiddenRoots: ['/tmp/controller'], - solutionIsolation: 'v1', }); }); it('rejects another option where the workspace value is required', () => { - expect(() => - parseExecutionComparisonArgs([ - '--workspace', - '--spec-id', - '17', - '--solution-isolation', - 'v1', - '--forbidden-root', - '/tmp/controller', - ]), - ).toThrow(); + expect(() => parseExecutionComparisonArgs(['--workspace', '--spec-id', '17'])).toThrow(); }); it('rejects missing, non-integer, and unknown options', () => { expect(() => parseExecutionComparisonArgs(['--workspace', '/tmp/petri-editor'])).toThrow('Usage:'); expect(() => - parseExecutionComparisonArgs([ - '--workspace', - '/tmp/petri-editor', - '--spec-id', - '1.5', - '--solution-isolation', - 'v1', - '--forbidden-root', - '/tmp/controller', - ]), - ).toThrow('Usage:'); - expect(() => - parseExecutionComparisonArgs(['--workspace', '/tmp/petri-editor', '--spec-id', '1']), - ).toThrow('Usage:'); - expect(() => - parseExecutionComparisonArgs([ - '--workspace', - '/tmp/petri-editor', - '--spec-id', - '1', - '--solution-isolation', - 'prompt-only', - '--forbidden-root', - '/tmp/controller', - ]), + parseExecutionComparisonArgs(['--workspace', '/tmp/petri-editor', '--spec-id', '1.5']), ).toThrow('Usage:'); expect(() => parseExecutionComparisonArgs(['--unknown', 'value'])).toThrow(); }); diff --git a/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts b/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts index 6afd30d58..6ac3cc097 100644 --- a/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts +++ b/src/dev/execution-comparison/__tests__/historical-replay-target.test.ts @@ -1,46 +1,27 @@ import { createHash } from 'node:crypto'; -import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; import { afterEach, describe, expect, it } from 'vitest'; -import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; -import { - assertExecuteProjectionPlanReady, - projectExecuteGraph, -} from '../../../executor/execute-projection.js'; +import { runCommand } from '../../../app/command-runner.js'; import { openWorkspaceDb } from '../../../graph/index.js'; import { queryGraph } from '../../../graph/queries.js'; -import { runClaudeExecutionWorkspace } from '../../end-to-end-comparison/claude-adapter.js'; -import { - createNetworkDeniedCommandRunner, - type NetworkDeniedCommandRunner, -} from '../../end-to-end-comparison/solution-isolation.js'; -import { isPetrinautOptimizationExecutionCaseContract } from '../case-contract.js'; -import { - HistoricalReplayTargetPreparationError, - prepareHistoricalReplayTarget, - type HistoricalReplayTargetDependencies, -} from '../historical-replay-target.js'; +import { prepareHistoricalReplayTarget } from '../historical-replay-target.js'; import { prepareExecutionTarget, resolveExecutionCase } from '../operator-cli.js'; const roots: string[] = []; -const frozenCasesRoot = fileURLToPath( - new URL('../../../../testing/execution-comparisons/cases/', import.meta.url), -); afterEach(async () => { await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); }); -describe('historical replay target preparation', () => { - it('admits the exact two-commit prefix and returns a launchable Brunch target without installing', async () => { +describe('learning-first historical replay preparation', () => { + it('creates a remote-free Brunch target from the pinned tree and exact packet', async () => { const fixture = await createBrunchFixture(); const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); let installCalled = false; - const ready = await prepareHistoricalReplayTarget( { lane: 'brunch', @@ -49,45 +30,28 @@ describe('historical replay target preparation', () => { targetDir: fixture.targetDir, controllerRoot: fixture.controllerDir, }, - portableDependencies({ + { dependencyInstallRunner: async () => { installCalled = true; return { exitCode: 0, stdout: '', stderr: '' }; }, - }), + }, ); - if (ready.lane !== 'brunch') throw new Error('expected a Brunch-ready target'); + if (ready.lane !== 'brunch') throw new Error('expected Brunch target'); expect(ready).toMatchObject({ status: 'ready', - lane: 'brunch', caseId: 'brunch-host-landing-v1', sourceCommit: fixture.sourceCommit, sourceTree: fixture.sourceTree, - specId: expect.any(Number), dependencyPreparation: { recipe: 'none', status: 'not_required' }, - isolationPolicy: { executor: 'brunch' }, launch: { command: 'npx', - args: expect.arrayContaining([ - '--workspace', - fixture.targetDir, - '--spec-id', - expect.any(String), - '--solution-isolation', - 'v1', - ]), + args: expect.arrayContaining(['--workspace', fixture.targetDir, '--spec-id']), }, }); - expect(ready.specId).toBeGreaterThan(0); - expect(ready.baseSha).toBe(await git(fixture.targetDir, ['rev-parse', 'HEAD'])); - expect(await git(fixture.targetDir, ['rev-list', '--count', ready.baseSha])).toBe('2'); - expect(await git(fixture.targetDir, ['rev-parse', `${ready.baseSha}^`])).toBe(ready.materializedCommit); - expect( - (await git(fixture.targetDir, ['diff', '--name-only', ready.materializedCommit, ready.baseSha])).split( - '\n', - ), - ).toEqual(['public-contract.json', 'spec.md']); + expect(ready.launch.args).not.toContain('--solution-isolation'); + expect(await git(fixture.targetDir, ['remote'])).toBe(''); expect(await readFile(join(fixture.targetDir, 'spec.md'))).toEqual(fixture.specification); expect(installCalled).toBe(false); const graph = queryGraph(await openWorkspaceDb(fixture.targetDir), ready.specId); @@ -95,418 +59,45 @@ describe('historical replay target preparation', () => { kind: 'requirement', body: fixture.specification.toString('utf8'), }); - const projection = projectExecuteGraph({ - specId: ready.specId, - graphLsn: graph.lsn, - mode: 'brownfield', - nodes: graph.nodes, - edges: graph.edges, - }); - expect(() => assertExecuteProjectionPlanReady(projection)).not.toThrow(); }, 30_000); - it('executes the production deep operation through an injected branded verifier factory', async () => { + it('returns a restricted Claude launch from the same pinned packet', async () => { const fixture = await createBrunchFixture(); - const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); - let factoryCalls = 0; - - const ready = await prepareHistoricalReplayTarget( - { - lane: 'claude_code', - selectedCase: selected, - sourceRepositoryDir: fixture.sourceDir, - targetDir: fixture.targetDir, - controllerRoot: fixture.controllerDir, - }, - { - createVerifier: (forbiddenReadRoots: readonly string[]) => { - factoryCalls += 1; - return createNetworkDeniedCommandRunner({ - platform: 'darwin', - forbiddenReadRoots, - run: fakeSandboxRunner, - }); - }, - }, - ); - - expect(factoryCalls).toBe(1); - expect(ready).toMatchObject({ - status: 'ready', + const ready = await prepareExecutionTarget({ lane: 'claude_code', - baseSha: expect.stringMatching(/^[a-f0-9]{40}$/u), + caseReference: 'brunch-host-landing', + casesRoot: fixture.casesRoot, + sourceRepositoryDir: fixture.sourceDir, + targetDir: fixture.targetDir, + controllerRoot: fixture.controllerDir, }); - }); - - it('rejects an injected raw verifier that lacks the existing runtime brand', async () => { - const fixture = await createBrunchFixture(); - const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); + if (ready.lane !== 'claude_code' || ready.preparation !== 'historical_replay') { + throw new Error('expected historical Claude target'); + } - const rejected = prepareHistoricalReplayTarget( - { - lane: 'claude_code', - selectedCase: selected, - sourceRepositoryDir: fixture.sourceDir, - targetDir: fixture.targetDir, - controllerRoot: fixture.controllerDir, - }, - portableDependencies({ - createVerifier: (forbiddenReadRoots) => - ({ - recipeVersion: 1, - platform: 'darwin', - forbiddenReadRoots, - run: fakeSandboxRunner, - }) as unknown as NetworkDeniedCommandRunner, - }), + expect(ready.launch.args).toEqual( + expect.arrayContaining(['--strict-mcp-config', '--disallowedTools', 'WebFetch,WebSearch']), ); + expect(JSON.stringify(ready.launch)).toContain(fixture.sourceDir); + expect(await git(fixture.targetDir, ['remote'])).toBe(''); + }, 30_000); - await expect(rejected).rejects.toMatchObject({ - status: 'setup_failed', - phase: 'admission', - reasons: expect.arrayContaining([expect.objectContaining({ code: 'policy_weakened' })]), - }); - }); - - it('rejects readiness when a branded verifier reaches a required network probe', async () => { + it('removes an owned target when pinned source identity is wrong', async () => { const fixture = await createBrunchFixture(); const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); - let reachedRequiredProbe = false; - const reachableSandboxRunner: CommandRunner = async (command, args) => { - if (command !== 'sandbox-exec') { - throw new Error(`unexpected verifier command: ${command}`); - } - const verifiedCommand = args[2]; - if (verifiedCommand === '/usr/bin/curl' && args[3] === '--version') { - return { exitCode: 0, stdout: 'curl fake\n', stderr: '' }; - } - if (verifiedCommand === '/usr/bin/curl' && args.at(-1) === 'https://github.com') { - reachedRequiredProbe = true; - return { exitCode: 0, stdout: 'reachable\n', stderr: '' }; - } - return { exitCode: 1, stdout: '', stderr: 'denied by fake sandbox\n' }; - }; + Object.assign(selected.packet.contract.case.repository, { parentTree: 'f'.repeat(40) }); - const rejected = prepareHistoricalReplayTarget( - { + await expect( + prepareHistoricalReplayTarget({ lane: 'claude_code', selectedCase: selected, sourceRepositoryDir: fixture.sourceDir, targetDir: fixture.targetDir, controllerRoot: fixture.controllerDir, - }, - { - createVerifier: (forbiddenReadRoots) => - createNetworkDeniedCommandRunner({ - platform: 'darwin', - forbiddenReadRoots, - run: reachableSandboxRunner, - }), - }, - ); - - await expect(rejected).rejects.toMatchObject({ - status: 'setup_failed', - phase: 'admission', - reasons: [ - expect.objectContaining({ - code: 'network_probe_reachable', - detail: 'https://github.com', - }), - ], - }); - expect(reachedRequiredProbe).toBe(true); - await expect(readFile(join(fixture.targetDir, 'spec.md'))).rejects.toMatchObject({ code: 'ENOENT' }); - }); - - it('dispatches pinned preparation by repository substrate for the Brunch host-landing case', async () => { - const fixture = await createBrunchFixture(); - - const ready = await prepareExecutionTarget( - { - lane: 'claude_code', - caseReference: 'brunch-host-landing', - casesRoot: fixture.casesRoot, - sourceRepositoryDir: fixture.sourceDir, - targetDir: fixture.targetDir, - controllerRoot: fixture.controllerDir, - }, - portableDependencies({ - dependencyInstallRunner: async () => { - throw new Error('Brunch host landing must not install dependencies'); - }, }), - ); - - expect(ready).toMatchObject({ - status: 'ready', - preparation: 'historical_replay', - lane: 'claude_code', - caseId: 'brunch-host-landing-v1', - dependencyPreparation: { recipe: 'none', status: 'not_required' }, - isolationPolicy: { - executor: 'claude_code', - strictMcp: true, - webTools: false, - ambientSettings: false, - ambientPlugins: false, - permissionMode: 'dontAsk', - }, - launch: { command: 'claude', cwd: fixture.targetDir }, - }); - expect('specId' in ready).toBe(false); - if (ready.preparation !== 'historical_replay' || ready.lane !== 'claude_code') { - throw new Error('expected a Claude-ready historical replay target'); - } - const run = await runClaudeExecutionWorkspace( - { - prepared: ready, - evidenceDir: join(fixture.controllerDir, 'evidence'), - elapsedMinutes: 1, - }, - async (command, args, options) => { - if (command === 'claude') { - await writeFile(join(options.cwd, 'candidate.ts'), 'export const candidate = true;\n'); - return { exitCode: 0, stdout: '{"type":"result","subtype":"success"}\n', stderr: '' }; - } - return await runCommand(command, args, options); - }, - ); - const [rangeBase, rangeReview, rangeRemainder] = run.repository?.finalGitRange.split('..') ?? []; - expect(rangeBase).toBe(ready.baseSha); - expect(rangeReview).toMatch(/^[a-f0-9]{40}$/u); - expect(rangeRemainder).toBeUndefined(); - }, 30_000); - - it.each([ - ['wrong parent', rewritePrefixWithWrongParent, 'identity_mismatch'], - ['packet drift', driftPacketCommit, 'identity_mismatch'], - ['third commit', appendThirdCommit, 'git_history_present'], - ['remote and later ref', addRemoteAndRef, 'git_remote_present'], - ['escaping symlink', addEscapingSymlink, 'path_boundary_weakened'], - ] as const)( - 'rejects a %s before lane readiness and removes the partial target', - async (_name, mutate, reason) => { - const fixture = await createBrunchFixture(); - const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); - let mutated = false; - const runner: CommandRunner = async (command, args, options) => { - if ( - !mutated && - command === 'git' && - args[0] === 'rev-list' && - args[1] === '--count' && - args[2] === 'HEAD' && - options.cwd === fixture.targetDir - ) { - mutated = true; - await mutate(fixture.targetDir); - } - return await runCommand(command, args, options); - }; - - const rejected = prepareHistoricalReplayTarget( - { - lane: 'claude_code', - selectedCase: selected, - sourceRepositoryDir: fixture.sourceDir, - targetDir: fixture.targetDir, - controllerRoot: fixture.controllerDir, - }, - portableDependencies({ runner }), - ); - - await expect(rejected).rejects.toMatchObject({ - status: 'setup_failed', - phase: 'admission', - reasons: expect.arrayContaining([expect.objectContaining({ code: reason })]), - }); - await expect(readFile(join(fixture.targetDir, 'spec.md'))).rejects.toMatchObject({ code: 'ENOENT' }); - }, - 30_000, - ); - - it('retains path-boundary setup evidence without returning a partial launch descriptor', async () => { - const fixture = await createBrunchFixture(); - const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); - - const rejected = prepareHistoricalReplayTarget( - { - lane: 'brunch', - selectedCase: selected, - sourceRepositoryDir: fixture.sourceDir, - targetDir: fixture.targetDir, - controllerRoot: fixture.controllerDir, - forbiddenRoots: [join(fixture.targetDir, 'nested-controller')], - }, - portableDependencies(), - ); - - await expect(rejected).rejects.toBeInstanceOf(HistoricalReplayTargetPreparationError); - await expect(rejected).rejects.toMatchObject({ - status: 'setup_failed', - phase: 'admission', - reasons: expect.arrayContaining([expect.objectContaining({ code: 'path_boundary_weakened' })]), - }); + ).rejects.toMatchObject({ status: 'setup_failed', phase: 'source_materialization' }); await expect(readFile(join(fixture.targetDir, 'spec.md'))).rejects.toMatchObject({ code: 'ENOENT' }); - }, 30_000); - - it('rejects a pre-existing target without deleting caller-owned files', async () => { - const fixture = await createBrunchFixture(); - const selected = await resolveExecutionCase('brunch-host-landing', fixture.casesRoot); - await mkdir(fixture.targetDir); - await writeFile(join(fixture.targetDir, 'keep.txt'), 'caller owned\n'); - - await expect( - prepareHistoricalReplayTarget( - { - lane: 'brunch', - selectedCase: selected, - sourceRepositoryDir: fixture.sourceDir, - targetDir: fixture.targetDir, - controllerRoot: fixture.controllerDir, - }, - portableDependencies(), - ), - ).rejects.toMatchObject({ - status: 'setup_failed', - phase: 'source_materialization', - }); - await expect(readFile(join(fixture.targetDir, 'keep.txt'), 'utf8')).resolves.toBe('caller owned\n'); }); - - it('admits non-ignored untracked Petrinaut dependency artifacts when tracked source stays clean', async () => { - const root = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-replay-')); - roots.push(root); - const sourceDir = join(root, 'source'); - const controllerDir = join(root, 'controller'); - const targetDir = join(root, 'target'); - await Promise.all([mkdir(sourceDir), mkdir(controllerDir)]); - const selected = await resolveExecutionCase('petrinaut-optimization-v1', frozenCasesRoot); - if (!isPetrinautOptimizationExecutionCaseContract(selected.packet.contract)) { - throw new Error('expected the frozen Petrinaut contract'); - } - const runner = createSyntheticPinnedGitRunner({ - sourceDir, - targetDir, - sourceCommit: selected.packet.contract.case.repository.parentCommit, - sourceTree: selected.packet.contract.case.repository.parentTree, - reportUntrackedArtifact: true, - }); - const installCalls: { command: string; args: readonly string[]; cwd: string }[] = []; - - const ready = await prepareHistoricalReplayTarget( - { - lane: 'claude_code', - selectedCase: selected, - sourceRepositoryDir: sourceDir, - targetDir, - controllerRoot: controllerDir, - }, - portableDependencies({ - runner, - dependencyInstallRunner: async (command, args, options) => { - installCalls.push({ command, args, cwd: options.cwd }); - await writeFile(join(options.cwd, '.pnp.cjs'), '// generated dependency artifact\n'); - return { exitCode: 0, stdout: '', stderr: '' }; - }, - }), - ); - - expect(installCalls).toEqual([ - { - command: 'corepack', - args: ['yarn', 'install', '--immutable', '--mode=skip-build'], - cwd: targetDir, - }, - ]); - await expect(readFile(join(targetDir, '.pnp.cjs'), 'utf8')).resolves.toBe( - '// generated dependency artifact\n', - ); - expect(ready).toMatchObject({ - status: 'ready', - lane: 'claude_code', - dependencyPreparation: { - recipe: 'petrinaut-yarn-immutable-v1', - command: 'corepack', - args: ['yarn', 'install', '--immutable', '--mode=skip-build'], - status: 'passed', - exitCode: 0, - }, - isolationPolicy: { - executor: 'claude_code', - strictMcp: true, - mcpServers: [], - webTools: false, - ambientSettings: false, - ambientPlugins: false, - permissionMode: 'dontAsk', - nativeSandbox: { - enabled: true, - failIfUnavailable: true, - allowedDomains: [], - }, - }, - launch: { command: 'claude', cwd: targetDir }, - }); - expect('specId' in ready).toBe(false); - }, 30_000); - - it('rejects Petrinaut dependency preparation that mutates tracked source', async () => { - const root = await mkdtemp(join(tmpdir(), 'brunch-petrinaut-mutation-')); - roots.push(root); - const sourceDir = join(root, 'source'); - const controllerDir = join(root, 'controller'); - const targetDir = join(root, 'target'); - await Promise.all([mkdir(sourceDir), mkdir(controllerDir)]); - const selected = await resolveExecutionCase('petrinaut-optimization-v1', frozenCasesRoot); - if (!isPetrinautOptimizationExecutionCaseContract(selected.packet.contract)) { - throw new Error('expected the frozen Petrinaut contract'); - } - - const rejected = prepareHistoricalReplayTarget( - { - lane: 'claude_code', - selectedCase: selected, - sourceRepositoryDir: sourceDir, - targetDir, - controllerRoot: controllerDir, - }, - portableDependencies({ - runner: createSyntheticPinnedGitRunner({ - sourceDir, - targetDir, - sourceCommit: selected.packet.contract.case.repository.parentCommit, - sourceTree: selected.packet.contract.case.repository.parentTree, - detectTrackedMutation: true, - }), - dependencyInstallRunner: async (_command, _args, options) => { - await writeFile(join(options.cwd, 'package.json'), '{"mutated":true}\n'); - return { exitCode: 0, stdout: '', stderr: '' }; - }, - }), - ); - - await expect(rejected).rejects.toMatchObject({ - status: 'setup_failed', - phase: 'dependency_preparation', - cause: { - outcome: { - status: 'failed', - exitCode: 0, - failureStage: 'tracked_source_cleanliness', - }, - observation: { - commandResult: { - exitCode: 0, - }, - trackedSourceStatus: 'M package.json', - }, - }, - }); - await expect(rejected).rejects.toThrow('modified tracked source'); - await expect(readFile(join(targetDir, 'package.json'))).rejects.toMatchObject({ code: 'ENOENT' }); - }, 30_000); }); async function createBrunchFixture(): Promise<{ @@ -558,40 +149,18 @@ async function createBrunchFixture(): Promise<{ mode: 'brownfield', scope: 'single_feature', surface: 'backend', - repository: { - substrate: 'pinned_git', - parentCommit: sourceCommit, - parentTree: sourceTree, - }, - }, - budgets: { - elapsedMinutes: 90, - mechanicalInterventions: 2, - substantiveHumanInterventions: 0, - }, - delivery: { - runtimeNetwork: 'forbidden', - dependencyInstallNetwork: 'forbidden', - }, - acceptance: { - publicCommand: '/brunch:land', - executionTerminal: 'promotion_prepared', + repository: { substrate: 'pinned_git', parentCommit: sourceCommit, parentTree: sourceTree }, }, + budgets: { elapsedMinutes: 90, mechanicalInterventions: 2, substantiveHumanInterventions: 0 }, + delivery: { runtimeNetwork: 'forbidden', dependencyInstallNetwork: 'forbidden' }, + acceptance: { publicCommand: '/brunch:land', executionTerminal: 'promotion_prepared' }, rules: ['Work only in the target repository.', 'Stop after promotion_prepared without landing.'], }, null, 2, )}\n`, ); - return { - casesRoot, - controllerDir, - sourceDir, - sourceCommit, - sourceTree, - targetDir, - specification, - }; + return { casesRoot, controllerDir, sourceDir, sourceCommit, sourceTree, targetDir, specification }; } async function git(cwd: string, args: readonly string[]): Promise { @@ -599,174 +168,3 @@ async function git(cwd: string, args: readonly string[]): Promise { if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); return result.stdout.trim(); } - -async function appendThirdCommit(targetDir: string): Promise { - await git(targetDir, [ - '-c', - 'user.name=Historical Rival', - '-c', - 'user.email=historical-rival@example.invalid', - 'commit', - '--allow-empty', - '-m', - 'Third commit rival', - ]); -} - -async function driftPacketCommit(targetDir: string): Promise { - await writeFile(join(targetDir, 'spec.md'), '# Drifted packet\n'); - await git(targetDir, ['add', '--', 'spec.md']); - await git(targetDir, [ - '-c', - 'user.name=Historical Rival', - '-c', - 'user.email=historical-rival@example.invalid', - 'commit', - '--amend', - '--no-edit', - ]); -} - -async function rewritePrefixWithWrongParent(targetDir: string): Promise { - const originalRootTree = await git(targetDir, ['rev-parse', 'HEAD^^{tree}']); - const finalTree = await git(targetDir, ['rev-parse', 'HEAD^{tree}']); - const replacementRoot = await git(targetDir, [ - '-c', - 'user.name=Historical Rival', - '-c', - 'user.email=historical-rival@example.invalid', - 'commit-tree', - originalRootTree, - '-m', - 'Replacement root', - ]); - const replacementChild = await git(targetDir, [ - '-c', - 'user.name=Historical Rival', - '-c', - 'user.email=historical-rival@example.invalid', - 'commit-tree', - finalTree, - '-p', - replacementRoot, - '-m', - 'Replacement packet child', - ]); - await git(targetDir, ['update-ref', 'refs/heads/main', replacementChild]); -} - -async function addRemoteAndRef(targetDir: string): Promise { - await git(targetDir, ['remote', 'add', 'origin', 'https://github.com/hashintel/brunch.git']); - await git(targetDir, ['branch', 'later-solution']); -} - -async function addEscapingSymlink(targetDir: string): Promise { - await symlink('../controller', join(targetDir, 'escaped-controller')); -} - -function createSyntheticPinnedGitRunner(input: { - readonly sourceDir: string; - readonly targetDir: string; - readonly sourceCommit: string; - readonly sourceTree: string; - readonly detectTrackedMutation?: boolean; - readonly reportUntrackedArtifact?: boolean; -}): CommandRunner { - const materializedCommit = 'a'.repeat(40); - const baseSha = 'b'.repeat(40); - let commits = 0; - return async (command, args, options) => { - if (command === 'tar') { - await writeFile(join(options.cwd, 'package.json'), '{"private":true}\n'); - return { exitCode: 0, stdout: '', stderr: '' }; - } - if (command !== 'git') { - return await runCommand(command, args, options); - } - if (args[0] === 'archive') { - const output = args.find((arg) => arg.startsWith('--output='))?.slice('--output='.length); - if (output === undefined) throw new Error('synthetic archive call omitted --output'); - await writeFile(output, 'synthetic archive'); - return { exitCode: 0, stdout: '', stderr: '' }; - } - if (args[0] === 'rev-parse' && options.cwd === input.sourceDir) { - const revision = args.at(-1); - if (revision === `${input.sourceCommit}^{commit}`) { - return { exitCode: 0, stdout: `${input.sourceCommit}\n`, stderr: '' }; - } - if (revision === `${input.sourceCommit}^{tree}`) { - return { exitCode: 0, stdout: `${input.sourceTree}\n`, stderr: '' }; - } - } - if (args.includes('commit')) { - commits += 1; - return { exitCode: 0, stdout: '', stderr: '' }; - } - if (args[0] === 'rev-parse') { - const revision = args.at(-1); - if (revision === 'HEAD') { - return { - exitCode: 0, - stdout: `${commits === 1 ? materializedCommit : baseSha}\n`, - stderr: '', - }; - } - if (revision === `${baseSha}^`) { - return { exitCode: 0, stdout: `${materializedCommit}\n`, stderr: '' }; - } - } - if (args[0] === 'rev-list' && args[1] === '--count') { - return { exitCode: 0, stdout: '2\n', stderr: '' }; - } - if (args[0] === 'rev-list' && args[1] === '--max-parents=0') { - return { exitCode: 0, stdout: `${materializedCommit}\n`, stderr: '' }; - } - if (args[0] === 'diff') { - return { exitCode: 0, stdout: 'public-contract.json\nspec.md\n', stderr: '' }; - } - if (args[0] === 'for-each-ref') { - return { exitCode: 0, stdout: 'refs/heads/main\n', stderr: '' }; - } - if (args[0] === 'status') { - if ( - input.detectTrackedMutation === true && - (await readFile(join(input.targetDir, 'package.json'), 'utf8')) !== '{"private":true}\n' - ) { - return { exitCode: 0, stdout: ' M package.json\n', stderr: '' }; - } - if (input.reportUntrackedArtifact === true && !args.includes('--untracked-files=no')) { - return { exitCode: 0, stdout: '?? .pnp.cjs\n', stderr: '' }; - } - return { exitCode: 0, stdout: '', stderr: '' }; - } - if (args[0] === 'remote') { - return { exitCode: 0, stdout: '', stderr: '' }; - } - return { exitCode: 0, stdout: '', stderr: '' }; - }; -} - -function portableDependencies( - overrides: HistoricalReplayTargetDependencies = {}, -): HistoricalReplayTargetDependencies { - return { - createVerifier: (forbiddenReadRoots) => - createNetworkDeniedCommandRunner({ - platform: 'darwin', - forbiddenReadRoots, - run: fakeSandboxRunner, - }), - ...overrides, - }; -} - -const fakeSandboxRunner: CommandRunner = async (command, args) => { - if (command !== 'sandbox-exec') { - throw new Error(`unexpected verifier command: ${command}`); - } - const verifiedCommand = args[2]; - if (verifiedCommand === '/usr/bin/curl' && args[3] === '--version') { - return { exitCode: 0, stdout: 'curl fake\n', stderr: '' }; - } - return { exitCode: 1, stdout: '', stderr: 'denied by fake sandbox\n' }; -}; diff --git a/src/dev/execution-comparison/__tests__/operator-cli.test.ts b/src/dev/execution-comparison/__tests__/operator-cli.test.ts index 5bfe45d57..4b65c528b 100644 --- a/src/dev/execution-comparison/__tests__/operator-cli.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-cli.test.ts @@ -242,144 +242,3 @@ describe('execution comparison oracle CLI', () => { } }); }); - -describe('Petrinaut preflight operator command', () => { - it('requires absolute closed roots and dispatches no provider or arbitrary command input', async () => { - const root = await mkdtemp(join(tmpdir(), 'brunch-preflight-cli-')); - const sourceRepositoryDir = join(root, 'source'); - const workRoot = join(root, 'work'); - const outputRoot = join(root, 'evidence'); - await Promise.all([mkdir(sourceRepositoryDir), mkdir(workRoot), mkdir(outputRoot)]); - const parentTargetDir = join(workRoot, 'parent'); - const referenceTargetDir = join(workRoot, 'reference'); - const calls: unknown[] = []; - const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); - try { - await runExecutionComparisonOperatorCli( - [ - 'petrinaut-preflight', - '--source-repository', - sourceRepositoryDir, - '--parent-target', - parentTargetDir, - '--reference-target', - referenceTargetDir, - '--output-root', - outputRoot, - ], - { - runPetrinautPreflight: async (input) => { - calls.push(input); - return { - receiptFile: join(outputRoot, 'petrinaut-historical-preflight-receipt.json'), - receipt: { - schemaVersion: 1, - caseId: 'petrinaut-optimization-v1', - status: 'passed', - setupStatus: 'valid', - commandTrace: [ - 'prepare_parent_target', - 'materialize_reference', - 'prepare_reference_dependencies', - 'run_compiled_oracle', - 'cleanup_workspaces', - ], - cleanup: { - parentWorkspace: 'removed', - referenceWorkspace: 'removed', - }, - evidence: {}, - }, - }; - }, - }, - ); - - expect(calls).toEqual([ - { - sourceRepositoryDir, - parentTargetDir, - referenceTargetDir, - outputRoot, - }, - ]); - expect(JSON.stringify(calls)).not.toMatch(/claude|brunch.*provider|command|referenceCommit/iu); - await expect( - runExecutionComparisonOperatorCli( - [ - 'petrinaut-preflight', - '--source-repository', - sourceRepositoryDir, - '--parent-target', - parentTargetDir, - '--reference-target', - referenceTargetDir, - '--output-root', - outputRoot, - ], - { - runPetrinautPreflight: async () => ({ - receiptFile: join(outputRoot, 'invalid-receipt.json'), - receipt: { - schemaVersion: 1, - caseId: 'petrinaut-optimization-v1', - status: 'setup_failed', - setupStatus: 'invalid', - commandTrace: [ - 'prepare_parent_target', - 'materialize_reference', - 'prepare_reference_dependencies', - 'run_compiled_oracle', - 'cleanup_workspaces', - ], - cleanup: { - parentWorkspace: 'not_created', - referenceWorkspace: 'not_created', - }, - evidence: {}, - failure: { - phase: 'parent_preparation', - messageSha256: `sha256:${'f'.repeat(64)}`, - }, - }, - }), - }, - ), - ).rejects.toThrow('setup_failed'); - await expect( - runExecutionComparisonOperatorCli( - [ - 'petrinaut-preflight', - '--source-repository', - 'relative-source', - '--parent-target', - parentTargetDir, - '--reference-target', - referenceTargetDir, - '--output-root', - outputRoot, - ], - { runPetrinautPreflight: async () => Promise.reject(new Error('must not dispatch')) }, - ), - ).rejects.toThrow('absolute'); - await expect( - runExecutionComparisonOperatorCli([ - 'petrinaut-preflight', - '--source-repository', - sourceRepositoryDir, - '--parent-target', - parentTargetDir, - '--reference-target', - referenceTargetDir, - '--output-root', - outputRoot, - '--command', - 'yarn anything', - ]), - ).rejects.toThrow('unknown execution comparison operator option: --command'); - } finally { - stdout.mockRestore(); - await rm(root, { recursive: true, force: true }); - } - }); -}); diff --git a/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts b/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts deleted file mode 100644 index 95c4b2c99..000000000 --- a/src/dev/execution-comparison/__tests__/petrinaut-historical-preflight.test.ts +++ /dev/null @@ -1,691 +0,0 @@ -import { createHash, randomUUID } from 'node:crypto'; -import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -import { afterEach, describe, expect, it } from 'vitest'; - -import { runCommand, type CommandRunner } from '../../../app/command-runner.js'; -import { createNetworkDeniedCommandRunner } from '../../end-to-end-comparison/solution-isolation.js'; -import { resolveExecutionCase, type ResolvedExecutionCase } from '../operator-cli.js'; -import { - PETRINAUT_HISTORICAL_REFERENCE_COMMIT, - parsePetrinautHistoricalPreflightReceipt, - runPetrinautHistoricalPreflight, - type PetrinautHistoricalPreflightDependencies, -} from '../petrinaut-historical-preflight.js'; - -const roots: string[] = []; -const frozenCasesRoot = fileURLToPath( - new URL('../../../../testing/execution-comparisons/cases/', import.meta.url), -); -const frozenParentCommit = '5c7a2d9db5caa851c38938f4b1bac19005b0e978'; -const frozenParentTree = 'a3e08cf75e00cc9016c931f4665341506e03533e'; - -afterEach(async () => { - await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); -}); - -describe('Petrinaut historical provider preflight', () => { - it('prepares the parent, calibrates a disjoint reference, retains redacted evidence, and cleans up', async () => { - const fixture = await createFixture(); - const selectedCase = await selectedCaseForFixture(fixture); - const order: string[] = []; - const forbiddenRootCalls: string[][] = []; - let parentRemovedBeforeReferenceInstall = false; - - const result = await runPetrinautHistoricalPreflight( - { - sourceRepositoryDir: fixture.sourceDir, - parentTargetDir: fixture.parentTargetDir, - referenceTargetDir: fixture.referenceTargetDir, - controllerRoot: fixture.controllerDir, - receiptFile: fixture.receiptFile, - }, - { - runner: fixtureRunner(fixture), - selectedCase, - referenceCommit: fixture.referenceCommit, - dependencyInstallRunner: async (command, args, options) => { - order.push(options.cwd === fixture.parentTargetDir ? 'parent-install' : 'reference-install'); - expect({ command, args }).toEqual({ - command: 'corepack', - args: ['yarn', 'install', '--immutable', '--mode=skip-build'], - }); - if (options.cwd === fixture.referenceTargetDir) { - try { - await readFile(fixture.parentTargetDir); - } catch (error) { - parentRemovedBeforeReferenceInstall = - typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT'; - } - } - await writeFile(join(options.cwd, '.pnp.cjs'), '// immutable install artifact\n'); - return { - exitCode: 0, - stdout: `installed ${options.cwd}\n`, - stderr: 'bounded warning\n', - }; - }, - createVerifier: (forbiddenRoots) => { - forbiddenRootCalls.push([...forbiddenRoots]); - return createNetworkDeniedCommandRunner({ - platform: 'darwin', - forbiddenReadRoots: forbiddenRoots, - run: fakeSandboxRunner, - }); - }, - loadOraclePack: async () => ({ - oracleId: 'petrinaut-optimization-oracles-v1', - packSha256: `sha256:${'c'.repeat(64)}`, - }), - runOracle: async ({ candidateRoot }) => { - order.push('oracle'); - expect(candidateRoot).toBe(fixture.referenceTargetDir); - return passingOracleReport(); - }, - }, - ); - - expect(PETRINAUT_HISTORICAL_REFERENCE_COMMIT).toBe('276e17d7b0f80c8a80d5abe01849bbb67c6169d0'); - expect(order).toEqual(['parent-install', 'reference-install', 'oracle']); - expect(forbiddenRootCalls).toHaveLength(1); - expect(forbiddenRootCalls[0]).toContain(fixture.referenceTargetDir); - expect(parentRemovedBeforeReferenceInstall).toBe(true); - expect(result.receipt).toMatchObject({ - schemaVersion: 1, - caseId: 'petrinaut-optimization-v1', - status: 'passed', - setupStatus: 'valid', - parent: { - sourceCommit: frozenParentCommit, - sourceTree: frozenParentTree, - dependencyPreparation: { - recipe: 'petrinaut-yarn-immutable-v1', - status: 'passed', - exitCode: 0, - }, - }, - reference: { - sourceCommit: fixture.referenceCommit, - sourceTree: fixture.referenceTree, - dependencyPreparation: { - recipe: 'petrinaut-yarn-immutable-v1', - status: 'passed', - exitCode: 0, - }, - }, - oracle: { - id: 'petrinaut-optimization-oracles-v1', - packSha256: `sha256:${'c'.repeat(64)}`, - reportStatus: 'passed', - }, - cleanup: { - parentWorkspace: 'removed', - referenceWorkspace: 'removed', - }, - }); - expect(result.receipt.commandTrace).toEqual([ - 'prepare_parent_target', - 'materialize_reference', - 'prepare_reference_dependencies', - 'run_compiled_oracle', - 'cleanup_workspaces', - ]); - expect(result.receipt.commandTrace.join('\n')).not.toMatch(/claude|brunch.*provider|lane/iu); - await expect(readFile(fixture.parentTargetDir)).rejects.toMatchObject({ code: 'ENOENT' }); - await expect(readFile(fixture.referenceTargetDir)).rejects.toMatchObject({ code: 'ENOENT' }); - - const rawReceipt = await readFile(fixture.receiptFile, 'utf8'); - expect(parsePetrinautHistoricalPreflightReceipt(JSON.parse(rawReceipt))).toEqual(result.receipt); - expect(rawReceipt).not.toContain(fixture.sourceDir); - expect(rawReceipt).not.toContain(fixture.parentTargetDir); - expect(rawReceipt).not.toContain(fixture.referenceTargetDir); - expect(rawReceipt).not.toContain(fixture.controllerDir); - expect(rawReceipt).not.toContain('.comparison-source.json'); - if ( - result.receipt.parent === undefined || - result.receipt.reference === undefined || - result.receipt.oracle === undefined - ) { - throw new Error('passing receipt omitted required evidence'); - } - expect(result.receipt.oracle.reportSha256).toBe(sha256(`${JSON.stringify(passingOracleReport())}\n`)); - for (const [key, filename] of [ - ['parentDependency', 'parent-dependency.json'], - ['referenceDependency', 'reference-dependency.json'], - ['oracleSummary', 'oracle-summary.json'], - ] as const) { - const metadata = result.receipt.evidence[key]; - expect(metadata?.file).toBe(filename); - if (metadata === undefined) throw new Error(`${key} evidence missing`); - const retained = await readFile(join(dirname(fixture.receiptFile), metadata.file), 'utf8'); - expect(sha256(retained)).toBe(metadata.sha256); - expect(Buffer.byteLength(retained, 'utf8')).toBe(metadata.bytes); - expect(retained).not.toContain(fixture.sourceDir); - expect(retained).not.toContain(fixture.parentTargetDir); - expect(retained).not.toContain(fixture.referenceTargetDir); - expect(retained).not.toContain(fixture.controllerDir); - } - expect(rawReceipt).toContain('"file": "parent-dependency.json"'); - expect(rawReceipt).toContain('"file": "reference-dependency.json"'); - expect(rawReceipt).toContain('"file": "oracle-summary.json"'); - const { parent: _parent, ...missingParent } = result.receipt; - expect(() => parsePetrinautHistoricalPreflightReceipt(missingParent)).toThrow(); - expect(() => - parsePetrinautHistoricalPreflightReceipt({ - ...result.receipt, - launch: { command: 'claude', cwd: fixture.parentTargetDir }, - }), - ).toThrow(); - }, 30_000); - - it('retains setup_failed evidence and never calibrates after parent preparation fails', async () => { - const fixture = await createFixture(); - let oracleCalled = false; - - const result = await runFixturePreflight(fixture, { - dependencyInstallRunner: async () => ({ - exitCode: 9, - stdout: '', - stderr: `parent install failed in ${fixture.parentTargetDir}\n`, - }), - runOracle: async () => { - oracleCalled = true; - return passingOracleReport(); - }, - }); - - expect(result.receipt).toMatchObject({ - status: 'setup_failed', - setupStatus: 'invalid', - failure: { - phase: 'parent_preparation', - dependencyStage: 'install', - }, - cleanup: { - parentWorkspace: 'not_created', - referenceWorkspace: 'not_created', - }, - }); - expect(result.receipt.parent).toBeUndefined(); - expect(result.receipt.reference?.dependencyPreparation).toBeUndefined(); - expect(result.receipt.oracle).toBeUndefined(); - expect(oracleCalled).toBe(false); - await expect(readFile(fixture.receiptFile, 'utf8')).resolves.toContain('"setup_failed"'); - const evidence = result.receipt.evidence.parentDependency; - expect(evidence?.file).toBe('parent-dependency.json'); - if (evidence === undefined) throw new Error('parent dependency evidence missing'); - const evidenceBytes = await readFile(join(dirname(fixture.receiptFile), evidence.file), 'utf8'); - expect(sha256(evidenceBytes)).toBe(evidence.sha256); - expect(evidenceBytes).toContain('"failureStage": "install"'); - expect(evidenceBytes).toContain('"exitCode": 9'); - expect(evidenceBytes).not.toContain(fixture.parentTargetDir); - }, 30_000); - - it('cleans both workspaces and retains the exact phase when reference installation fails', async () => { - const fixture = await createFixture(); - let installs = 0; - let oracleCalled = false; - const sensitiveMarker = `token=${randomUUID()}`; - - const result = await runFixturePreflight(fixture, { - dependencyInstallRunner: async () => { - installs += 1; - return installs === 1 - ? { exitCode: 0, stdout: 'parent ok\n', stderr: '' } - : { - exitCode: 7, - stdout: 'x'.repeat(200_000), - stderr: `reference install failed in ${fixture.referenceTargetDir}; ${sensitiveMarker}\n`, - }; - }, - runOracle: async () => { - oracleCalled = true; - return passingOracleReport(); - }, - }); - - expect(result.receipt).toMatchObject({ - status: 'setup_failed', - failure: { - phase: 'reference_dependency_preparation', - dependencyStage: 'install', - }, - cleanup: { - parentWorkspace: 'removed', - referenceWorkspace: 'removed', - }, - }); - expect(result.receipt.parent).toBeDefined(); - expect(result.receipt.reference?.dependencyPreparation).toMatchObject({ - status: 'failed', - failureStage: 'install', - exitCode: 7, - }); - expect(result.receipt.oracle).toBeUndefined(); - expect(oracleCalled).toBe(false); - const evidence = result.receipt.evidence.referenceDependency; - expect(evidence).toMatchObject({ - file: 'reference-dependency.json', - truncated: true, - bytes: expect.any(Number), - sha256: expect.stringMatching(/^sha256:[a-f0-9]{64}$/u), - }); - if (evidence === undefined) { - throw new Error('reference dependency evidence missing'); - } - expect(evidence.bytes).toBeLessThan(140_000); - const evidenceBytes = await readFile(join(dirname(fixture.receiptFile), evidence.file), 'utf8'); - expect(sha256(evidenceBytes)).toBe(evidence.sha256); - expect(evidenceBytes).toContain('"failureStage": "install"'); - expect(evidenceBytes).toContain('"exitCode": 7'); - expect(evidenceBytes).toContain('reference install failed'); - expect(evidenceBytes).not.toContain(fixture.referenceTargetDir); - expect(evidenceBytes).not.toContain(sensitiveMarker); - expect(JSON.stringify(result.receipt)).not.toContain(fixture.referenceTargetDir); - }, 30_000); - - it('retains a successful install result separately from tracked-source cleanliness rejection', async () => { - const fixture = await createFixture(); - let installs = 0; - - const result = await runFixturePreflight(fixture, { - dependencyInstallRunner: async (_command, _args, options) => { - installs += 1; - if (installs === 2) { - await writeFile(join(options.cwd, 'package.json'), '{"name":"mutated-reference"}\n'); - } - return { - exitCode: 0, - stdout: `immutable install passed in ${options.cwd}\n`, - stderr: '', - }; - }, - }); - - expect(result.receipt).toMatchObject({ - status: 'setup_failed', - failure: { - phase: 'reference_dependency_preparation', - dependencyStage: 'tracked_source_cleanliness', - }, - reference: { - dependencyPreparation: { - status: 'failed', - exitCode: 0, - failureStage: 'tracked_source_cleanliness', - }, - }, - }); - const evidence = result.receipt.evidence.referenceDependency; - if (evidence === undefined) throw new Error('reference dependency evidence missing'); - const retained = await readFile(join(dirname(fixture.receiptFile), evidence.file), 'utf8'); - expect(sha256(retained)).toBe(evidence.sha256); - expect(retained).toContain('"exitCode": 0'); - expect(retained).toContain('"failureStage": "tracked_source_cleanliness"'); - expect(retained).toContain('"trackedSourceStatus": "M package.json"'); - expect(retained).not.toContain(fixture.referenceTargetDir); - }, 30_000); - - it('retains sanitized command output for an oracle preparation failure', async () => { - const fixture = await createFixture(); - const report = setupFailedOracleReport(); - - const result = await runFixturePreflight(fixture, { - dependencyInstallRunner: passingInstall, - runOracle: async ({ onPreparationResult }) => { - await onPreparationResult?.({ - id: 'petrinaut-ui-build', - commandResult: { - exitCode: 1, - stdout: 'x'.repeat(50_000), - stderr: `build failed in ${fixture.referenceTargetDir}; token=oracle-secret\n`, - }, - }); - return report; - }, - }); - - const evidence = result.receipt.evidence.oracleSummary; - if (evidence === undefined) throw new Error('oracle summary evidence missing'); - expect(evidence.truncated).toBe(true); - expect(evidence.bytes).toBeLessThan(100_000); - const retained = await readFile(join(dirname(fixture.receiptFile), evidence.file), 'utf8'); - expect(sha256(retained)).toBe(evidence.sha256); - expect(retained).toContain('"id": "petrinaut-ui-build"'); - expect(retained).toContain('"exitCode": 1'); - expect(retained).toContain('build failed'); - expect(retained).not.toContain(fixture.referenceTargetDir); - expect(retained).not.toContain('oracle-secret'); - }, 30_000); - - it('rejects historical identity or path leakage in the parent target before calibration', async () => { - const fixture = await createFixture(); - let installs = 0; - let oracleCalled = false; - - const result = await runFixturePreflight(fixture, { - dependencyInstallRunner: async (_command, _args, options) => { - installs += 1; - if (installs === 1) { - await writeFile( - join(options.cwd, 'historical-reference.txt'), - `${fixture.referenceCommit}\n${fixture.referenceTargetDir}\n`, - ); - } - return { exitCode: 0, stdout: '', stderr: '' }; - }, - runOracle: async () => { - oracleCalled = true; - return passingOracleReport(); - }, - }); - - expect(result.receipt).toMatchObject({ - status: 'setup_failed', - failure: { phase: 'reference_materialization' }, - cleanup: { - parentWorkspace: 'removed', - referenceWorkspace: 'removed', - }, - }); - expect(result.receipt.reference?.dependencyPreparation).toBeUndefined(); - expect(result.receipt.oracle).toBeUndefined(); - expect(oracleCalled).toBe(false); - const rawReceipt = await readFile(fixture.receiptFile, 'utf8'); - expect(rawReceipt).not.toContain(fixture.referenceTargetDir); - }, 30_000); - - it.each([ - ['setup failure', setupFailedOracleReport(), 'setup_failed'], - ['calibration assertion', assertionFailedOracleReport(), 'assertion_failed'], - ] as const)( - 'preserves the %s distinction from the unchanged oracle', - async (_name, report, status) => { - const fixture = await createFixture(); - const result = await runFixturePreflight(fixture, { - dependencyInstallRunner: passingInstall, - runOracle: async () => report, - }); - - expect(result.receipt.status).toBe(status); - expect(result.receipt.setupStatus).toBe('invalid'); - expect(result.receipt.oracle).toMatchObject({ - reportStatus: status, - id: 'petrinaut-optimization-oracles-v1', - }); - expect(result.receipt.failure).toBeUndefined(); - expect(result.receipt.cleanup).toEqual({ - parentWorkspace: 'removed', - referenceWorkspace: 'removed', - }); - }, - 30_000, - ); - - it('rejects output collisions before creating either workspace', async () => { - const fixture = await createFixture(); - await writeFile(fixture.receiptFile, 'caller-owned evidence\n'); - - await expect( - runFixturePreflight(fixture, { - dependencyInstallRunner: passingInstall, - }), - ).rejects.toMatchObject({ code: 'EEXIST' }); - await expect(readFile(fixture.receiptFile, 'utf8')).resolves.toBe('caller-owned evidence\n'); - await expect(readFile(fixture.parentTargetDir)).rejects.toMatchObject({ code: 'ENOENT' }); - await expect(readFile(fixture.referenceTargetDir)).rejects.toMatchObject({ code: 'ENOENT' }); - }); - - it('rejects retained evidence collisions before running dependency preparation', async () => { - const fixture = await createFixture(); - const evidenceFile = join(dirname(fixture.receiptFile), 'parent-dependency.json'); - await writeFile(evidenceFile, 'caller-owned evidence\n'); - let installCalled = false; - - await expect( - runFixturePreflight(fixture, { - dependencyInstallRunner: async () => { - installCalled = true; - return { exitCode: 0, stdout: '', stderr: '' }; - }, - }), - ).rejects.toThrow('retained evidence parent-dependency.json'); - expect(installCalled).toBe(false); - await expect(readFile(evidenceFile, 'utf8')).resolves.toBe('caller-owned evidence\n'); - await expect(readFile(fixture.receiptFile)).rejects.toMatchObject({ - code: 'ENOENT', - }); - }); - - it.each([ - ['parent/reference', (fixture: Awaited>) => fixture.parentTargetDir], - [ - 'source/parent', - (fixture: Awaited>) => join(fixture.sourceDir, 'target'), - ], - [ - 'output/reference', - (fixture: Awaited>) => join(dirname(fixture.receiptFile), 'reference'), - ], - ])('rejects %s root overlap before materialization', async (_name, conflictingReference) => { - const fixture = await createFixture(); - const referenceTargetDir = conflictingReference(fixture); - - await expect( - runPetrinautHistoricalPreflight( - { - sourceRepositoryDir: fixture.sourceDir, - parentTargetDir: fixture.parentTargetDir, - referenceTargetDir, - controllerRoot: fixture.controllerDir, - receiptFile: fixture.receiptFile, - }, - { - selectedCase: await selectedCaseForFixture(fixture), - referenceCommit: fixture.referenceCommit, - runner: fixtureRunner(fixture), - dependencyInstallRunner: passingInstall, - createVerifier: portableVerifier, - loadOraclePack: fixedOraclePack, - runOracle: async () => passingOracleReport(), - }, - ), - ).rejects.toThrow('disjoint'); - await expect(readFile(fixture.receiptFile)).rejects.toMatchObject({ code: 'ENOENT' }); - }); -}); - -async function createFixture(): Promise<{ - readonly root: string; - readonly sourceDir: string; - readonly controllerDir: string; - readonly parentTargetDir: string; - readonly referenceTargetDir: string; - readonly receiptFile: string; - readonly parentCommit: string; - readonly referenceCommit: string; - readonly referenceTree: string; -}> { - const root = await realpath(await mkdtemp(join(tmpdir(), 'brunch-petrinaut-preflight-'))); - roots.push(root); - const sourceDir = join(root, 'source'); - const controllerDir = join(root, 'controller'); - const workRoot = join(root, 'work'); - const evidenceRoot = join(root, 'evidence'); - await Promise.all([mkdir(sourceDir), mkdir(controllerDir), mkdir(workRoot), mkdir(evidenceRoot)]); - - await writeFile(join(sourceDir, 'package.json'), '{"name":"petrinaut-fixture","private":true}\n'); - await writeFile(join(sourceDir, 'parent.ts'), 'export const parent = true;\n'); - await git(sourceDir, ['init', '--initial-branch=main']); - await git(sourceDir, ['add', '--all']); - await commit(sourceDir, 'Pinned parent'); - const parentCommit = await git(sourceDir, ['rev-parse', 'HEAD']); - await writeFile(join(sourceDir, 'reference.ts'), 'export const mergedReference = true;\n'); - await git(sourceDir, ['add', '--all']); - await commit(sourceDir, 'Merged reference'); - const referenceCommit = await git(sourceDir, ['rev-parse', 'HEAD']); - const referenceTree = await git(sourceDir, ['rev-parse', 'HEAD^{tree}']); - - return { - root, - sourceDir, - controllerDir, - parentTargetDir: join(workRoot, 'parent'), - referenceTargetDir: join(workRoot, 'reference'), - receiptFile: join(evidenceRoot, 'receipt.json'), - parentCommit, - referenceCommit, - referenceTree, - }; -} - -async function selectedCaseForFixture( - _fixture: Awaited>, -): Promise { - return await resolveExecutionCase('petrinaut-optimization', frozenCasesRoot); -} - -async function runFixturePreflight( - fixture: Awaited>, - dependencies: PetrinautHistoricalPreflightDependencies, -) { - return await runPetrinautHistoricalPreflight( - { - sourceRepositoryDir: fixture.sourceDir, - parentTargetDir: fixture.parentTargetDir, - referenceTargetDir: fixture.referenceTargetDir, - controllerRoot: fixture.controllerDir, - receiptFile: fixture.receiptFile, - }, - { - selectedCase: await selectedCaseForFixture(fixture), - referenceCommit: fixture.referenceCommit, - runner: fixtureRunner(fixture), - createVerifier: portableVerifier, - loadOraclePack: fixedOraclePack, - ...dependencies, - }, - ); -} - -function fixtureRunner(fixture: Awaited>): CommandRunner { - return async (command, args, options) => { - if (command === 'git' && options.cwd === fixture.sourceDir) { - if (args.at(-1) === `${frozenParentCommit}^{commit}`) { - return { exitCode: 0, stdout: `${frozenParentCommit}\n`, stderr: '' }; - } - if (args.at(-1) === `${frozenParentCommit}^{tree}`) { - return { exitCode: 0, stdout: `${frozenParentTree}\n`, stderr: '' }; - } - if (args[0] === 'archive' && args.at(-1) === frozenParentCommit) { - return await runCommand(command, [...args.slice(0, -1), fixture.parentCommit], options); - } - } - return await runCommand(command, args, options); - }; -} - -async function git(cwd: string, args: readonly string[]): Promise { - const result = await runCommand('git', args, { cwd }); - if (result.exitCode !== 0) throw new Error(result.stderr || result.stdout); - return result.stdout.trim(); -} - -async function commit(cwd: string, message: string): Promise { - await git(cwd, [ - '-c', - 'user.name=Petrinaut Preflight Fixture', - '-c', - 'user.email=petrinaut-preflight@example.invalid', - 'commit', - '-m', - message, - ]); -} - -const fakeSandboxRunner: CommandRunner = async (command, args) => { - if (command !== 'sandbox-exec') throw new Error(`unexpected verifier command: ${command}`); - if (args[2] === '/usr/bin/curl' && args[3] === '--version') { - return { exitCode: 0, stdout: 'curl fake\n', stderr: '' }; - } - return { exitCode: 1, stdout: '', stderr: 'denied by fake sandbox\n' }; -}; - -const portableVerifier: NonNullable = ( - forbiddenReadRoots, -) => - createNetworkDeniedCommandRunner({ - platform: 'darwin', - forbiddenReadRoots, - run: fakeSandboxRunner, - }); - -const fixedOraclePack: NonNullable< - PetrinautHistoricalPreflightDependencies['loadOraclePack'] -> = async () => ({ - oracleId: 'petrinaut-optimization-oracles-v1', - packSha256: `sha256:${'c'.repeat(64)}`, -}); - -const passingInstall: CommandRunner = async () => ({ - exitCode: 0, - stdout: 'immutable install passed\n', - stderr: '', -}); - -function passingOracleReport() { - return { - schemaVersion: 1 as const, - caseId: 'petrinaut-optimization-v1' as const, - oracleId: 'petrinaut-optimization-oracles-v1' as const, - status: 'passed' as const, - preparation: [ - { - id: 'petrinaut-ui-build', - status: 'passed' as const, - exitCode: 0, - }, - ], - checks: [ - { - id: 'route-and-accessibility' as const, - claims: ['AC1'], - status: 'passed' as const, - evidence: ['route accessible'], - }, - ], - consoleErrors: [], - failedRequests: [], - }; -} - -function setupFailedOracleReport() { - return { - ...passingOracleReport(), - status: 'setup_failed' as const, - checks: [], - setupFailure: 'browser server did not become ready', - }; -} - -function assertionFailedOracleReport() { - return { - ...passingOracleReport(), - status: 'assertion_failed' as const, - checks: [ - { - id: 'route-and-accessibility' as const, - claims: ['AC1'], - status: 'failed' as const, - evidence: ['focused route missing'], - }, - ], - }; -} - -function sha256(value: string): string { - return `sha256:${createHash('sha256').update(value).digest('hex')}`; -} diff --git a/src/dev/execution-comparison/historical-replay-target.ts b/src/dev/execution-comparison/historical-replay-target.ts index f15edbbf7..07f8c6b69 100644 --- a/src/dev/execution-comparison/historical-replay-target.ts +++ b/src/dev/execution-comparison/historical-replay-target.ts @@ -11,16 +11,12 @@ import { import { createClaudeExecutionLaunch } from '../end-to-end-comparison/claude-adapter.js'; import { materializeExactExecutionPacket } from '../end-to-end-comparison/public-packet.js'; import { - admitHistoricalReplay, createBrunchSolutionIsolationPolicy, createClaudeSolutionIsolationPolicy, - createNetworkDeniedCommandRunner, materializePinnedSourceTree, - SolutionIsolationAdmissionError, + verifyPreparedHistoricalReplay, type BrunchSolutionIsolationPolicy, type ClaudeSolutionIsolationPolicy, - type IsolationAdmissionReason, - type NetworkDeniedCommandRunner, } from '../end-to-end-comparison/solution-isolation.js'; import { assertControllerIsolation } from '../end-to-end-comparison/study-contract.js'; import { seedBrownfieldBrunchExecutionWorkspace } from './brunch-lane.js'; @@ -141,7 +137,6 @@ export interface HistoricalReplayCaseSelection { export interface HistoricalReplayTargetDependencies { readonly runner?: CommandRunner; readonly dependencyInstallRunner?: CommandRunner; - readonly createVerifier?: (forbiddenReadRoots: readonly string[]) => NetworkDeniedCommandRunner; readonly onPetrinautDependencyPreparation?: ( observation: PetrinautDependencyPreparationObservation, ) => Promise | void; @@ -150,7 +145,6 @@ export interface HistoricalReplayTargetDependencies { export class HistoricalReplayTargetPreparationError extends Error { readonly status = 'setup_failed' as const; readonly phase: HistoricalReplayPreparationPhase; - readonly reasons: readonly IsolationAdmissionReason[]; override readonly cause: unknown; constructor(phase: HistoricalReplayPreparationPhase, cause: unknown) { @@ -160,7 +154,6 @@ export class HistoricalReplayTargetPreparationError extends Error { this.name = 'HistoricalReplayTargetPreparationError'; this.phase = phase; this.cause = cause; - this.reasons = cause instanceof SolutionIsolationAdmissionError ? cause.reasons : []; } } @@ -254,19 +247,12 @@ export async function prepareHistoricalReplayTarget( phase = 'admission'; const claudePolicy = createClaudeSolutionIsolationPolicy(input.targetDir, forbiddenRoots); const brunchPolicy = createBrunchSolutionIsolationPolicy(input.targetDir); - await admitHistoricalReplay({ + await verifyPreparedHistoricalReplay({ prefix: { ...materialized, baseSha, packetFiles, }, - policies: [claudePolicy, brunchPolicy], - forbiddenRoots, - networkProbeUrls: [], - verifier: - dependencies.createVerifier?.(forbiddenRoots) ?? - createNetworkDeniedCommandRunner({ forbiddenReadRoots: forbiddenRoots }), - localChecks: [], runner, }); @@ -293,7 +279,6 @@ export async function prepareHistoricalReplayTarget( launch: createBrunchExecutionLaunch({ workspaceDir: input.targetDir, specId: seeded.specId, - forbiddenRoots, }), }; } diff --git a/src/dev/execution-comparison/operator-cli.ts b/src/dev/execution-comparison/operator-cli.ts index c721e9afe..e4b20b887 100644 --- a/src/dev/execution-comparison/operator-cli.ts +++ b/src/dev/execution-comparison/operator-cli.ts @@ -131,8 +131,7 @@ export async function prepareExecutionTarget( if ( input.sourceRepositoryDir !== undefined || dependencies.runner !== undefined || - dependencies.dependencyInstallRunner !== undefined || - dependencies.createVerifier !== undefined + dependencies.dependencyInstallRunner !== undefined ) { throw new Error('--source-repository is valid only for pinned execution cases'); } diff --git a/src/dev/execution-comparison/petrinaut-historical-preflight.ts b/src/dev/execution-comparison/petrinaut-historical-preflight.ts deleted file mode 100644 index a3ef96b40..000000000 --- a/src/dev/execution-comparison/petrinaut-historical-preflight.ts +++ /dev/null @@ -1,707 +0,0 @@ -import { createHash } from 'node:crypto'; -import { lstat, open, readdir, realpath, rm } from 'node:fs/promises'; -import { basename, dirname, isAbsolute, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -import { z } from 'zod'; - -import { runCommand, type CommandRunner } from '../../app/command-runner.js'; -import { materializePinnedSourceTree } from '../end-to-end-comparison/solution-isolation.js'; -import { assertControllerIsolation } from '../end-to-end-comparison/study-contract.js'; -import { - HistoricalReplayTargetPreparationError, - PetrinautDependencyPreparationError, - prepareHistoricalReplayTarget, - preparePetrinautHistoricalReplayDependencies, - type HistoricalReplayTargetDependencies, - type PetrinautDependencyPreparationObservation, - type PetrinautDependencyPreparationOutcome, -} from './historical-replay-target.js'; -import { resolveExecutionCase, type ResolvedExecutionCase } from './operator-cli.js'; -import { - createPreflightEvidenceWriter, - PREFLIGHT_EVIDENCE_FILES, - retainedPreflightEvidenceSchema, - type RetainedPreflightEvidence, -} from './petrinaut-historical-preflight/evidence.js'; -import { - runPetrinautOptimizationOracle, - type PetrinautOraclePreparationObservation, -} from './petrinaut-optimization-oracle.js'; -import type { PetrinautOptimizationOracleReport } from './petrinaut-optimization-oracle/types.js'; - -export const PETRINAUT_HISTORICAL_REFERENCE_COMMIT = '276e17d7b0f80c8a80d5abe01849bbb67c6169d0' as const; - -const CASE_REFERENCE = 'petrinaut-optimization-v1'; -const ORACLE_ID = 'petrinaut-optimization-oracles-v1'; -const DEFAULT_CASES_ROOT = fileURLToPath( - new URL('../../../testing/execution-comparisons/cases/', import.meta.url), -); -const HASH = /^sha256:[a-f0-9]{64}$/u; -const GIT_OBJECT = /^[a-f0-9]{40}$/u; -const COMMAND_TRACE = [ - 'prepare_parent_target', - 'materialize_reference', - 'prepare_reference_dependencies', - 'run_compiled_oracle', - 'cleanup_workspaces', -] as const; - -const dependencyPreparationBase = z.object({ - recipe: z.literal('petrinaut-yarn-immutable-v1'), - command: z.literal('corepack'), - args: z.tuple([ - z.literal('yarn'), - z.literal('install'), - z.literal('--immutable'), - z.literal('--mode=skip-build'), - ]), - exitCode: z.number().int(), -}); -const dependencyPreparationSchema = z.discriminatedUnion('status', [ - dependencyPreparationBase - .extend({ - status: z.literal('passed'), - exitCode: z.literal(0), - }) - .strict(), - dependencyPreparationBase - .extend({ - status: z.literal('failed'), - failureStage: z.enum(['install', 'tracked_source_cleanliness']), - }) - .strict(), -]); - -const retainedEvidenceSetSchema = z - .object({ - parentDependency: retainedPreflightEvidenceSchema.optional(), - referenceDependency: retainedPreflightEvidenceSchema.optional(), - oracleSummary: retainedPreflightEvidenceSchema.optional(), - }) - .strict(); - -const cleanupStatusSchema = z.enum(['removed', 'not_created', 'failed']); -const receiptStatusSchema = z.enum(['passed', 'setup_failed', 'assertion_failed']); -const preflightPhaseSchema = z.enum([ - 'parent_preparation', - 'reference_materialization', - 'reference_dependency_preparation', - 'oracle_pack', - 'oracle_calibration', - 'evidence_retention', - 'cleanup', -]); - -const historicalIdentitySchema = z - .object({ - sourceCommit: z.string().regex(GIT_OBJECT), - sourceTree: z.string().regex(GIT_OBJECT), - }) - .strict(); - -export const petrinautHistoricalPreflightReceiptSchema = z - .object({ - schemaVersion: z.literal(1), - caseId: z.literal('petrinaut-optimization-v1'), - status: receiptStatusSchema, - setupStatus: z.enum(['valid', 'invalid']), - parent: historicalIdentitySchema - .extend({ - materializedCommit: z.string().regex(GIT_OBJECT), - packetCommit: z.string().regex(GIT_OBJECT), - dependencyPreparation: dependencyPreparationSchema, - }) - .strict() - .optional(), - reference: historicalIdentitySchema - .extend({ - dependencyPreparation: dependencyPreparationSchema.optional(), - }) - .strict() - .optional(), - oracle: z - .object({ - id: z.literal(ORACLE_ID), - packSha256: z.string().regex(HASH), - reportSha256: z.string().regex(HASH), - reportStatus: z.enum(['passed', 'setup_failed', 'assertion_failed']), - }) - .strict() - .optional(), - evidence: retainedEvidenceSetSchema, - commandTrace: z.tuple([ - z.literal('prepare_parent_target'), - z.literal('materialize_reference'), - z.literal('prepare_reference_dependencies'), - z.literal('run_compiled_oracle'), - z.literal('cleanup_workspaces'), - ]), - cleanup: z - .object({ - parentWorkspace: cleanupStatusSchema, - referenceWorkspace: cleanupStatusSchema, - }) - .strict(), - failure: z - .object({ - phase: preflightPhaseSchema, - dependencyStage: z.enum(['install', 'tracked_source_cleanliness']).optional(), - messageSha256: z.string().regex(HASH), - }) - .strict() - .optional(), - }) - .strict() - .superRefine((receipt, context) => { - if (receipt.status === 'passed' && receipt.setupStatus !== 'valid') { - context.addIssue({ code: 'custom', message: 'passed receipt must be setup-valid' }); - } - if (receipt.status !== 'passed' && receipt.setupStatus !== 'invalid') { - context.addIssue({ code: 'custom', message: 'failed receipt must be setup-invalid' }); - } - if ( - (receipt.status === 'passed' || receipt.status === 'assertion_failed') && - (receipt.parent === undefined || - receipt.reference?.dependencyPreparation?.status !== 'passed' || - receipt.oracle === undefined || - receipt.evidence.parentDependency === undefined || - receipt.evidence.referenceDependency === undefined || - receipt.evidence.oracleSummary === undefined) - ) { - context.addIssue({ - code: 'custom', - message: 'completed calibration receipt requires parent, reference, oracle, and retained evidence', - }); - } - if (receipt.oracle !== undefined && receipt.oracle.reportStatus !== receipt.status) { - context.addIssue({ - code: 'custom', - message: 'receipt status must match the retained oracle report status', - }); - } - if ( - receipt.status !== 'setup_failed' && - (receipt.cleanup.parentWorkspace !== 'removed' || receipt.cleanup.referenceWorkspace !== 'removed') - ) { - context.addIssue({ - code: 'custom', - message: 'completed calibration receipt requires owned workspace cleanup', - }); - } - for (const [key, expectedFile] of Object.entries(PREFLIGHT_EVIDENCE_FILES) as [ - keyof typeof PREFLIGHT_EVIDENCE_FILES, - (typeof PREFLIGHT_EVIDENCE_FILES)[keyof typeof PREFLIGHT_EVIDENCE_FILES], - ][]) { - const retained = receipt.evidence[key]; - if (retained !== undefined && retained.file !== expectedFile) { - context.addIssue({ - code: 'custom', - message: `${key} evidence uses the wrong fixed filename`, - }); - } - } - if ( - (receipt.parent?.dependencyPreparation !== undefined || - (receipt.failure?.phase === 'parent_preparation' && receipt.failure.dependencyStage !== undefined)) && - receipt.evidence.parentDependency === undefined && - receipt.failure?.phase !== 'evidence_retention' - ) { - context.addIssue({ - code: 'custom', - message: 'parent dependency result requires retained evidence', - }); - } - if ( - (receipt.reference?.dependencyPreparation !== undefined || - (receipt.failure?.phase === 'reference_dependency_preparation' && - receipt.failure.dependencyStage !== undefined)) && - receipt.evidence.referenceDependency === undefined && - receipt.failure?.phase !== 'evidence_retention' - ) { - context.addIssue({ - code: 'custom', - message: 'reference dependency result requires retained evidence', - }); - } - if ( - (receipt.oracle !== undefined || receipt.failure?.phase === 'oracle_calibration') && - receipt.evidence.oracleSummary === undefined && - receipt.failure?.phase !== 'evidence_retention' - ) { - context.addIssue({ - code: 'custom', - message: 'oracle calibration requires retained summary evidence', - }); - } - }); - -export type PetrinautHistoricalPreflightReceipt = z.infer; - -export interface PetrinautHistoricalPreflightInput { - readonly sourceRepositoryDir: string; - readonly parentTargetDir: string; - readonly referenceTargetDir: string; - readonly controllerRoot: string; - readonly receiptFile: string; -} - -interface ClosedOraclePack { - readonly oracleId: typeof ORACLE_ID; - readonly packSha256: string; -} - -export interface PetrinautHistoricalPreflightDependencies extends HistoricalReplayTargetDependencies { - /** Test-only frozen-case substitution for self-contained Git fixtures. */ - readonly selectedCase?: ResolvedExecutionCase; - /** Test-only reference substitution for self-contained Git fixtures. */ - readonly referenceCommit?: string; - readonly loadOraclePack?: (caseDir: string) => Promise; - readonly runOracle?: (input: { - readonly candidateRoot: string; - readonly caseDir: string; - readonly onPreparationResult?: ( - observation: PetrinautOraclePreparationObservation, - ) => Promise | void; - }) => Promise; -} - -export async function runPetrinautHistoricalPreflight( - input: PetrinautHistoricalPreflightInput, - dependencies: PetrinautHistoricalPreflightDependencies = {}, -): Promise<{ - readonly receiptFile: string; - readonly receipt: PetrinautHistoricalPreflightReceipt; -}> { - const roots = await validateInput(input); - const receiptHandle = await open(roots.receiptFile, 'wx'); - const runner = dependencies.runner ?? runCommand; - const dependencyInstallRunner = dependencies.dependencyInstallRunner ?? runCommand; - const evidenceWriter = createPreflightEvidenceWriter({ - outputRoot: roots.outputRoot, - redactedRoots: [ - roots.sourceRepositoryDir, - roots.parentTargetDir, - roots.referenceTargetDir, - roots.controllerRoot, - roots.outputRoot, - ], - }); - let phase: z.infer = 'parent_preparation'; - let parent: - | { - readonly sourceCommit: string; - readonly sourceTree: string; - readonly materializedCommit: string; - readonly packetCommit: string; - readonly dependencyPreparation: PetrinautDependencyPreparationOutcome; - } - | undefined; - let reference: - | { - readonly sourceCommit: string; - readonly sourceTree: string; - readonly dependencyPreparation?: PetrinautDependencyPreparationOutcome; - } - | undefined; - let oracle: PetrinautHistoricalPreflightReceipt['oracle']; - let parentObservation: PetrinautDependencyPreparationObservation | undefined; - let referenceObservation: PetrinautDependencyPreparationObservation | undefined; - let oracleSummary: - | { - readonly kind: 'report'; - readonly report: PetrinautOptimizationOracleReport; - } - | { readonly kind: 'failure'; readonly error: unknown } - | undefined; - let parentRemovedEarly = false; - const oraclePreparation: PetrinautOraclePreparationObservation[] = []; - let evidence: { - readonly parentDependency?: RetainedPreflightEvidence; - readonly referenceDependency?: RetainedPreflightEvidence; - readonly oracleSummary?: RetainedPreflightEvidence; - } = {}; - let status: PetrinautHistoricalPreflightReceipt['status'] = 'setup_failed'; - let failure: PetrinautHistoricalPreflightReceipt['failure']; - const retainDependencyObservation = async ( - scope: 'parent' | 'reference', - observation: PetrinautDependencyPreparationObservation, - ): Promise => { - if (scope === 'parent') parentObservation = observation; - else referenceObservation = observation; - await dependencies.onPetrinautDependencyPreparation?.(observation); - }; - - try { - const selectedCase = - dependencies.selectedCase ?? (await resolveExecutionCase(CASE_REFERENCE, DEFAULT_CASES_ROOT)); - if (selectedCase.caseId !== CASE_REFERENCE) { - throw new Error('Petrinaut preflight received a different frozen case'); - } - const parentReady = await prepareHistoricalReplayTarget( - { - lane: 'claude_code', - selectedCase, - sourceRepositoryDir: roots.sourceRepositoryDir, - targetDir: roots.parentTargetDir, - controllerRoot: roots.controllerRoot, - forbiddenRoots: [roots.referenceTargetDir, roots.outputRoot], - }, - { - runner, - dependencyInstallRunner, - onPetrinautDependencyPreparation: async (observation) => - await retainDependencyObservation('parent', observation), - ...(dependencies.createVerifier === undefined ? {} : { createVerifier: dependencies.createVerifier }), - }, - ); - if (parentReady.dependencyPreparation.recipe !== 'petrinaut-yarn-immutable-v1') { - throw new Error('Petrinaut parent used a different dependency recipe'); - } - parent = { - sourceCommit: parentReady.sourceCommit, - sourceTree: parentReady.sourceTree, - materializedCommit: parentReady.materializedCommit, - packetCommit: parentReady.baseSha, - dependencyPreparation: parentReady.dependencyPreparation, - }; - - phase = 'reference_materialization'; - const materializedReference = await materializePinnedSourceTree({ - sourceRepositoryDir: roots.sourceRepositoryDir, - sourceCommit: dependencies.referenceCommit ?? PETRINAUT_HISTORICAL_REFERENCE_COMMIT, - targetDir: roots.referenceTargetDir, - runner, - }); - reference = { - sourceCommit: materializedReference.sourceCommit, - sourceTree: materializedReference.sourceTree, - }; - await assertParentContainsNoReference( - roots.parentTargetDir, - materializedReference.sourceCommit, - roots.referenceTargetDir, - runner, - ); - phase = 'cleanup'; - if ((await removeOwnedWorkspace(roots.parentTargetDir)) !== 'removed') { - throw new Error('owned parent workspace cleanup failed'); - } - parentRemovedEarly = true; - - phase = 'reference_dependency_preparation'; - const referencePreparation = await preparePetrinautHistoricalReplayDependencies({ - targetDir: roots.referenceTargetDir, - runner, - dependencyInstallRunner, - onObservation: async (observation) => { - reference = { - sourceCommit: materializedReference.sourceCommit, - sourceTree: materializedReference.sourceTree, - dependencyPreparation: observation.outcome, - }; - await retainDependencyObservation('reference', observation); - }, - }); - reference = { - sourceCommit: materializedReference.sourceCommit, - sourceTree: materializedReference.sourceTree, - dependencyPreparation: referencePreparation, - }; - - phase = 'oracle_pack'; - const oraclePack = await (dependencies.loadOraclePack ?? loadClosedOraclePack)(selectedCase.caseDir); - if (oraclePack.oracleId !== ORACLE_ID || !HASH.test(oraclePack.packSha256)) { - throw new Error('Petrinaut preflight received a different compiled oracle pack'); - } - - phase = 'oracle_calibration'; - let report: PetrinautOptimizationOracleReport; - try { - report = await (dependencies.runOracle ?? runPetrinautOptimizationOracle)({ - candidateRoot: roots.referenceTargetDir, - caseDir: selectedCase.caseDir, - onPreparationResult: (observation) => { - oraclePreparation.push(observation); - }, - }); - } catch (error) { - oracleSummary = { kind: 'failure', error }; - throw error; - } - oracleSummary = { kind: 'report', report }; - const reportBytes = `${JSON.stringify(report)}\n`; - oracle = { - id: ORACLE_ID, - packSha256: oraclePack.packSha256, - reportSha256: sha256(reportBytes), - reportStatus: report.status, - }; - status = report.status; - } catch (error) { - status = 'setup_failed'; - const dependencyError = findDependencyPreparationError(error); - failure = { - phase, - ...(dependencyError === undefined ? {} : { dependencyStage: dependencyError.outcome.failureStage }), - messageSha256: sha256(redactedError(error, roots)), - }; - } - - const cleanup = await cleanupWorkspaces(roots, parentRemovedEarly); - if (cleanup.parentWorkspace === 'failed' || cleanup.referenceWorkspace === 'failed') { - status = 'setup_failed'; - failure = { - phase: 'cleanup', - messageSha256: sha256('owned workspace cleanup failed'), - }; - } - try { - phase = 'evidence_retention'; - if (parentObservation !== undefined) { - evidence = { - ...evidence, - parentDependency: await evidenceWriter.writeDependency('parent', parentObservation), - }; - } - if (referenceObservation !== undefined) { - evidence = { - ...evidence, - referenceDependency: await evidenceWriter.writeDependency('reference', referenceObservation), - }; - } - if (oracleSummary !== undefined) { - evidence = { - ...evidence, - oracleSummary: - oracleSummary.kind === 'report' - ? await evidenceWriter.writeOracle(oracleSummary.report, oraclePreparation) - : await evidenceWriter.writeOracleFailure(oracleSummary.error, oraclePreparation), - }; - } - } catch (error) { - status = 'setup_failed'; - failure = { - phase: 'evidence_retention', - messageSha256: sha256(redactedError(error, roots)), - }; - } - const receipt = parsePetrinautHistoricalPreflightReceipt({ - schemaVersion: 1, - caseId: CASE_REFERENCE, - status, - setupStatus: status === 'passed' ? 'valid' : 'invalid', - ...(parent === undefined ? {} : { parent }), - ...(reference === undefined ? {} : { reference }), - ...(oracle === undefined ? {} : { oracle }), - evidence, - commandTrace: COMMAND_TRACE, - cleanup, - ...(failure === undefined ? {} : { failure }), - }); - const receiptBytes = `${JSON.stringify(receipt, null, 2)}\n`; - try { - await receiptHandle.writeFile(receiptBytes, 'utf8'); - } finally { - await receiptHandle.close(); - } - return { receiptFile: roots.receiptFile, receipt }; -} - -export function parsePetrinautHistoricalPreflightReceipt( - value: unknown, -): PetrinautHistoricalPreflightReceipt { - return petrinautHistoricalPreflightReceiptSchema.parse(value); -} - -async function loadClosedOraclePack(caseDir: string): Promise { - const [{ loadControllerOraclePack }, { resolveCompiledExecutionOracle }] = await Promise.all([ - import('./oracle-pack.js'), - import('../execution-comparison-operator.js'), - ]); - const compiled = resolveCompiledExecutionOracle(ORACLE_ID); - const pack = await loadControllerOraclePack({ - caseDir, - implementationFiles: compiled.implementationFiles, - }); - if (pack.manifest.id !== ORACLE_ID) { - throw new Error('Petrinaut preflight received a different compiled oracle'); - } - return { oracleId: ORACLE_ID, packSha256: pack.packSha256 }; -} - -async function validateInput(input: PetrinautHistoricalPreflightInput): Promise<{ - readonly sourceRepositoryDir: string; - readonly parentTargetDir: string; - readonly referenceTargetDir: string; - readonly controllerRoot: string; - readonly receiptFile: string; - readonly outputRoot: string; -}> { - for (const [name, path] of Object.entries(input)) { - if (!isAbsolute(path)) { - throw new Error(`Petrinaut preflight ${name} must be an absolute path`); - } - } - const sourceRepositoryDir = await existingRealDirectory(input.sourceRepositoryDir, 'source repository'); - const controllerRoot = await existingRealDirectory(input.controllerRoot, 'controller root'); - const outputRoot = await existingRealDirectory(dirname(input.receiptFile), 'output root'); - const parentRoot = await existingRealDirectory(dirname(input.parentTargetDir), 'parent workspace root'); - const referenceRoot = await existingRealDirectory( - dirname(input.referenceTargetDir), - 'reference workspace root', - ); - const parentTargetDir = resolve(parentRoot, basename(input.parentTargetDir)); - const referenceTargetDir = resolve(referenceRoot, basename(input.referenceTargetDir)); - const receiptFile = resolve(outputRoot, basename(input.receiptFile)); - await assertAbsent(parentTargetDir, 'parent target'); - await assertAbsent(referenceTargetDir, 'reference target'); - for (const file of Object.values(PREFLIGHT_EVIDENCE_FILES)) { - await assertAbsent(resolve(outputRoot, file), `retained evidence ${file}`); - } - assertPairwiseDisjoint([ - sourceRepositoryDir, - controllerRoot, - parentTargetDir, - referenceTargetDir, - outputRoot, - ]); - return { - sourceRepositoryDir, - parentTargetDir, - referenceTargetDir, - controllerRoot, - receiptFile, - outputRoot, - }; -} - -async function existingRealDirectory(path: string, name: string): Promise { - const stat = await lstat(path); - if (!stat.isDirectory() || stat.isSymbolicLink()) { - throw new Error(`Petrinaut preflight ${name} must be a real directory`); - } - return await realpath(path); -} - -async function assertAbsent(path: string, name: string): Promise { - try { - await lstat(path); - } catch (error) { - if (isNodeError(error, 'ENOENT')) return; - throw error; - } - throw new Error(`Petrinaut preflight ${name} must not already exist`); -} - -function assertPairwiseDisjoint(roots: readonly string[]): void { - for (let index = 0; index < roots.length - 1; index += 1) { - assertControllerIsolation({ - controllerRoot: roots[index]!, - targetRoots: roots.slice(index + 1), - }); - } -} - -async function assertParentContainsNoReference( - parentTargetDir: string, - referenceCommit: string, - referenceTargetDir: string, - runner: CommandRunner, -): Promise { - for (const forbidden of [referenceCommit, referenceTargetDir]) { - const result = await runner('git', ['grep', '-F', '--quiet', forbidden, '--', '.'], { - cwd: parentTargetDir, - timeoutMs: 30_000, - maxOutputBytes: 16 * 1024, - }); - if (result.exitCode === 0) { - throw new Error('historical reference leaked into the parent target'); - } - if (result.exitCode !== 1) { - throw new Error(`parent reference-leak scan failed: ${result.stderr || result.stdout}`); - } - for (const entry of await readdir(parentTargetDir, { withFileTypes: true })) { - if (!entry.isFile()) continue; - const handle = await open(resolve(parentTargetDir, entry.name), 'r'); - try { - const buffer = Buffer.alloc(64 * 1024); - const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); - if (buffer.subarray(0, bytesRead).toString('utf8').includes(forbidden)) { - throw new Error('historical reference leaked into the parent target'); - } - } finally { - await handle.close(); - } - } - } -} - -async function cleanupWorkspaces( - input: { - readonly parentTargetDir: string; - readonly referenceTargetDir: string; - }, - parentRemovedEarly: boolean, -): Promise<{ - readonly parentWorkspace: 'removed' | 'not_created' | 'failed'; - readonly referenceWorkspace: 'removed' | 'not_created' | 'failed'; -}> { - const [parentWorkspace, referenceWorkspace] = await Promise.all([ - parentRemovedEarly ? Promise.resolve('removed' as const) : removeOwnedWorkspace(input.parentTargetDir), - removeOwnedWorkspace(input.referenceTargetDir), - ]); - return { parentWorkspace, referenceWorkspace }; -} - -async function removeOwnedWorkspace(path: string): Promise<'removed' | 'not_created' | 'failed'> { - try { - await lstat(path); - } catch (error) { - return isNodeError(error, 'ENOENT') ? 'not_created' : 'failed'; - } - try { - await rm(path, { recursive: true }); - return 'removed'; - } catch { - return 'failed'; - } -} - -function redactedError( - error: unknown, - roots: { - readonly sourceRepositoryDir: string; - readonly parentTargetDir: string; - readonly referenceTargetDir: string; - readonly controllerRoot: string; - readonly outputRoot: string; - }, -): string { - let message = error instanceof Error ? error.message : String(error); - for (const root of Object.values(roots)) { - message = message.split(root).join(''); - } - return message; -} - -function findDependencyPreparationError(error: unknown): PetrinautDependencyPreparationError | undefined { - if (error instanceof PetrinautDependencyPreparationError) return error; - if (error instanceof HistoricalReplayTargetPreparationError) { - return findDependencyPreparationError(error.cause); - } - return undefined; -} - -function sha256(value: string): string { - return `sha256:${createHash('sha256').update(value).digest('hex')}`; -} - -function isNodeError(error: unknown, code: string): boolean { - return ( - typeof error === 'object' && - error !== null && - 'code' in error && - (error as { code?: string }).code === code - ); -} diff --git a/src/dev/execution-comparison/petrinaut-historical-preflight/evidence.ts b/src/dev/execution-comparison/petrinaut-historical-preflight/evidence.ts deleted file mode 100644 index 07c88d3dc..000000000 --- a/src/dev/execution-comparison/petrinaut-historical-preflight/evidence.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { createHash } from 'node:crypto'; -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { z } from 'zod'; - -import type { PetrinautDependencyPreparationObservation } from '../historical-replay-target.js'; -import type { PetrinautOraclePreparationObservation } from '../petrinaut-optimization-oracle/runner.js'; -import type { PetrinautOptimizationOracleReport } from '../petrinaut-optimization-oracle/types.js'; - -const HASH = /^sha256:[a-f0-9]{64}$/u; -const MAX_STREAM_BYTES = 48 * 1024; -const MAX_STATUS_BYTES = 16 * 1024; -const MAX_ORACLE_STREAM_BYTES = 8 * 1024; - -export const PREFLIGHT_EVIDENCE_FILES = { - parentDependency: 'parent-dependency.json', - referenceDependency: 'reference-dependency.json', - oracleSummary: 'oracle-summary.json', -} as const; - -export const retainedPreflightEvidenceSchema = z - .object({ - file: z.enum([ - PREFLIGHT_EVIDENCE_FILES.parentDependency, - PREFLIGHT_EVIDENCE_FILES.referenceDependency, - PREFLIGHT_EVIDENCE_FILES.oracleSummary, - ]), - sha256: z.string().regex(HASH), - bytes: z.number().int().positive(), - truncated: z.boolean(), - }) - .strict(); - -export type RetainedPreflightEvidence = z.infer; - -export function createPreflightEvidenceWriter(input: { - readonly outputRoot: string; - readonly redactedRoots: readonly string[]; -}): { - readonly writeDependency: ( - scope: 'parent' | 'reference', - observation: PetrinautDependencyPreparationObservation, - ) => Promise; - readonly writeOracle: ( - report: PetrinautOptimizationOracleReport, - preparation: readonly PetrinautOraclePreparationObservation[], - ) => Promise; - readonly writeOracleFailure: ( - error: unknown, - preparation: readonly PetrinautOraclePreparationObservation[], - ) => Promise; -} { - return { - writeDependency: async (scope, observation) => { - const stdout = sanitizeBounded(observation.commandResult.stdout, input.redactedRoots, MAX_STREAM_BYTES); - const stderr = sanitizeBounded(observation.commandResult.stderr, input.redactedRoots, MAX_STREAM_BYTES); - const spawnError = - observation.commandResult.spawnError === undefined - ? undefined - : sanitizeBounded(observation.commandResult.spawnError, input.redactedRoots, MAX_STATUS_BYTES); - const trackedSourceStatus = - observation.trackedSourceStatus === undefined - ? undefined - : sanitizeBounded(observation.trackedSourceStatus, input.redactedRoots, MAX_STATUS_BYTES); - return await retain( - input.outputRoot, - scope === 'parent' - ? PREFLIGHT_EVIDENCE_FILES.parentDependency - : PREFLIGHT_EVIDENCE_FILES.referenceDependency, - { - schemaVersion: 1, - kind: 'dependency_preparation', - scope, - recipe: observation.outcome.recipe, - command: observation.outcome.command, - args: observation.outcome.args, - status: observation.outcome.status, - exitCode: observation.outcome.exitCode, - ...('failureStage' in observation.outcome - ? { failureStage: observation.outcome.failureStage } - : {}), - commandResult: { - stdout: stdout.value, - stderr: stderr.value, - ...(spawnError === undefined ? {} : { spawnError: spawnError.value }), - aborted: observation.commandResult.aborted === true, - timedOut: observation.commandResult.timedOut === true, - outputTruncated: - observation.commandResult.outputTruncated === true || stdout.truncated || stderr.truncated, - }, - ...(trackedSourceStatus === undefined ? {} : { trackedSourceStatus: trackedSourceStatus.value }), - }, - observation.commandResult.outputTruncated === true || - stdout.truncated || - stderr.truncated || - spawnError?.truncated === true || - trackedSourceStatus?.truncated === true, - ); - }, - writeOracle: async (report, preparation) => { - const setupFailure = - report.setupFailure === undefined - ? undefined - : sanitizeBounded(report.setupFailure, input.redactedRoots, MAX_STATUS_BYTES); - const consoleErrors = sanitizeList(report.consoleErrors, input.redactedRoots); - const failedRequests = sanitizeList(report.failedRequests, input.redactedRoots); - const preparationLogs = sanitizeOraclePreparation(preparation, input.redactedRoots); - return await retain( - input.outputRoot, - PREFLIGHT_EVIDENCE_FILES.oracleSummary, - { - schemaVersion: 1, - kind: 'oracle_summary', - oracleId: report.oracleId, - status: report.status, - preparation: report.preparation, - preparationLogs: preparationLogs.values, - checks: report.checks.map(({ id, claims, status, evidence }) => ({ - id, - claims, - status, - evidence: sanitizeList(evidence, input.redactedRoots).values, - })), - ...(setupFailure === undefined ? {} : { setupFailure: setupFailure.value }), - consoleErrors: consoleErrors.values, - failedRequests: failedRequests.values, - }, - setupFailure?.truncated === true || - consoleErrors.truncated || - failedRequests.truncated || - preparationLogs.truncated, - ); - }, - writeOracleFailure: async (error, preparation) => { - const failure = sanitizeBounded( - error instanceof Error ? error.message : String(error), - input.redactedRoots, - MAX_STATUS_BYTES, - ); - const preparationLogs = sanitizeOraclePreparation(preparation, input.redactedRoots); - return await retain( - input.outputRoot, - PREFLIGHT_EVIDENCE_FILES.oracleSummary, - { - schemaVersion: 1, - kind: 'oracle_summary', - oracleId: 'petrinaut-optimization-oracles-v1', - status: 'setup_failed', - thrownFailure: failure.value, - preparationLogs: preparationLogs.values, - }, - failure.truncated || preparationLogs.truncated, - ); - }, - }; -} - -function sanitizeOraclePreparation( - observations: readonly PetrinautOraclePreparationObservation[], - roots: readonly string[], -): { readonly values: readonly unknown[]; readonly truncated: boolean } { - let truncated = false; - const values = observations.map(({ id, commandResult }) => { - const stdout = sanitizeBounded(commandResult.stdout, roots, MAX_ORACLE_STREAM_BYTES); - const stderr = sanitizeBounded(commandResult.stderr, roots, MAX_ORACLE_STREAM_BYTES); - const spawnError = - commandResult.spawnError === undefined - ? undefined - : sanitizeBounded(commandResult.spawnError, roots, MAX_ORACLE_STREAM_BYTES); - truncated ||= - commandResult.outputTruncated === true || - stdout.truncated || - stderr.truncated || - spawnError?.truncated === true; - return { - id, - exitCode: commandResult.exitCode, - stdout: stdout.value, - stderr: stderr.value, - ...(spawnError === undefined ? {} : { spawnError: spawnError.value }), - aborted: commandResult.aborted === true, - timedOut: commandResult.timedOut === true, - outputTruncated: commandResult.outputTruncated === true || stdout.truncated || stderr.truncated, - }; - }); - return { values, truncated }; -} - -async function retain( - outputRoot: string, - file: RetainedPreflightEvidence['file'], - value: unknown, - truncated: boolean, -): Promise { - const bytes = `${JSON.stringify(value, null, 2)}\n`; - await writeFile(join(outputRoot, file), bytes, { - encoding: 'utf8', - flag: 'wx', - }); - return retainedPreflightEvidenceSchema.parse({ - file, - sha256: sha256(bytes), - bytes: Buffer.byteLength(bytes, 'utf8'), - truncated, - }); -} - -function sanitizeList( - values: readonly string[], - roots: readonly string[], -): { readonly values: readonly string[]; readonly truncated: boolean } { - let remaining = MAX_STREAM_BYTES; - let truncated = false; - const sanitized: string[] = []; - for (const value of values) { - if (remaining <= 0) { - truncated = true; - break; - } - const result = sanitizeBounded(value, roots, remaining); - sanitized.push(result.value); - remaining -= Buffer.byteLength(result.value, 'utf8'); - truncated ||= result.truncated; - } - return { values: sanitized, truncated }; -} - -function sanitizeBounded( - value: string, - roots: readonly string[], - maxBytes: number, -): { readonly value: string; readonly truncated: boolean } { - let sanitized = value; - for (const root of [...roots].sort((left, right) => right.length - left.length)) { - sanitized = sanitized.split(root).join(''); - } - sanitized = sanitized - .replace(/:\/\/[^/\s:@]+:[^/\s@]+@/gu, '://@') - .replace(/\b(token|password|authorization|api[_-]?key)\s*[:=]\s*[^\s,;]+/giu, '$1=') - .replace(/(^|[\s("'=])\/(?:[^/\s"'(),;]+\/)*[^/\s"'(),;]*/gmu, '$1') - .replace(/[A-Za-z]:\\(?:[^\\\s"'(),;]+\\)*[^\\\s"'(),;]*/gu, ''); - const buffer = Buffer.from(sanitized, 'utf8'); - if (buffer.byteLength <= maxBytes) { - return { value: sanitized, truncated: false }; - } - return { - value: buffer.subarray(0, maxBytes).toString('utf8'), - truncated: true, - }; -} - -function sha256(value: string): string { - return `sha256:${createHash('sha256').update(value).digest('hex')}`; -} diff --git a/testing/comparisons/missions/README.md b/testing/comparisons/missions/README.md index ffc5dc268..c882cdbc2 100644 --- a/testing/comparisons/missions/README.md +++ b/testing/comparisons/missions/README.md @@ -9,6 +9,8 @@ Current library: - [`minimal-petri-net-editor.md`](minimal-petri-net-editor.md) — Petri-editor elicitation and end-to-end origin mission +- [`brunch-host-landing.md`](brunch-host-landing.md) — Brunch host-landing brownfield mission +- [`petrinaut-optimization.md`](petrinaut-optimization.md) — Petrinaut optimization brownfield mission This `README.md` is the directory's reserved control file, not a mission. `/compare-specs` must exclude it from mission listing, resolution, revision, and creation. @@ -31,4 +33,6 @@ The top-level session drives exactly one direct `interactive_shell` comparison h Missions remain editable. A revision changes only the mission and future runs; it never rewrites an existing run directory, private mission snapshot, harness-setup snapshot, transcript, target output, or report. Ephemeral assembly belongs under `.fixtures/scratch/comparisons/`; deliberately retained immutable evidence belongs under `.fixtures/runs/agent-as-user-comparison/`. +Each mission with a matching directory under `testing/execution-comparisons/cases/` may also be run as an execution-only comparison through `/compare-execution `. That path freezes `spec.md` and `public-contract.json`, captures `provenance.json` before the first lane, retains validated attempts and case-owned oracle reports, and writes the validity-first `report.md` consumed by `/comparison-publish`. Elicitation and execution remain separate phases unless an explicitly retained end-to-end matrix is run. + The retained operator-only report may reproduce the complete private mission as its baseline. It must visibly separate that top-level-session-only baseline from each harness's exact visible framing and transcript, outcomes, and unchanged harness-authored document so elicitation and leakage remain legible. It does not choose a winner or impose a fixed rubric.