diff --git a/.agents/skills/comparison-publish/SKILL.md b/.agents/skills/comparison-publish/SKILL.md new file mode 100644 index 000000000..958c284a9 --- /dev/null +++ b/.agents/skills/comparison-publish/SKILL.md @@ -0,0 +1,103 @@ +--- +name: comparison-publish +description: Explicitly validate and idempotently publish one retained Brunch comparison report to the canonical Notion Comparison Reports database. Use only when the operator invokes /comparison-publish with a run directory. +argument-hint: "" +disable-model-invocation: true +--- + +# Comparison Publish + +Publish one completed comparison run with `/comparison-publish `. This skill is operator-invoked post-run reporting; it never conducts an experiment, starts a lane, judges a live run, or publishes automatically. + +Before proceeding, load [Comparison reporting](../comparison-reporting/SKILL.md) for validity-first evidence policy and [Notion reporting](../notion-reporting/SKILL.md) for safe mutation and verification. Comparison reporting owns the report grammar. Notion reporting owns formatting and the smallest safe mutation. + +## Fixed destination and identity + +- Hub: `brunch Testing Scenarios`, page id `3a53c81f-e024-8045-98e3-cf55aef279eb`. +- Database: `Comparison Reports`, data source `collection://01130dd8-6a1c-420a-b84d-166766550163`. +- Upsert key: the compound identity `Run ID + Phase`. Neither field alone is unique. +- One database row is the report page. Never create a separate report page plus a register row. + +Always discover the current Notion MCP tool schema, fetch the hub, fetch the database/data source, and inspect its live properties before querying or writing. Names and option ids in this file are intent, not permission to assume a stale schema. + +## 1. Resolve and validate retained evidence + +Resolve the argument from the repository root to one real run directory. Reject an absent directory, symlink, traversal outside the repository, or ambiguous path. Require regular files named `report.md` and `provenance.json`. The report must be a completed retained report, not a draft assembled for publication. + +Parse `provenance.json` against `src/dev/comparison-provenance.ts`. Require: + +- schema version `1`; +- one comparison kind: `elicitation`, `execution`, or `end_to_end`; +- the exact run id and canonical capture timestamp; +- root package name and release version; +- exact tag or `null`; +- full 40-character controller commit, matching clickable commit URL, branch or `null`, and dirty boolean. + +The run-directory identity and the provenance run id must agree. Treat a missing, malformed, or mismatched provenance artifact as a hard stop. Never infer historical provenance from the current checkout, current `HEAD`, current package version, a later tag, a report date, or operator memory. + +Validate the retained contracts that exist for the declared kind: + +- **Elicitation:** require the approved setup snapshot, lane-visible transcripts/interactions, lane terminal and cleanup evidence, and unchanged produced-document references claimed by `report.md`. Distinguish the approachable shared-actor procedure from a rigorous campaign. +- **Execution:** parse every claimed attempt with `src/dev/execution-comparison/artifact-contract.ts`; use only audience-safe attempt packets and the two public case files named by comparison reporting. Require validity, terminal, cleanup, common command/browser, and retained-output references claimed by `report.md`. +- **End-to-end:** validate the study contract, exact-byte handoffs, four-cell matrix or explicitly retained missing cells, requirement ledger, and underlying execution attempts with `src/dev/end-to-end-comparison/` contracts. + +Do not recursively inspect controller-only case directories. A missing contract or report claim unsupported by retained evidence stops publication; do not repair evidence, infer a result, or silently downgrade the gap. + +## 2. Produce the publication copy + +Read `report.md` using comparison reporting's evidence order: study design, provenance, validity before outcomes, common evidence, process evidence, lane-only diagnostics, then bounded interpretation. Preserve failed and invalid attempts. + +Create an audience-safe Notion body. Never publish controller-only oracle definitions, hidden fixtures, expected states, selector mappings, reveal material, private controller paths, secrets, or opaque reasoning. Include only aggregate hidden-oracle outcomes and portable evidence references. Private mission text may appear only when the fixed destination has been explicitly approved as an operator/controller audience; otherwise summarize its public concern without reproducing it. + +The page must include an **Identity** section containing: + +- Run ID and Phase; +- capture timestamp; +- root package name and release version; +- exact tag, or `none`; +- full controller commit SHA as its retained commit URL; +- branch, or `detached`; +- dirty state. + +Keep validity before outcomes in the page body. Do not add a winner, score, parity claim, causal claim, or broad benchmark claim that the frozen procedure and valid retained evidence do not support. + +## 3. Map live database properties + +Map the publication copy to the current live properties: + +- `Report`: report-page title; +- `Date`: provenance capture date; +- `Release`: root package version (the Identity section also records the exact tag); +- `Commit`: provenance controller commit URL; +- `Phase`: `Elicitation`, `Execution`, or `End-to-end` from comparison kind; +- `Result`: the report's explicit bounded outcome, using an existing live option; +- `Evidence`: retained-evidence completeness, using an existing live option; +- `Status`: validity/publication status, using an existing live option; +- `Run ID`: exact provenance run id; +- `Case`: exact public case/mission identifier supported by the report. + +Do not invent an option or coerce an unsupported classification. If an exact mapping is absent, show the operator the mismatch and stop before mutation. + +## 4. Compound upsert + +Query the live data source for exact `Run ID + Phase` equality: + +- **Zero matches:** create one row in `Comparison Reports` with the mapped properties and the audience-safe report body. +- **One match:** fetch that row page first. Update only changed properties and the generated report content. Preserve comments, child pages/databases, and unrelated child content. Do not replace a full page merely for convenience. +- **More than one match:** stop and report the duplicate row URLs. Never pick one, merge them, or create another row. + +Use the smallest safe mutation. Keep the row's existing page identity on update so repeated publication is idempotent. + +## 5. Verify every write + +Re-query the database and fetch the report page after every write. Confirm: + +1. exactly one row matches `Run ID + Phase`; +2. every mapped property has the intended typed value; +3. the Identity section contains the immutable release, full SHA/link, tag, branch, and dirty state; +4. validity precedes outcomes and retained evidence links resolve; +5. controller-only material is absent; +6. existing comments, child pages/databases, and unrelated content remain; and +7. a second publication would update the same page rather than create another. + +Report completion only after this readback. Return the row URL, compound identity, create/update disposition, release, full commit SHA, and verification result. diff --git a/.changeset/publish-comparison-provenance.md b/.changeset/publish-comparison-provenance.md new file mode 100644 index 000000000..da26e9dbd --- /dev/null +++ b/.changeset/publish-comparison-provenance.md @@ -0,0 +1,6 @@ +--- +"@hashintel/brunch": minor +--- + +- Build coherent execution plans by requiring frontier-verified multi-slice epics to converge through one terminal integration slice. +- Add immutable comparison provenance and guarded Notion report publishing. diff --git a/.pi/prompts/compare-execution.md b/.pi/prompts/compare-execution.md index beb11f369..84207e8ff 100644 --- a/.pi/prompts/compare-execution.md +++ b/.pi/prompts/compare-execution.md @@ -61,7 +61,18 @@ Display all of the following together before any target preparation or launch: Ask through ordinary typed text for explicit **approve**, **revise**, or **reject**. Ambiguity, questions, qualifications, partial approval, or silence are not approval. Revise and redisplay the complete setup, or reject and stop. Do not prepare a target, start a shell, invoke an executor, or invoke the oracle before explicit approval. -After approval, save byte-identical `spec.md`, `public-contract.json`, and `run-setup.md` snapshots under the unused scratch run identity. Keep the two frozen inputs separate and unchanged. +Immediately after approval and before saving snapshots or preparing the first lane, capture the controller checkout once: + +```sh +npx tsx src/dev/comparison-provenance.ts capture \ + --run-directory .fixtures/scratch/execution-comparisons/ \ + --comparison-kind execution \ + --run-id +``` + +Treat an existing `provenance.json`, malformed metadata, or capture failure as a setup collision/failure and stop. Never overwrite it or reconstruct provenance from the checkout used later for reporting. + +Then save byte-identical `spec.md`, `public-contract.json`, and `run-setup.md` snapshots under the same scratch run identity. Keep the two frozen inputs separate and unchanged. ## Run approved lanes @@ -154,8 +165,8 @@ The immutable writer rejects an existing attempt id. Never overwrite, delete, re ## Report -After all selected lanes terminate and cleanup is proven, write `report.md` under the run scratch root. Present validity before outcomes for each lane. Then report factual terminal state, cleanup, produced output, common command results, and unchanged browser-oracle results. +After all selected lanes terminate and cleanup is proven, write `report.md` beside the unchanged run-start `provenance.json` under the run scratch root. Present validity before outcomes for each lane. Then report factual terminal state, cleanup, produced output, common command results, and unchanged browser-oracle results. Keep common evidence and Brunch-only diagnostics visibly separate. Use `not_assessable` for unavailable common evidence. Do not score, rank, choose a winner, claim reliability, infer parity from unavailable or product-private evidence, or let Brunch-only run/Petri/debug data improve its common result. Do not turn ordinary visual hierarchy, clarity, or drag feel into an automatic mechanical verdict. -Finish with the case id, run id, executor order, each attempt id and terminal/validity state, cleanup result, retained output paths, browser report paths, attempt-record paths, and `report.md`. State any remaining human witness plainly. +Finish with the case id, run id, `provenance.json` path, executor order, each attempt id and terminal/validity state, cleanup result, retained output paths, browser report paths, attempt-record paths, and `report.md`. State any remaining human witness plainly. diff --git a/.pi/prompts/compare-specs.md b/.pi/prompts/compare-specs.md index 8920d4a27..40e34efd5 100644 --- a/.pi/prompts/compare-specs.md +++ b/.pi/prompts/compare-specs.md @@ -61,7 +61,18 @@ Run setup is intentionally small, run-specific, and never written into the reusa 5. Display together: the complete private mission; selected harnesses and order; exact per-harness target-visible framing; mission id; collision-safe run id; requested target document path; scratch and retained paths; and each adapter. Clearly label the mission **top-level-session-only** and the framing **harness-visible**. 6. Ask through ordinary text for explicit **approve**, **revise**, or **reject**. Revise and redisplay the complete setup, or reject and stop; do not launch partially. -Before the first harness, copy the exact approved private mission and separate setup into the unused scratch run identity as `private-mission.md` and `harness-setup.md`. They are retained with the run but never merged into one target packet. +Immediately after approval and before copying snapshots or starting the first harness, capture the controller checkout once: + +```sh +npx tsx src/dev/comparison-provenance.ts capture \ + --run-directory .fixtures/scratch/comparisons/ \ + --comparison-kind elicitation \ + --run-id +``` + +Run this from the Brunch repository root. Treat an existing `provenance.json`, malformed metadata, or capture failure as a setup collision/failure and stop; never overwrite it or reconstruct provenance from a later checkout. + +Then copy the exact approved private mission and separate setup into the same unused scratch run identity as `private-mission.md` and `harness-setup.md`. They are retained with `provenance.json` but never merged into one target packet. ## Run the approved mission @@ -83,11 +94,11 @@ Run selected harnesses sequentially. For each harness: 7. Acquire the harness-authored document if it exists. Never author, reconstruct, complete, rewrite, or improve it. Missing or partial output remains missing or partial. 8. On every outcome, retain state, harness-visible interaction, target-cwd/session identity, final process status, document that exists, and cleanup notes. Kill remaining processes, dismiss the completed shell record, and verify no comparison-harness shell or process remains before starting another. -Do not notify completion while any harness is ready, running, or waiting. After every selected harness is finished or failed, give one aggregate notification. Review scratch assembly, then deliberately copy it to the unused immutable run identity without changing snapshots, transcripts, or harness-authored documents. +Do not notify completion while any harness is ready, running, or waiting. After every selected harness is finished or failed, give one aggregate notification. Review scratch assembly, then deliberately copy it to the unused immutable run identity without changing `provenance.json`, snapshots, transcripts, or harness-authored documents. ## Retained run and operator-only report -The retained run contains distinct `private-mission.md` and `harness-setup.md` snapshots, every exact harness-visible transcript, every harness-authored document that exists, target identities/outcomes/cleanup, explanations for missing output, and `report.md`. +The retained run contains the unchanged run-start `provenance.json`, distinct `private-mission.md` and `harness-setup.md` snapshots, every exact harness-visible transcript, every harness-authored document that exists, target identities/outcomes/cleanup, explanations for missing output, and `report.md`. Write `report.md` for an operator-only cold reader. It may reproduce the full private mission, but must keep these sections visibly separate: @@ -99,4 +110,4 @@ Write `report.md` for an operator-only cold reader. It may reproduce the full pr Do not add a fixed rubric, score, scripted/API judge, statistics, recommendation, automatic winner, or inferred winner. -Finish by reporting the saved mission path, scratch path, immutable run path, harness terminal states, cleanup result, and `report.md` path. Do not claim runtime correctness, isolation, user consistency, notification timing, or usefulness merely because static instructions or checks passed; those judgments belong to the operator reviewing the real interaction and artifacts. +Finish by reporting the saved mission path, scratch path, immutable run path, `provenance.json` path, harness terminal states, cleanup result, and `report.md` path. Do not claim runtime correctness, isolation, user consistency, notification timing, or usefulness merely because static instructions or checks passed; those judgments belong to the operator reviewing the real interaction and artifacts. diff --git a/docs/praxis/comparison-runs.md b/docs/praxis/comparison-runs.md index efa300658..a4a9c963b 100644 --- a/docs/praxis/comparison-runs.md +++ b/docs/praxis/comparison-runs.md @@ -47,6 +47,27 @@ npm run dev-cli -- export --workspace --spec-id 1 --out `rpc` and `print` are deterministic observations. `mutate` is the explicit local write seam. `export` turns reviewed workbench state into a candidate fixture; review it before tracking it. +## Immutable run-start provenance + +Every new elicitation, execution, or end-to-end comparison captures `provenance.json` immediately after the operator approves the setup and before the first lane starts. The snapshot records the root package release, exact tag when present, full controller commit and URL, branch, dirty state, run identity, and capture time: + +```sh +npx tsx src/dev/comparison-provenance.ts capture \ + --run-directory \ + --comparison-kind \ + --run-id +``` + +The writer is intentionally collision-failing. Never overwrite this file, recapture it after a checkout changes, or infer historical provenance from the checkout used to publish a report. Keep the snapshot when promoting a run from scratch to `.fixtures/runs/`. Existing historical runs without a retained snapshot remain unchanged and must not be backfilled from memory or current Git state. + +After the run is complete and its retained `report.md` is final, publish explicitly: + +```text +/comparison-publish +``` + +The publication skill validates `report.md`, `provenance.json`, and the available kind-specific contracts; produces a validity-first, controller-safe copy; and upserts the existing Notion Comparison Reports database by `Run ID + Phase`. A repeated invocation updates the same report row, while duplicate matching rows stop publication for operator resolution. Publication never infers the release or commit from the publisher's checkout. + ## Quick start: compare execution From a trusted top-level project Pi session, run: @@ -81,6 +102,8 @@ immutable `ExecutionAttempt` format. Reports must present elicitation validity, execution validity, and requirement-level output evidence separately. This one-case tracer supports within-executor/spec contrasts only—not a winner, reliability estimate, or causal claim. +After the complete end-to-end setup is approved, capture `end_to_end` provenance in its scratch run root before starting either elicitation lane. Promote that exact snapshot beside the handoffs, attempts, ledger, and report; the later execution matrix does not recapture it. + The first promoted witness is [`petri-editor-e2e-20260721T132600Z`](../../.fixtures/runs/end-to-end-comparison/petri-editor-e2e-20260721T132600Z/). It retains both exact handoffs, all four valid failed execution cells, portable common evidence, the @@ -125,10 +148,11 @@ The round-one materials remain useful for focused improvement and regression stu The rigorous campaign loop is: 1. instantiate [`comparison-runs/mission-packet.md`](comparison-runs/mission-packet.md), keeping the reveal key outside every target cwd; -2. drive one fresh lane with [the actor recipe](../../.agents/skills/agent-as-user-comparison/SKILL.md); -3. retain target-visible interaction, validity/intervention notes, cleanup status, and the target-authored ready document under `.fixtures/scratch/comparisons//`; -4. when the study needs structured judgment, run the masked-outcome and unblinded-process passes from the judgment prompt pack and have the named human adjudicator review the drafts; and -5. after review, promote the portable bundle to `.fixtures/runs/agent-as-user-comparison//` and run `npm run check:promoted-run-paths` before commit. +2. after setup approval, capture immutable `elicitation` provenance under `.fixtures/scratch/comparisons//` before the first lane; +3. drive one fresh lane with [the actor recipe](../../.agents/skills/agent-as-user-comparison/SKILL.md); +4. retain target-visible interaction, validity/intervention notes, cleanup status, and the target-authored ready document under `.fixtures/scratch/comparisons//`; +5. when the study needs structured judgment, run the masked-outcome and unblinded-process passes from the judgment prompt pack and have the named human adjudicator review the drafts; and +6. after review, promote the portable bundle, including unchanged `provenance.json`, to `.fixtures/runs/agent-as-user-comparison//` and run `npm run check:promoted-run-paths` before commit. Use push-driven hands-free control: project config lowers the query fallback floor to 5 seconds, forwards quiet output after roughly 3 seconds, and prunes superseded viewport reads before each LLM call. Send input and end the turn; act on the pushed quiet update. Query only when no push arrives or the incremental tail is ambiguous, and then read the current tail—never page historical scrollback. diff --git a/memory/PLAN.md b/memory/PLAN.md index 78a7f641b..0d8b86515 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -118,6 +118,7 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand - `end-to-end-comparison-tracer` ([FE-1239](https://linear.app/hash/issue/FE-1239/trace-elicitation-through-execution)) — **complete on `ka/fe-1239-end-to-end-comparison-tracer`, stacked on FE-1230:** two rigorous Petri-editor elicitation outputs crossed exact immutable handoffs into the staged 2×2 Brunch/Claude matrix; all four valid failed cells, unchanged-oracle evidence, and the validity-first requirement ledger are promoted. Definition and retained witness below. - `executor-slice-admission-parity` ([FE-1240](https://linear.app/hash/issue/FE-1240/prevent-invalid-scoped-slices-from-reaching-execution)) — **complete on `ka/fe-1240-slice-admission-parity`, restacked on `next` after FE-1239 landed:** incomplete scoped-slice worker context is rejected during deterministic plan admission, exact findings enter bounded repair before `slice_execute`, and the execution boundary remains fail-closed. Definition below. - `executor-plan-coherence` ([FE-1250](https://linear.app/hash/issue/FE-1250/build-coherent-execution-plans)) — **implementation complete on `ka/fe-1250-coherent-execution-plans`, based on FE-1240 via `next`:** frontier-verified multi-slice epics now require one ordinary terminal member over every sibling, planner conduct establishes shared foundations, and workers preserve cumulative public contracts for the canonical harness. Fast verification passes; next is the separately authorized unchanged Petri comparison rerun. No new browser gate, durable plan kind, or executor lifecycle phase. Definition below. +- `comparison-publication-workflow` ([FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports)) — **active on `ka/fe-1251-comparison-publication`, stacked on FE-1250:** capture immutable controller and release provenance before comparison lanes start, then explicitly publish retained validity-first reports into the canonical Notion database through a guarded idempotent skill. Definition below. - **Carved from FE-1167 (2026-07-13):** the Execute-mode evidence sub-list — Execute entry beats on thin vs rich seeds (assessment honesty: Ask on thin, Proceed on rich) and the FE-1107/KA residue (close-or-narrow, demo/walkthrough session via `TESTING_PLAN.md`, post-KA plan pass). The former sticky-posture question is no longer KA residue: FE-1187 `remediation-4` owns the persistent Specify elicitation-style audit/SPEC revision, and its Continue lexical audit owns the old `continue` ambiguity. Full context in the archived FE-1167 definition (`docs/archive/PLAN_HISTORY.md`). - `planning-process-model` — **moved to the KA stream 2026-07-13; reshaped by D126-L**: the durable scope handoff is settled, so this item now owns only plan projection and epistemic-horizon questions beyond committed scopes. Definition below. - **[1.x data-model handoff owed by FE-1187](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md):** a concise colleague-facing note must distinguish current canon from directional vocabulary before the consolidated outer checkpoint. Current: `{milestone, frontier, scope}` with executor-derived slices; basis (`explicit | implicit`) orthogonal to settlement (`advisory | settled`); no persisted readiness grade or spec-global elicitation-gap table; active-branch Pi JSONL reads; one CommandExecutor mutation authority with spec-local LSN/change log; no new projected `vv_obligation` (legacy rows remain readable). Directional only: explain stored `thesis` as pitch/concept without renaming it yet, and avoid new coupling to spec-local `term` while its possible workspace lift remains future work. Persisted judgment-shaped reconciliation needs remain current, but are YAGNI-suspect: add no kinds/consumers/orchestration dependency without fresh evidence; re-evaluate derivation/removal when KA work first needs that table. Delivered note: [`Brunch 1.x data-model handoff`](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md); it does not block the deterministic TUI queue. @@ -472,6 +473,19 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Verification:** structural skill-contract plus recursive relative-link check chained into `npm run check:skills`; Markdown links and promoted-run paths; tracked-file lint/format; default tests and build; authoritative full CI gate. - **Traceability:** FE-1210/FE-1215; `agent-as-user-comparison`; `docs/praxis/comparison-runs.md`; FE-1230 `ExecutionAttempt`, masked outcome, and unblinded process packets. +### comparison-publication-workflow + +- **Name:** Publish comparison reports with immutable provenance +- **Linear / branch:** [FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports); `ka/fe-1251-comparison-publication`, stacked on FE-1250. +- **Kind:** project workflow tooling — write-once run-start provenance plus explicitly invoked, verified Notion publication; no product runtime mutation or autonomous experiment conduct. +- **Certainty:** proving. +- **Status:** active 2026-07-22. +- **Objective:** make retained comparison reports reproducibly attributable to the tested Brunch release and controller checkout, then publish each `Run ID + Phase` exactly once into the canonical Comparison Reports database with safe update semantics. +- **Acceptance:** elicitation, execution, and end-to-end flows capture a schema-versioned `provenance.json` before their first lane and reject collisions or malformed inputs; `/comparison-publish ` requires retained provenance and report evidence, applies validity-first interpretation and controller-only redaction, creates or updates exactly one matching database row, stops on duplicates, and verifies every mutation by re-query and fetch; Release and Commit are filterable Notion properties and remain absent on historical rows without exact retained evidence. +- **Boundary:** in — dev-only provenance contract/CLI, comparison prompts and run documentation, project-local publication skill, reporting-skill guardrails, and the live Testing Scenarios schema/legend. Out — product runtime publication, inferred historical provenance, automatic scoring, unsupervised experiment conduct, or changes to comparison artifact contracts. +- **Dependencies:** operationally stacked after FE-1250; builds on FE-1232 reporting and Notion safety contracts plus retained comparison artifacts established by FE-1210/FE-1230/FE-1239. +- **Verification:** focused provenance tests; reporting skill checks; default verify gate; disposable create-then-update publication exercise plus duplicate detection; final Notion schema, row, and legend readback. + ### executor-plan-synthesis - **Name:** Synthesize and validate executor plans from approved scopes diff --git a/scripts/check-reporting-skills.mjs b/scripts/check-reporting-skills.mjs index 7220acc93..4a4ed0d7c 100644 --- a/scripts/check-reporting-skills.mjs +++ b/scripts/check-reporting-skills.mjs @@ -194,8 +194,38 @@ requirePhrases('.agents/skills/comparison-reporting/references/report-examples.m 'Hidden fixtures and exact oracle journeys are intentionally omitted', ]); +const publishSkillPath = '.agents/skills/comparison-publish/SKILL.md'; +const publishSkill = requirePhrases(publishSkillPath, [ + 'name: comparison-publish', + 'disable-model-invocation: true', + '/comparison-publish ', + '../comparison-reporting/SKILL.md', + '../notion-reporting/SKILL.md', + 'collection://01130dd8-6a1c-420a-b84d-166766550163', + 'Run ID + Phase', + 'provenance.json', + 'Never infer historical provenance from the current checkout', + 'validity before outcomes', + 'controller-only', + 'smallest safe mutation', + 'More than one match', + 'Re-query the database and fetch the report page after every write', +]); +const publishDescription = publishSkill.match(/^description:\s*(.+)$/m)?.[1] ?? ''; +if ( + !publishDescription.includes('comparison') || + !publishDescription.includes('Notion') || + !publishDescription.includes('/comparison-publish') +) { + fail(`${publishSkillPath}: description must name the explicit comparison publication trigger`); +} +if (!/^disable-model-invocation:\s*true$/m.test(publishSkill)) { + fail(`${publishSkillPath}: publication skill must disable model invocation`); +} + checkRelativeMarkdownLinks('.agents/skills/notion-reporting'); checkRelativeMarkdownLinks('.agents/skills/comparison-reporting'); +checkRelativeMarkdownLinks('.agents/skills/comparison-publish'); if (errors.length > 0) { console.error(`check:reporting-skills FAILED (${errors.length})`); @@ -203,4 +233,4 @@ if (errors.length > 0) { process.exit(1); } -console.log('check:reporting-skills OK — 2 reporting skill contracts consistent'); +console.log('check:reporting-skills OK — 3 reporting skill contracts consistent'); diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index fe1e63d55..823430c95 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -4,7 +4,7 @@ This directory owns Brunch-only development loops and curation seams. Nothing he ## Ownership -`src/dev/**` owns nine things: +`src/dev/**` owns ten things: - the human-facing dev launcher (`scripts/dev.ts` → `src/dev/dev-cli.ts`) - the explicit graph-curation seam for fixture shaping (`graph-curation.ts`) @@ -13,6 +13,7 @@ This directory owns Brunch-only development loops and curation seams. Nothing he - controller-owned execution-comparison packets, lane adapters, immutable evidence contracts, and independently executable black-box oracle journeys (`execution-comparison/`) - controller-owned end-to-end comparison composition (`end-to-end-comparison.ts` and `end-to-end-comparison/`): content-addressed study and exact-byte handoff contracts, isolated Brunch/Claude adapters, a closed four-cell matrix over unchanged `ExecutionAttempt` leaves, requirement traceability, and audience-safe redaction - the shell-callable execution-comparison operator wrapper (`execution-comparison-operator.ts`), which lists/resolves frozen cases, prepares only Brunch/Claude targets, invokes the existing oracle after lane termination, and validates immutable attempt records for the project-local `/compare-execution` prompt +- the shell-callable comparison provenance writer (`comparison-provenance.ts`), which captures one write-once release/controller snapshot after setup approval and before any comparison lane - the standalone component preview harness (`scripts/dev-components.ts` → `src/dev/component-preview.ts`) for previewing `.pi/components` in isolation on a real terminal, with no workspace/session/DB - the agent-drivable PTY walkthrough fallback (`npm run tui-driver` → `src/dev/tui-driver.ts`): named `expect`-pumped PTY sessions with guarded fifo control, headless-xterm screen rendering, and wait-for-text; use it when the canonical project-local `pi-interactive-shell` overlay cannot bind in a sandbox/headless host; sessions live under gitignored `.fixtures/scratch/tui-driver/` diff --git a/src/dev/comparison-provenance.test.ts b/src/dev/comparison-provenance.test.ts new file mode 100644 index 000000000..6e285f7cb --- /dev/null +++ b/src/dev/comparison-provenance.test.ts @@ -0,0 +1,187 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { + captureComparisonProvenance, + parseComparisonProvenance, + runComparisonProvenanceCli, +} from './comparison-provenance.js'; + +const execFileAsync = promisify(execFile); +const roots: string[] = []; + +afterEach(async () => { + await Promise.all(roots.splice(0).map(async (root) => await rm(root, { recursive: true, force: true }))); +}); + +describe('comparison provenance capture', () => { + it('captures a clean untagged controller checkout', async () => { + const repository = await createControllerRepository(); + const result = await captureComparisonProvenance({ + runDirectory: join(repository.root, '.fixtures', 'run-clean'), + comparisonKind: 'elicitation', + runId: 'mission-20260722T140000Z-a1b2', + controllerRoot: repository.root, + capturedAt: '2026-07-22T14:00:00.000Z', + }); + + expect(result.provenance).toEqual({ + schemaVersion: 1, + comparisonKind: 'elicitation', + runId: 'mission-20260722T140000Z-a1b2', + capturedAt: '2026-07-22T14:00:00.000Z', + rootPackage: { + name: '@hashintel/brunch-test', + version: '1.0.0-alpha.9', + }, + exactTag: null, + controller: { + commitSha: repository.commitSha, + commitUrl: `https://github.com/hashintel/brunch/commit/${repository.commitSha}`, + branch: 'main', + dirty: false, + }, + }); + expect( + parseComparisonProvenance(JSON.parse(await readFile(result.provenancePath, 'utf8')) as unknown), + ).toEqual(result.provenance); + }); + + it('captures an exact release tag', async () => { + const repository = await createControllerRepository({ tag: 'v1.0.0-alpha.9' }); + const { provenance } = await captureComparisonProvenance({ + runDirectory: join(repository.root, '.fixtures', 'run-tagged'), + comparisonKind: 'execution', + runId: 'petri-editor-20260722T140000Z-a1b2', + controllerRoot: repository.root, + }); + + expect(provenance.exactTag).toBe('v1.0.0-alpha.9'); + expect(provenance.rootPackage.version).toBe('1.0.0-alpha.9'); + }); + + it('records dirty state before writing the provenance artifact', async () => { + const repository = await createControllerRepository(); + await writeFile(join(repository.root, 'dirty.txt'), 'uncommitted\n'); + + const { provenance } = await captureComparisonProvenance({ + runDirectory: join(repository.root, '.fixtures', 'run-dirty'), + comparisonKind: 'end_to_end', + runId: 'petri-e2e-20260722T140000Z', + controllerRoot: repository.root, + }); + + expect(provenance.controller.dirty).toBe(true); + }); + + it('rejects a second write to the same run identity', async () => { + const repository = await createControllerRepository(); + const input = { + runDirectory: join(repository.root, '.fixtures', 'run-collision'), + comparisonKind: 'execution' as const, + runId: 'collision-run', + controllerRoot: repository.root, + }; + + await captureComparisonProvenance(input); + await expect(captureComparisonProvenance(input)).rejects.toThrow('comparison provenance already exists'); + }); + + it('rejects malformed capture input and malformed retained provenance', async () => { + const repository = await createControllerRepository(); + await expect( + captureComparisonProvenance({ + runDirectory: join(repository.root, '.fixtures', 'run-invalid'), + comparisonKind: 'execution', + runId: '../escape', + controllerRoot: repository.root, + }), + ).rejects.toThrow('comparison run id'); + await expect( + runComparisonProvenanceCli([ + 'capture', + '--run-directory', + join(repository.root, '.fixtures', 'run-invalid-kind'), + '--comparison-kind', + 'unknown', + '--run-id', + 'valid-run', + '--controller-root', + repository.root, + ]), + ).rejects.toThrow('comparison kind'); + expect(() => + parseComparisonProvenance({ + schemaVersion: 1, + comparisonKind: 'execution', + runId: 'valid-run', + capturedAt: 'not-a-date', + rootPackage: { name: '@hashintel/brunch', version: '1.0.0-alpha.9' }, + exactTag: null, + controller: { + commitSha: repository.commitSha, + commitUrl: `https://github.com/hashintel/brunch/commit/${repository.commitSha}`, + branch: 'main', + dirty: false, + }, + }), + ).toThrow('canonical ISO timestamp'); + }); + + it('rejects malformed root package metadata', async () => { + const repository = await createControllerRepository(); + await writeFile(join(repository.root, 'package.json'), '{"name":"missing-version"}\n'); + + await expect( + captureComparisonProvenance({ + runDirectory: join(repository.root, '.fixtures', 'run-malformed-package'), + comparisonKind: 'elicitation', + runId: 'malformed-package', + controllerRoot: repository.root, + }), + ).rejects.toThrow('controller root package version'); + }); +}); + +async function createControllerRepository(input: { readonly tag?: string } = {}): Promise<{ + readonly root: string; + readonly commitSha: string; +}> { + const root = await mkdtemp(join(tmpdir(), 'brunch-comparison-provenance-')); + roots.push(root); + await writeFile( + join(root, 'package.json'), + `${JSON.stringify( + { + name: '@hashintel/brunch-test', + version: '1.0.0-alpha.9', + repository: { + type: 'git', + url: 'git+https://github.com/hashintel/brunch.git', + }, + }, + null, + 2, + )}\n`, + ); + await git(root, ['init', '--initial-branch=main']); + await git(root, ['config', 'user.name', 'Comparison Test']); + await git(root, ['config', 'user.email', 'comparison@example.test']); + await git(root, ['add', 'package.json']); + await git(root, ['commit', '-m', 'fixture']); + if (input.tag !== undefined) await git(root, ['tag', input.tag]); + return { + root, + commitSha: await git(root, ['rev-parse', 'HEAD']), + }; +} + +async function git(cwd: string, args: readonly string[]): Promise { + const result = await execFileAsync('git', args, { cwd, encoding: 'utf8' }); + return result.stdout.trim(); +} diff --git a/src/dev/comparison-provenance.ts b/src/dev/comparison-provenance.ts new file mode 100644 index 000000000..9ec50d56d --- /dev/null +++ b/src/dev/comparison-provenance.ts @@ -0,0 +1,346 @@ +import { execFile } from 'node:child_process'; +import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export const COMPARISON_PROVENANCE_SCHEMA_VERSION = 1 as const; +export const COMPARISON_PROVENANCE_FILENAME = 'provenance.json'; + +export type ComparisonKind = 'elicitation' | 'execution' | 'end_to_end'; + +export interface ComparisonProvenance { + readonly schemaVersion: typeof COMPARISON_PROVENANCE_SCHEMA_VERSION; + readonly comparisonKind: ComparisonKind; + readonly runId: string; + readonly capturedAt: string; + readonly rootPackage: { + readonly name: string; + readonly version: string; + }; + readonly exactTag: string | null; + readonly controller: { + readonly commitSha: string; + readonly commitUrl: string; + readonly branch: string | null; + readonly dirty: boolean; + }; +} + +interface RootPackageJson { + readonly name?: unknown; + readonly version?: unknown; + readonly repository?: unknown; +} + +const DEFAULT_CONTROLLER_ROOT = fileURLToPath(new URL('../../', import.meta.url)); +const RUN_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; +const COMMIT_SHA_PATTERN = /^[a-f0-9]{40}$/u; + +export async function captureComparisonProvenance(input: { + readonly runDirectory: string; + readonly comparisonKind: ComparisonKind; + readonly runId: string; + readonly controllerRoot?: string; + readonly capturedAt?: string; +}): Promise<{ readonly provenancePath: string; readonly provenance: ComparisonProvenance }> { + const comparisonKind = parseComparisonKind(input.comparisonKind); + const runId = parseRunId(input.runId); + const capturedAt = parseCapturedAt(input.capturedAt ?? new Date().toISOString()); + const controllerRoot = await resolveControllerRoot(input.controllerRoot ?? DEFAULT_CONTROLLER_ROOT); + const rootPackage = await loadRootPackage(controllerRoot); + const commitSha = await gitOutput(controllerRoot, ['rev-parse', 'HEAD']); + if (!COMMIT_SHA_PATTERN.test(commitSha)) { + throw new Error(`controller commit must be a full 40-character SHA, received: ${commitSha}`); + } + + const provenance: ComparisonProvenance = { + schemaVersion: COMPARISON_PROVENANCE_SCHEMA_VERSION, + comparisonKind, + runId, + capturedAt, + rootPackage: { + name: rootPackage.name, + version: rootPackage.version, + }, + exactTag: await gitOptionalOutput(controllerRoot, ['describe', '--tags', '--exact-match', 'HEAD']), + controller: { + commitSha, + commitUrl: `${rootPackage.repositoryUrl}/commit/${commitSha}`, + branch: await gitOptionalOutput(controllerRoot, ['symbolic-ref', '--quiet', '--short', 'HEAD']), + dirty: (await gitOutput(controllerRoot, ['status', '--porcelain=v1', '--untracked-files=all'])) !== '', + }, + }; + + const runDirectory = resolve(input.runDirectory); + await mkdir(runDirectory, { recursive: true }); + const provenancePath = join(runDirectory, COMPARISON_PROVENANCE_FILENAME); + try { + await writeFile(provenancePath, `${JSON.stringify(provenance, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + }); + } catch (error) { + if (isNodeError(error) && error.code === 'EEXIST') { + throw new Error(`comparison provenance already exists: ${provenancePath}`); + } + throw error; + } + + return { provenancePath, provenance }; +} + +export function parseComparisonProvenance(value: unknown): ComparisonProvenance { + const record = requiredRecord(value, 'comparison provenance'); + const rootPackage = requiredRecord(record.rootPackage, 'comparison provenance rootPackage'); + const controller = requiredRecord(record.controller, 'comparison provenance controller'); + const parsed: ComparisonProvenance = { + schemaVersion: requiredLiteral( + record.schemaVersion, + COMPARISON_PROVENANCE_SCHEMA_VERSION, + 'comparison provenance schemaVersion', + ), + comparisonKind: parseComparisonKind(record.comparisonKind), + runId: parseRunId(record.runId), + capturedAt: parseCapturedAt(record.capturedAt), + rootPackage: { + name: requiredNonEmptyString(rootPackage.name, 'comparison provenance rootPackage.name'), + version: requiredNonEmptyString(rootPackage.version, 'comparison provenance rootPackage.version'), + }, + exactTag: + record.exactTag === null + ? null + : requiredNonEmptyString(record.exactTag, 'comparison provenance exactTag'), + controller: { + commitSha: requiredPattern( + controller.commitSha, + COMMIT_SHA_PATTERN, + 'comparison provenance controller.commitSha', + ), + commitUrl: requiredHttpUrl(controller.commitUrl, 'comparison provenance controller.commitUrl'), + branch: + controller.branch === null + ? null + : requiredNonEmptyString(controller.branch, 'comparison provenance controller.branch'), + dirty: requiredBoolean(controller.dirty, 'comparison provenance controller.dirty'), + }, + }; + const expectedCommitUrlSuffix = `/commit/${parsed.controller.commitSha}`; + if (!parsed.controller.commitUrl.endsWith(expectedCommitUrlSuffix)) { + throw new Error('comparison provenance controller.commitUrl must identify controller.commitSha'); + } + return parsed; +} + +export async function runComparisonProvenanceCli(args: readonly string[]): Promise { + const [command, ...rest] = args; + if (command !== 'capture') { + throw new Error( + 'Usage: comparison-provenance capture --run-directory --comparison-kind --run-id [--controller-root ]', + ); + } + const options = parseOptions(rest); + assertOnlyOptions(options, ['run-directory', 'comparison-kind', 'run-id', 'controller-root']); + const comparisonKind = parseComparisonKind(required(options, 'comparison-kind')); + const controllerRoot = options.get('controller-root'); + const result = await captureComparisonProvenance({ + runDirectory: required(options, 'run-directory'), + comparisonKind, + runId: required(options, 'run-id'), + ...(controllerRoot === undefined ? {} : { controllerRoot }), + }); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); +} + +async function resolveControllerRoot(path: string): Promise { + const root = await realpath(resolve(path)); + const gitRoot = await realpath(await gitOutput(root, ['rev-parse', '--show-toplevel'])); + if (root !== gitRoot) { + throw new Error(`controller root must be the Git worktree root: ${root}`); + } + return root; +} + +async function loadRootPackage( + controllerRoot: string, +): Promise<{ readonly name: string; readonly version: string; readonly repositoryUrl: string }> { + let value: unknown; + try { + value = JSON.parse(await readFile(join(controllerRoot, 'package.json'), 'utf8')) as unknown; + } catch (error) { + throw new Error( + `controller root package.json is unreadable or malformed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const packageJson = requiredRecord(value, 'controller root package.json') as RootPackageJson; + return { + name: requiredNonEmptyString(packageJson.name, 'controller root package name'), + version: requiredNonEmptyString(packageJson.version, 'controller root package version'), + repositoryUrl: normalizeRepositoryUrl(packageJson.repository), + }; +} + +function normalizeRepositoryUrl(repository: unknown): string { + const raw = + typeof repository === 'string' + ? repository + : isRecord(repository) && typeof repository.url === 'string' + ? repository.url + : undefined; + if (raw === undefined || raw.trim() === '') { + throw new Error('controller root package repository URL is required'); + } + const trimmed = raw.trim().replace(/^git\+/u, ''); + const sshMatch = /^git@([^:]+):(.+)$/u.exec(trimmed); + const normalized = sshMatch + ? `https://${sshMatch[1]}/${sshMatch[2]}` + : trimmed.replace(/^ssh:\/\/git@/u, 'https://'); + const withoutGitSuffix = normalized.replace(/\.git$/u, '').replace(/\/$/u, ''); + return requiredHttpUrl(withoutGitSuffix, 'controller root package repository URL'); +} + +async function gitOutput(cwd: string, args: readonly string[]): Promise { + try { + const result = await execFileAsync('git', args, { + cwd, + encoding: 'utf8', + maxBuffer: 1024 * 1024, + }); + return result.stdout.trim(); + } catch (error) { + throw new Error( + `git ${args.join(' ')} failed: ${isExecError(error) ? error.stderr.trim() || error.message : String(error)}`, + ); + } +} + +async function gitOptionalOutput(cwd: string, args: readonly string[]): Promise { + try { + return await gitOutput(cwd, args); + } catch { + return null; + } +} + +function parseComparisonKind(value: unknown): ComparisonKind { + if (value !== 'elicitation' && value !== 'execution' && value !== 'end_to_end') { + throw new Error('comparison kind must be elicitation, execution, or end_to_end'); + } + return value; +} + +function parseRunId(value: unknown): string { + if (typeof value !== 'string' || !RUN_ID_PATTERN.test(value)) { + throw new Error( + 'comparison run id must start with an alphanumeric character and contain only alphanumerics, dot, underscore, or hyphen (maximum 128 characters)', + ); + } + return value; +} + +function parseCapturedAt(value: unknown): string { + const capturedAt = requiredNonEmptyString(value, 'comparison provenance capturedAt'); + const date = new Date(capturedAt); + if (!Number.isFinite(date.valueOf()) || date.toISOString() !== capturedAt) { + throw new Error('comparison provenance capturedAt must be a canonical ISO timestamp'); + } + return capturedAt; +} + +function parseOptions(args: readonly string[]): Map { + const options = new Map(); + for (let index = 0; index < args.length; index += 2) { + const name = args[index]; + const value = args[index + 1]; + if (!name?.startsWith('--') || value === undefined || value.startsWith('--')) { + throw new Error(`invalid comparison provenance option near ${name ?? '(end)'}`); + } + const key = name.slice(2); + if (options.has(key)) throw new Error(`duplicate comparison provenance option: --${key}`); + options.set(key, value); + } + return options; +} + +function assertOnlyOptions(options: ReadonlyMap, allowed: readonly string[]): void { + for (const name of options.keys()) { + if (!allowed.includes(name)) throw new Error(`unknown comparison provenance option: --${name}`); + } +} + +function required(options: ReadonlyMap, name: string): string { + const value = options.get(name); + if (value === undefined || value.length === 0) throw new Error(`missing required option --${name}`); + return value; +} + +function requiredRecord(value: unknown, label: string): Record { + if (!isRecord(value)) throw new Error(`${label} must be an object`); + return value; +} + +function requiredNonEmptyString(value: unknown, label: string): string { + if (typeof value !== 'string' || value.trim() === '') + throw new Error(`${label} must be a non-empty string`); + return value; +} + +function requiredPattern(value: unknown, pattern: RegExp, label: string): string { + const text = requiredNonEmptyString(value, label); + if (!pattern.test(text)) throw new Error(`${label} has an invalid format`); + return text; +} + +function requiredHttpUrl(value: unknown, label: string): string { + const text = requiredNonEmptyString(value, label); + let url: URL; + try { + url = new URL(text); + } catch { + throw new Error(`${label} must be an absolute URL`); + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') { + throw new Error(`${label} must use http or https`); + } + return url.toString().replace(/\/$/u, ''); +} + +function requiredBoolean(value: unknown, label: string): boolean { + if (typeof value !== 'boolean') throw new Error(`${label} must be a boolean`); + return value; +} + +function requiredLiteral(value: unknown, expected: Value, label: string): Value { + if (value !== expected) throw new Error(`${label} must be ${expected}`); + return expected; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error; +} + +function isExecError( + error: unknown, +): error is Error & { readonly stderr: string; readonly stdout: string; readonly code: number } { + return ( + error instanceof Error && + 'stderr' in error && + typeof error.stderr === 'string' && + 'stdout' in error && + typeof error.stdout === 'string' + ); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + runComparisonProvenanceCli(process.argv.slice(2)).catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +}