From f1ec7e0212ee0082c3a8e93e95185a08077ecc15 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 24 Jul 2026 14:37:53 +0200 Subject: [PATCH] FE-1266: Skip comparison oracles for non-runtime PRs Co-authored-by: Cursor --- .changeset/afraid-weeks-knock.md | 2 + .github/workflows/test.yml | 16 ++- AGENTS.md | 10 +- CONTRIBUTING.md | 6 +- memory/SPEC.md | 11 +- .../tooling--conditional-comparison-gate.md | 121 ++++++++++++++++++ package.json | 4 +- scripts/ci-test-lanes.mjs | 94 ++++++++++++++ scripts/ci-test-lanes.test.mjs | 114 +++++++++++++++++ scripts/comparison-test-inventory.test.mjs | 84 ++++++++++++ scripts/test-workflow.contract.test.mjs | 32 +++++ ...ct-research-workspace-oracle.slow.test.ts} | 0 12 files changed, 481 insertions(+), 13 deletions(-) create mode 100644 .changeset/afraid-weeks-knock.md create mode 100644 memory/cards/tooling--conditional-comparison-gate.md create mode 100644 scripts/ci-test-lanes.mjs create mode 100644 scripts/ci-test-lanes.test.mjs create mode 100644 scripts/comparison-test-inventory.test.mjs create mode 100644 scripts/test-workflow.contract.test.mjs rename src/dev/execution-comparison/__tests__/{prospect-research-workspace-oracle.test.ts => prospect-research-workspace-oracle.slow.test.ts} (100%) diff --git a/.changeset/afraid-weeks-knock.md b/.changeset/afraid-weeks-knock.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/afraid-weeks-knock.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f85e81e26..e34fbb7e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,14 @@ jobs: node-version: 24 cache: npm + - name: Select test lanes + id: test-lanes + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.sha }} + run: node scripts/ci-test-lanes.mjs + - name: Configure git identity # The current slow tests in test:full create real repositories and commits. run: | @@ -55,8 +63,12 @@ jobs: - name: Check (read-only lint + format + skills) run: npm run check - - name: Test (full suite incl. slow tests) - run: npm run test:full + - name: Test (default + non-comparison slow) + run: npm run test && npm run test:slow:core + + - name: Test (expensive comparison oracles) + if: steps.test-lanes.outputs.comparison == 'true' + run: npm run test:comparison - name: Build run: npm run build diff --git a/AGENTS.md b/AGENTS.md index 7917faa88..366a3e5e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,13 +94,13 @@ Frontier-item traceability, scope-card inheritance, and the verification-ownersh ## verification -**Defaults are fast; the real full gate is CI.** `npm run test` and `npm run verify` deliberately exclude `*.slow.test.ts` so the everyday loop stays quick. A test earns the `.slow` marker only when its execution cost warrants exclusion from routine local verification — never merely because it is inconvenient or flaky. The `.github/workflows/test.yml` **Test** workflow runs the *full* suite (`test:full`, including slow tests) plus `check` and `build` on every PR and merge-queue entry — that is the authoritative gate, not the local pre-commit run. Run the `:full` variant locally only when you have a reason to (below). +**Defaults are fast; the real full gate is CI.** `npm run test` and `npm run verify` deliberately exclude `*.slow.test.ts` so the everyday loop stays quick. A test earns the `.slow` marker only when its execution cost warrants exclusion from routine local verification — never merely because it is inconvenient or flaky. The `.github/workflows/test.yml` **Test** workflow always runs `check`, `build`, default tests, and non-comparison slow tests under one stable `Full gate` check. Expensive comparison oracles may be omitted only for a complete pull-request diff wholly inside the closed non-runtime allowlist; unknown evidence runs them, and merge-queue entries always run the full suite. CI remains the authoritative gate, not the local pre-commit run. **Inner loop** (run after every meaningful edit): `npm run fix` — lint:fix then format. Run focused tests with `npm test -- ` while iterating. **Checkpoint / pre-commit** (fast, the default): `npm run verify` — fix → default tests → build. The default suite excludes `*.slow.test.ts`. This is the routine local gate. -**Full gate** (locally optional; CI always runs it): `npm run verify:full` — fix → all tests (incl. slow tests) → build. Run it locally when you have changed a slow test or the production seam it witnesses — currently host landing, slice integration, run promotion, or worktree behavior — otherwise let CI run it. `npm run test:slow` runs just the slow group when that is all you need. +**Full gate** (locally optional; merge-queue CI always runs it): `npm run verify:full` — fix → all tests (incl. slow tests) → build. Run it locally when you have changed a slow test or the production seam it witnesses — currently host landing, slice integration, run promotion, worktree behavior, or comparison-controller behavior — otherwise let CI run it. `npm run test:slow` runs the core-slow and comparison lanes; use `test:slow:core` or `test:comparison` for one lane. **PR release intent** (required for ordinary PRs into `next`): before submit or update, run `npx changeset status --since=origin/next`. If the published package changes, run `npm run changeset`; otherwise record the no-release decision with `npm run changeset -- --empty`. Commit the generated `.changeset/*.md` file. The local verify commands do not cover this base-aware CI check. @@ -115,9 +115,11 @@ Frontier-item traceability, scope-card inheritance, and the verification-ownersh | `npm run fix` | lint:fix → fmt | yes | | `npm run test` | all Vitest tests except `*.slow.test.ts` | no | | `npm run test:full` | all Vitest tests incl. slow tests | no | -| `npm run test:slow` | just `*.slow.test.ts` | no | +| `npm run test:slow` | core-slow → comparison | no | +| `npm run test:slow:core` | slow tests except expensive comparison oracles | no | +| `npm run test:comparison` | expensive full-stack comparison oracles | no | | `npm run verify` | fix → test → build (fast default) | yes (via fix) | -| `npm run verify:full` | fix → test:full → build (full gate; CI runs this) | yes (via fix) | +| `npm run verify:full` | fix → test:full → build (full local/merge-queue gate) | yes (via fix) | | `npm run check` | lint → fmt:check → check:markdown-links → check:skills → check:promoted-run-paths | no | | `npm run check:markdown-links` | remark-validate-links over Markdown files | no | | `npm run check:skills` | ln-* skill consistency | no | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8cd7f42a1..9757c3283 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,12 +66,14 @@ Brunch stores local runtime state under the target workspace's `.brunch/` direct | `npm run dev` | Run the Brunch CLI directly from source. Defaults to `--mode tui`. | | `npm run dev-cli` | Select/create a temporary, named, existing, or seed-derived dev instance. | | `npm run test` | Run Vitest once (fast default; excludes `*.slow.test.ts`). | -| `npm run test:slow` | Run only `*.slow.test.ts`. | +| `npm run test:slow` | Run core-slow tests followed by expensive comparison oracles. | +| `npm run test:slow:core` | Run slow tests except the expensive comparison oracles. | +| `npm run test:comparison` | Run the expensive full-stack comparison oracles. | | `npm run test:full` | Run every Vitest test, including slow tests. | | `npm run fix` | Apply lint fixes, then format. | | `npm run check` | Read-only lint + format check. | | `npm run verify` | Fast local checkpoint: fix → test → build. Routine pre-commit run. | -| `npm run verify:full` | Full gate: fix → test:full → build. CI runs this on every PR; run it locally only when you touch host landing, slice integration, run promotion, or worktree behavior. | +| `npm run verify:full` | Full local gate: fix → test:full → build. Merge-queue CI always runs it; pull requests may omit only expensive comparison oracles for a closed non-runtime-only diff. | | `npm run build` | Build TypeScript, packaged Pi assets, and the web bundle. | | `npm run seed -- --workspace --seed /` | Seed a workspace from `.fixtures/seeds`. | | `npm run db:generate` | Generate Drizzle migrations. | diff --git a/memory/SPEC.md b/memory/SPEC.md index 56960dfa6..97cfe7fe9 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -318,6 +318,7 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D100-L | `project` is a distinct first-level live Specify-mode skill home for cross-plane derivation, not a `generate` sub-mode. `generate` fans out alternatives within a target plane from context; `project` starts from accepted upstream graph anchors and derives downstream plane candidates/drafts plus connecting edge intent. It uses the existing structured-exchange offer/terminal seams (`present_candidates`, `present_review_set`, and their declared `ask` continuations per D116-L) and hands exact graph expression back to `map` / review-set commitment; it adds no product tool, exchange schema family, or direct graph-write path. Depends on: D95-L, D96-L, D97-L, I51-L. | [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md), [`src/agents/subagents/TOPOLOGY.md`](../src/agents/subagents/TOPOLOGY.md) | active | | D139-L | FE-1253 admits the prospect research workspace as a deterministic greenfield execution regression case, not a second end-to-end campaign profile. Its public contract fixes one npm repository with a React/TypeScript frontend, Node.js/TypeScript backend, SQLite persistence, `npm test` / `npm run build` / `npm start`, environment-addressed port/database/fixture paths, a health endpoint, accessible mechanical controls, package-registry-only install, and denied runtime network. Pi-compatible qualification and Clay-compatible research are server-side interfaces exercised through controller-owned deterministic local fixtures; live provider quality is not scored. The saved mission remains available for exploratory specification work, while a future crossed campaign requires a separate explicit study contract. Supersedes FE-1253's unexecuted 2×2 campaign expansion. Depends on: D70-L, FE-1230, FE-1241. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `prospect-research-workspace-regression`; `testing/comparisons/missions/prospect-research-workspace.md`; `testing/execution-comparisons/cases/prospect-research-workspace/` | active — deterministic full-stack oracle calibrated; campaign expansion retired | +| D1-K | CI verification is event- and evidence-tiered: local `test:full` and every merge-group candidate run every retained test, while a pull-request event may omit only the expensive comparison-oracle lane and only when a closed non-runtime path allowlist proves the complete diff cannot affect shipped runtime, build/test infrastructure, or comparison-controller behavior. Missing, incomplete, unknown, or non-allowlisted diff evidence fails open to the comparison lane. Static checks, build, default tests, and non-comparison slow tests remain mandatory, and one stable `Full gate` status remains the branch-protection surface. No oracle, known-good fixture, or contrastive rival is deleted by this scheduling decision. Depends on: D68-L, D136-L, D139-L. | [`memory/cards/tooling--conditional-comparison-gate.md`](cards/tooling--conditional-comparison-gate.md); `.github/workflows/test.yml` | active — materialized 2026-07-24; outer CI witness pending | ### Critical Invariants @@ -714,9 +715,11 @@ The verification harness is established (oxlint + oxfmt + vitest). Commands foll | 1 | Lint:fix + format (inner loop, writes) | `npm run fix` | | 2 | Lint + format check (no writes; CI use) | `npm run check` | | 3 | Default tests (excludes `*.slow.test.ts`) | `npm run test` | -| 4 | Slow tests | `npm run test:slow` | -| 5 | All tests (incl. slow tests) | `npm run test:full` | -| 6 | Build | `npm run build` | +| 4 | Non-comparison slow tests | `npm run test:slow:core` | +| 5 | Expensive comparison oracles | `npm run test:comparison` | +| 6 | All slow tests | `npm run test:slow` | +| 7 | All tests (incl. slow tests) | `npm run test:full` | +| 8 | Build | `npm run build` | | fast | Local checkpoint / pre-commit (writes via `fix`) | `npm run verify` (= fix + test + build) | | full | Full gate (writes via `fix`; CI runs this) | `npm run verify:full` (= fix + all + build) | | — | Release-pack smoke (pre-publish only) | `npm run check:release-pack` | @@ -729,7 +732,7 @@ The release-pack smoke is deliberately outside `verify` (it packs, asserts runti - **Inner loop:** run `npm run fix` after every meaningful edit and focused tests with `npm test -- ` while iterating. Tooling: oxlint (lint + type-aware + type-check via tsgolint), oxfmt (format), vitest (test). See AGENTS.md. - **Local checkpoint (fast default):** `npm run verify` runs every test except `*.slow.test.ts`, then builds. A test earns the `.slow` marker only when its execution cost warrants exclusion from routine local verification — never merely because it is inconvenient or flaky. The checkpoint shortens development feedback without weakening the authoritative gate, which is CI. -- **Full gate:** the `.github/workflows/test.yml` **Test** workflow runs `npm run test:full` (all tests, incl. slow tests) plus `check` and `build` on every PR and merge-queue entry — this is the authoritative gate. Run `npm run verify:full` locally when you have changed a slow test or the production seam it witnesses (currently host landing, slice integration, run promotion, or worktree behavior); otherwise let CI run it. `npm run test:slow` runs just the slow group. Remaining failures must be fixed before proceeding. No override. +- **Full gate authority:** `npm run test:full` continues to mean every retained test and runs unconditionally for merge-group candidates. Under D1-K, pull-request events keep `check`, `build`, default tests, and non-comparison slow tests mandatory; they may omit only the expensive comparison-oracle lane and only for a complete diff wholly inside a closed non-runtime allowlist. Unknown or incomplete evidence runs comparison tests. One stable `Full gate` check remains authoritative, and no failure may be overridden. Run `npm run verify:full` locally when you have changed a slow test or the production seam it witnesses; otherwise let CI own the full pre-merge run. - **Failure protocol:** stop on first failure; the failure becomes the must-fix task; re-run the stack from step 1; only proceed when all checks pass. - **Frontier completion:** manual smoke can prove presentation life, but any durable product claim must also have an artifact/query oracle, property/round-trip test, contract test, or fixture assertion tied to the canonical store or projection handler that owns the fact. - **Harness/probe JSONL architecture:** the POC uses Tier-1/Tier-2 faux harnesses plus JSONL-backed probe runs as the current verification artifact model. Committed probe evidence lives under `.fixtures/runs///` with colocated `session.jsonl` and `report.json`; human-readable transcript rendering is a workspace-local `.brunch/debug/transcript.md` affordance for faux-harness/debug runs, not a default committed probe artifact. Brief-based golden fixtures are deferred; if they return, briefs are harness/probe inputs and the resulting JSONL-backed run is canonical. The debug transcript renderer uses Pi's canonical context construction, then keeps only user messages, assistant messages, and Brunch-owned custom tool results. diff --git a/memory/cards/tooling--conditional-comparison-gate.md b/memory/cards/tooling--conditional-comparison-gate.md new file mode 100644 index 000000000..d50902736 --- /dev/null +++ b/memory/cards/tooling--conditional-comparison-gate.md @@ -0,0 +1,121 @@ +# Route Expensive Comparison Oracles Conservatively + +Frontier: n/a +Status: active +Mode: single +Created: 2026-07-24 + +Posture: proving (category concern; no containing PLAN frontier). + +## Orientation + +- The containing seam is the repository verification harness: package test lanes feed the single required GitHub Actions `Full gate` status. +- This is dev-tooling rework, not a slice of an active product frontier; completed comparison frontiers remain the owners of the oracle behavior being scheduled. +- There is no volatile `HANDOFF.md` state. Existing uncommitted scope files are independent; none declares the workflow/package/script paths below as primary writes. +- The main risk is a false-negative path classification that skips an oracle affected through a transitive dependency; omission must therefore be fail-open and narrower than ordinary change-impact selection. + +## Build state + +- Local implementation and deterministic contracts are complete. +- Outer evidence remains owned by this card: re-enter when a pull request whose complete base-to-head diff is wholly inside the closed non-runtime allowlist runs the `Full gate`; capture that the comparison step is skipped and the stable job succeeds. The current runtime-changing PR can prove only the full-lane control. + +## Target Behavior + +The CI test gate omits expensive comparison oracles only when event and changed-path evidence prove they cannot affect the candidate result. + +## Full-card cold-start reads + +- `memory/SPEC.md` — D1-K and Verification Design, especially Verification Commands and Verification Policy +- `memory/PLAN.md` — Comparison lane context and completed `brownfield-comparison-cases` / prospect regression work +- `AGENTS.md` — verification commands, slow-test eligibility, and CI authority +- `src/dev/TOPOLOGY.md` — Brownfield Comparison Oracles and Prospect Research Regression Case +- `docs/praxis/comparison-runs.md` — unchanged controller-oracle discipline and retained comparison evidence +- `.github/workflows/test.yml` and `package.json` — current single-job sequencing and test-lane commands +- GitHub Actions PR #377 runs `30086902794`, `30087817994`, and `30088392996` — observed catch, rerun cost, and comparison-oracle timings + +## Boundary Crossings + +```text +GitHub pull-request or merge-group event +→ complete base/head changed-path evidence +→ fail-open test-lane selector +→ always-required static, build, default-test, and non-comparison slow-test gates +→ conditionally-required expensive comparison-oracle lane +→ stable Full gate check result +``` + +## Risks and Assumptions + +- RISK: a shared runtime, harness, dependency, lockfile, workflow, or test configuration change is mistaken for documentation-only work → MITIGATION: omission uses a closed non-runtime allowlist; every unknown, missing, renamed-across-boundary, or non-allowlisted path selects the comparison lane. +- RISK: `pull_request` and `merge_group` expose different refs or incomplete history → MITIGATION: keep full checkout history, test event/ref resolution as a pure function, and select comparison on any diff acquisition failure. +- RISK: splitting the command inventory silently drops or double-runs tests → MITIGATION: add an executable inventory contract and keep `test:full` as the local composition that runs every lane exactly once. +- RISK: changing workflow step conditions weakens the required branch-protection status → MITIGATION: retain one stable required `Full gate` job; skipped comparison work is an internal lane decision, never a skipped required job. +- RISK: the 160.6-second prospect oracle remains in the four-worker default lane and bypasses conditional routing → MITIGATION: reclassify it as an expensive comparison oracle in the same cutover. +- ASSUMPTION: a closed allowlist can identify changes that cannot alter shipped runtime, build output, test infrastructure, or comparison-controller behavior. + → IMPACT IF FALSE: an affected comparison regression could merge without its owning oracle running on the pull-request head. + → VALIDATE: table-driven path-classification rivals, unconditional merge-group selection, and a full-suite inventory contract. + → D1-K adopts the fail-open closed-allowlist policy. + +## Posture check + +- Lights up: one deterministic path from GitHub event and complete diff evidence to an auditable comparison-lane decision inside the existing required check. +- Stabilizes: CI remains fail-open, merge-group entries remain fully gated, and local `test:full` continues to mean every test. +- Retires: whether an unrelated non-runtime pull-request update can avoid the comparison-oracle cost without weakening the authoritative pre-merge gate. + +## Acceptance Criteria + +✓ `scripts/ci-test-lanes.test.mjs` — every `merge_group` input selects the expensive comparison lane regardless of paths. + +✓ `scripts/ci-test-lanes.test.mjs` — a pull-request diff containing only closed allowlisted non-runtime paths omits the comparison lane. + +✓ `scripts/ci-test-lanes.test.mjs` — runtime, comparison, test, fixture, package, lockfile, workflow, build-configuration, unknown, incomplete-diff, and allowlist-boundary rename rivals all select the comparison lane. + +✓ `scripts/comparison-test-inventory.test.mjs` — every current full-stack comparison oracle belongs to the comparison lane exactly once, including the prospect workspace oracle, while non-comparison slow tests remain in the mandatory PR lane. + +✓ `scripts/test-workflow.contract.test.mjs` — `.github/workflows/test.yml` obtains complete diff evidence, fails open to comparison, always runs static/build/default/non-comparison-slow gates, conditions only the expensive comparison lane for pull requests, and selects it unconditionally for merge groups. + +✓ `npm run test:comparison` — the extracted comparison-oracle lane runs all retained known-good and contrastive oracle coverage successfully. + +✓ `npm run test:full` — default, non-comparison slow, and comparison lanes compose to run the complete suite successfully with no excluded or duplicate test file. + +✓ GitHub Actions workflow run — a controlled non-runtime-only pull request keeps the stable `Full gate` check green without executing the comparison command, while a comparison-path control executes it. + +## Invariants preserved + +- No comparison oracle or contrastive rival is deleted or weakened — guarded by `scripts/comparison-test-inventory.test.mjs` and `npm run test:full`. +- The slow slice-integration test that caught PR #377’s foreign-worktree mismatch remains in the mandatory PR gate — guarded by the inventory contract and `npm run test:slow:core`. +- Merge-queue candidates run every test with no path-based omission — guarded by lane-selector and workflow contract tests. +- Diff uncertainty increases verification rather than reducing it — guarded by malformed/missing/rename rival cases in `scripts/ci-test-lanes.test.mjs`. +- `test:full` retains its canonical meaning for local verification and CI controls — guarded by the package-script inventory contract. +- One stable `Full gate` status remains the branch-protection surface — guarded by `scripts/test-workflow.contract.test.mjs`. + +## Verification Approach + +- Inner: table-driven pure selector and explicit test-inventory contract tests prove conservative routing and exact lane membership. +- Middle: workflow contract tests prove event/ref wiring, fail-open behavior, mandatory lane execution, and stable check topology without spending oracle runtime. +- Outer: two controlled GitHub Actions runs—one allowlisted non-runtime diff and one comparison-path control—prove actual skip/run behavior; this card owns that evidence and it must be captured before completion. + +## Cross-cutting obligations + +- Materialize D1-K consistently; do not leave `AGENTS.md` or `CONTRIBUTING.md` claiming every pull-request event runs `test:full`. +- Comparison-controller identity, known-good fixtures, and contrastive rivals remain unchanged; this slice changes scheduling only. +- Merge-group verification remains the unconditional authoritative full gate. +- Path classification is fail-open and inspectable; do not adopt a heuristic dependency graph or positive “relevant paths” list in this slice. +- Keep the comparison-lane extraction separate from fail-fast, sharding, caching, or scenario reduction so its safety and measured impact remain attributable. + +## Expected touched paths (tentative) + +```text +.github/workflows/test.yml ~ +package.json ~ +scripts/ +├── ci-test-lanes.mjs + +├── ci-test-lanes.test.mjs + +├── comparison-test-inventory.test.mjs + +└── test-workflow.contract.test.mjs + +src/dev/execution-comparison/__tests__/ +├── prospect-research-workspace-oracle.test.ts - +└── prospect-research-workspace-oracle.slow.test.ts + +AGENTS.md ~ +CONTRIBUTING.md ~ +``` diff --git a/package.json b/package.json index 40e464d33..95abc0f1d 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,9 @@ "db:studio": "drizzle-kit studio", "test": "vitest --run --maxWorkers=4 --exclude='**/*.slow.test.ts'", "test:full": "npm run test && npm run test:slow", - "test:slow": "vitest --run --maxWorkers=1 .slow.test.ts", + "test:slow": "npm run test:slow:core && npm run test:comparison", + "test:slow:core": "vitest --run --maxWorkers=1 .slow.test.ts --exclude='src/dev/execution-comparison/**/*.slow.test.ts' --exclude='src/dev/end-to-end-comparison/**/*.slow.test.ts'", + "test:comparison": "vitest --run --maxWorkers=1 src/dev/execution-comparison/__tests__/browser-oracle.slow.test.ts src/dev/execution-comparison/__tests__/host-landing-oracle.slow.test.ts src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.slow.test.ts src/dev/end-to-end-comparison/__tests__/factorial-browser-oracle.slow.test.ts", "test:watch": "vitest", "test:context-surfaces": "vitest --run src/agents/contexts src/app/__tests__/print-workspace-state.test.ts src/session/__tests__/transcript-markdown.test.ts", "test:context-surfaces:watch": "vitest src/agents/contexts src/app/__tests__/print-workspace-state.test.ts src/session/__tests__/transcript-markdown.test.ts", diff --git a/scripts/ci-test-lanes.mjs b/scripts/ci-test-lanes.mjs new file mode 100644 index 000000000..ad66a375b --- /dev/null +++ b/scripts/ci-test-lanes.mjs @@ -0,0 +1,94 @@ +import { execFile } from 'node:child_process'; +import { appendFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +const NON_RUNTIME_DIRECTORIES = ['.agents/', '.changeset/', '.claude/', '.codex/', 'docs/', 'memory/']; +const NON_RUNTIME_ROOT_FILES = new Set([ + 'AGENTS.md', + 'CLAUDE.md', + 'CONTRIBUTING.md', + 'README.md', + 'TESTING_FINDINGS.md', + 'TESTING_PLAN.md', +]); + +export function isClosedNonRuntimePath(path) { + return ( + NON_RUNTIME_ROOT_FILES.has(path) || + NON_RUNTIME_DIRECTORIES.some((directory) => path.startsWith(directory)) + ); +} + +export function selectCiTestLanes({ eventName, diffComplete, changedPaths }) { + if (eventName === 'merge_group') { + return { comparison: true, reason: 'merge-group-full-gate' }; + } + if (eventName !== 'pull_request') { + return { comparison: true, reason: 'unknown-event' }; + } + if (!diffComplete || changedPaths.length === 0) { + return { comparison: true, reason: 'incomplete-or-empty-diff' }; + } + if (changedPaths.every(isClosedNonRuntimePath)) { + return { comparison: false, reason: 'closed-non-runtime-diff' }; + } + return { comparison: true, reason: 'runtime-or-unknown-path' }; +} + +export async function changedPathsFromGit({ baseSha, headSha, cwd = process.cwd() }) { + if (!baseSha || !headSha) return { complete: false, paths: [] }; + try { + const { stdout } = await execFileAsync( + 'git', + ['diff', '--name-only', '-z', '--no-renames', `${baseSha}...${headSha}`], + { + cwd, + encoding: 'buffer', + maxBuffer: 4 * 1024 * 1024, + }, + ); + return { + complete: true, + paths: stdout + .toString('utf8') + .split('\0') + .filter((path) => path.length > 0), + }; + } catch { + return { complete: false, paths: [] }; + } +} + +async function writeOutputs(result) { + const lines = [`comparison=${String(result.comparison)}`, `reason=${result.reason}`]; + if (process.env.GITHUB_OUTPUT) { + await appendFile(process.env.GITHUB_OUTPUT, `${lines.join('\n')}\n`, 'utf8'); + } else { + process.stdout.write(`${lines.join('\n')}\n`); + } +} + +async function main() { + const eventName = process.env.EVENT_NAME ?? ''; + const diff = + eventName === 'pull_request' + ? await changedPathsFromGit({ + baseSha: process.env.BASE_SHA ?? '', + headSha: process.env.HEAD_SHA ?? '', + }) + : { complete: true, paths: [] }; + const result = selectCiTestLanes({ + eventName, + diffComplete: diff.complete, + changedPaths: diff.paths, + }); + await writeOutputs(result); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + await main(); +} diff --git a/scripts/ci-test-lanes.test.mjs b/scripts/ci-test-lanes.test.mjs new file mode 100644 index 000000000..efc8ed0e2 --- /dev/null +++ b/scripts/ci-test-lanes.test.mjs @@ -0,0 +1,114 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, mkdir, rename, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { changedPathsFromGit, selectCiTestLanes } from './ci-test-lanes.mjs'; + +const execFileAsync = promisify(execFile); +const temporaryRoots = []; + +afterEach(async () => { + await Promise.all(temporaryRoots.splice(0).map(async (path) => await rm(path, { recursive: true }))); +}); + +describe('CI test lane selection', () => { + it('always selects comparison oracles for merge-group candidates', () => { + expect( + selectCiTestLanes({ + eventName: 'merge_group', + diffComplete: true, + changedPaths: ['memory/PLAN.md'], + }), + ).toMatchObject({ comparison: true, reason: 'merge-group-full-gate' }); + }); + + it('omits comparison oracles for a complete closed non-runtime-only pull-request diff', () => { + expect( + selectCiTestLanes({ + eventName: 'pull_request', + diffComplete: true, + changedPaths: [ + '.agents/skills/linear-pr-cleanup/SKILL.md', + '.changeset/quiet-oracles-rest.md', + 'docs/praxis/comparison-runs.md', + 'memory/PLAN.md', + 'AGENTS.md', + ], + }), + ).toMatchObject({ comparison: false, reason: 'closed-non-runtime-diff' }); + }); + + it.each([ + ['runtime source', ['src/executor/run.ts']], + ['comparison controller', ['src/dev/execution-comparison/operator-cli.ts']], + ['comparison fixture', ['testing/execution-comparisons/cases/example/public-contract.json']], + ['test source', ['src/executor/__tests__/run.test.ts']], + ['package manifest', ['package.json']], + ['package lockfile', ['package-lock.json']], + ['workflow', ['.github/workflows/test.yml']], + ['build configuration', ['tsconfig.build.json']], + ['unknown root', ['new-root-config.json']], + ['rename crossing from runtime into docs', ['src/old.ts', 'docs/old.ts']], + ['rename crossing from docs into runtime', ['docs/new.ts', 'src/new.ts']], + ])('selects comparison oracles for %s changes', (_name, changedPaths) => { + expect( + selectCiTestLanes({ + eventName: 'pull_request', + diffComplete: true, + changedPaths, + }), + ).toMatchObject({ comparison: true, reason: 'runtime-or-unknown-path' }); + }); + + it.each([ + ['incomplete diff', { eventName: 'pull_request', diffComplete: false, changedPaths: ['memory/PLAN.md'] }], + ['empty diff', { eventName: 'pull_request', diffComplete: true, changedPaths: [] }], + [ + 'unknown event', + { eventName: 'workflow_dispatch', diffComplete: true, changedPaths: ['memory/PLAN.md'] }, + ], + ])('fails open for %s', (_name, input) => { + expect(selectCiTestLanes(input)).toMatchObject({ comparison: true }); + }); + + it('treats both sides of an allowlist-boundary rename as changed paths', async () => { + const root = await mkdtemp(join(tmpdir(), 'brunch-ci-lanes-')); + temporaryRoots.push(root); + await execFileAsync('git', ['init', '-q', '-b', 'main'], { cwd: root }); + await execFileAsync('git', ['config', 'user.email', 'ci@brunch.invalid'], { cwd: root }); + await execFileAsync('git', ['config', 'user.name', 'Brunch CI'], { cwd: root }); + await mkdir(join(root, 'docs')); + await writeFile(join(root, 'docs', 'note.md'), 'note\n'); + await execFileAsync('git', ['add', '--all'], { cwd: root }); + await execFileAsync('git', ['commit', '-q', '-m', 'base'], { cwd: root }); + const { stdout: baseSha } = await execFileAsync('git', ['rev-parse', 'HEAD'], { cwd: root }); + + await mkdir(join(root, 'src')); + await rename(join(root, 'docs', 'note.md'), join(root, 'src', 'note.ts')); + await execFileAsync('git', ['add', '--all'], { cwd: root }); + await execFileAsync('git', ['commit', '-q', '-m', 'rename'], { cwd: root }); + const { stdout: headSha } = await execFileAsync('git', ['rev-parse', 'HEAD'], { cwd: root }); + + const diff = await changedPathsFromGit({ + baseSha: baseSha.trim(), + headSha: headSha.trim(), + cwd: root, + }); + + expect(diff).toEqual({ + complete: true, + paths: ['docs/note.md', 'src/note.ts'], + }); + expect( + selectCiTestLanes({ + eventName: 'pull_request', + diffComplete: diff.complete, + changedPaths: diff.paths, + }), + ).toMatchObject({ comparison: true }); + }); +}); diff --git a/scripts/comparison-test-inventory.test.mjs b/scripts/comparison-test-inventory.test.mjs new file mode 100644 index 000000000..df526cffb --- /dev/null +++ b/scripts/comparison-test-inventory.test.mjs @@ -0,0 +1,84 @@ +import { access, readFile, readdir } from 'node:fs/promises'; +import { relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { describe, expect, it } from 'vitest'; + +const root = fileURLToPath(new URL('..', import.meta.url)); +const packageJson = JSON.parse(await readFile(resolve(root, 'package.json'), 'utf8')); + +const comparisonTests = [ + 'src/dev/end-to-end-comparison/__tests__/factorial-browser-oracle.slow.test.ts', + 'src/dev/execution-comparison/__tests__/browser-oracle.slow.test.ts', + 'src/dev/execution-comparison/__tests__/host-landing-oracle.slow.test.ts', + 'src/dev/execution-comparison/__tests__/petrinaut-optimization-oracle.slow.test.ts', + 'src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.slow.test.ts', +]; + +async function collectSlowTests(directory) { + const entries = await readdir(directory, { withFileTypes: true }); + const paths = await Promise.all( + entries.map(async (entry) => { + const path = resolve(directory, entry.name); + if (entry.isDirectory()) return await collectSlowTests(path); + return entry.name.endsWith('.slow.test.ts') ? [relative(root, path)] : []; + }), + ); + return paths.flat(); +} + +describe('comparison test lane inventory', () => { + it('owns every expensive comparison oracle exactly once', async () => { + const discovered = ( + await Promise.all([ + collectSlowTests(resolve(root, 'src/dev/execution-comparison')), + collectSlowTests(resolve(root, 'src/dev/end-to-end-comparison')), + ]) + ) + .flat() + .sort((left, right) => left.localeCompare(right)); + + expect(discovered).toEqual(comparisonTests); + for (const path of comparisonTests) { + expect(packageJson.scripts['test:comparison'].split(path)).toHaveLength(2); + } + }); + + it('routes the prospect full-stack oracle out of the default lane', async () => { + await expect( + access( + resolve( + root, + 'src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.slow.test.ts', + ), + ), + ).resolves.toBeUndefined(); + await expect( + access( + resolve(root, 'src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.test.ts'), + ), + ).rejects.toThrow(); + expect(packageJson.scripts.test).toContain('**/*.slow.test.ts'); + }); + + it('keeps non-comparison slow tests in the mandatory core lane', async () => { + const allSlowTests = (await collectSlowTests(resolve(root, 'src'))).sort((left, right) => + left.localeCompare(right), + ); + const coreSlowTests = allSlowTests.filter((path) => !comparisonTests.includes(path)); + + expect(coreSlowTests).toContain('src/app/__tests__/git-slice-integration-port.slow.test.ts'); + expect(packageJson.scripts['test:slow:core']).toContain('.slow.test.ts'); + expect(packageJson.scripts['test:slow:core']).toContain( + "--exclude='src/dev/execution-comparison/**/*.slow.test.ts'", + ); + expect(packageJson.scripts['test:slow:core']).toContain( + "--exclude='src/dev/end-to-end-comparison/**/*.slow.test.ts'", + ); + }); + + it('composes the complete suite from default, core-slow, and comparison lanes', () => { + expect(packageJson.scripts['test:slow']).toBe('npm run test:slow:core && npm run test:comparison'); + expect(packageJson.scripts['test:full']).toBe('npm run test && npm run test:slow'); + }); +}); diff --git a/scripts/test-workflow.contract.test.mjs b/scripts/test-workflow.contract.test.mjs new file mode 100644 index 000000000..be5ad7c31 --- /dev/null +++ b/scripts/test-workflow.contract.test.mjs @@ -0,0 +1,32 @@ +import { readFile } from 'node:fs/promises'; + +import { describe, expect, it } from 'vitest'; + +const workflow = await readFile(new URL('../.github/workflows/test.yml', import.meta.url), 'utf8'); + +describe('Test workflow lane contracts', () => { + it('derives lane selection from complete pull-request diff evidence', () => { + expect(workflow).toContain('id: test-lanes'); + expect(workflow).toContain('EVENT_NAME: ${{ github.event_name }}'); + expect(workflow).toContain('BASE_SHA: ${{ github.event.pull_request.base.sha }}'); + expect(workflow).toContain('HEAD_SHA: ${{ github.sha }}'); + expect(workflow).toContain('node scripts/ci-test-lanes.mjs'); + }); + + it('always runs the default and non-comparison slow tests', () => { + expect(workflow).toContain('name: Test (default + non-comparison slow)'); + expect(workflow).toContain('npm run test && npm run test:slow:core'); + }); + + it('conditions only the expensive comparison oracle lane', () => { + expect(workflow).toMatch( + /name: Test \(expensive comparison oracles\)\n\s+if: steps\.test-lanes\.outputs\.comparison == 'true'\n\s+run: npm run test:comparison/, + ); + expect(workflow).toMatch(/name: Build\n\s+run: npm run build/); + }); + + it('retains one stable required full-gate job', () => { + expect(workflow.match(/name: Full gate/g)).toHaveLength(1); + expect(workflow).toContain('merge_group:'); + }); +}); diff --git a/src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.test.ts b/src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.slow.test.ts similarity index 100% rename from src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.test.ts rename to src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.slow.test.ts