From 71fef0e4956ff084036d63a64ae6d44cac33788e Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 23:27:23 +0200 Subject: [PATCH 1/3] FE-1253: Add prospect research end-to-end comparison case --- .changeset/tricky-poems-do.md | 2 + docs/praxis/comparison-runs.md | 8 + memory/PLAN.md | 28 +- memory/SPEC.md | 7 + src/dev/TOPOLOGY.md | 8 +- .../__tests__/case-profile.test.ts | 39 +- .../__tests__/study-contract.test.ts | 21 + .../end-to-end-comparison/study-contract.ts | 4 + src/dev/execution-comparison-operator.ts | 64 ++- .../__tests__/brunch-lane.test.ts | 17 + .../__tests__/case-contract.test.ts | 55 ++- .../__tests__/operator-cli.test.ts | 8 + .../operator-oracle-dispatch.test.ts | 11 +- .../__tests__/oracle-pack.test.ts | 77 +++- ...prospect-research-workspace-oracle.test.ts | 160 +++++++ src/dev/execution-comparison/brunch-lane.ts | 118 ++++- src/dev/execution-comparison/case-contract.ts | 194 +++++++++ .../historical-replay-target.ts | 4 +- src/dev/execution-comparison/oracle-pack.ts | 92 ++++ .../prospect-research-workspace-oracle.ts | 5 + .../journey-runner.ts | 65 +++ .../journeys.ts | 379 ++++++++++++++++ .../lifecycle.ts | 241 +++++++++++ .../reference.ts | 60 +++ .../runner.ts | 107 +++++ .../sqlite-evidence.ts | 36 ++ .../types.ts | 45 ++ testing/comparisons/missions/README.md | 2 + .../missions/prospect-research-workspace.md | 48 +++ .../controller/requirement-registry.json | 117 +++++ .../shared-baseline.md | 34 ++ .../study-contract.json | 48 +++ .../controller/fixtures/provider-failure.json | 6 + .../controller/fixtures/research-batch.json | 53 +++ .../controller/known-good/index.html | 12 + .../controller/known-good/package.json | 18 + .../controller/known-good/src/behavior.ts | 12 + .../controller/known-good/src/client.tsx | 157 +++++++ .../controller/known-good/src/server.ts | 403 ++++++++++++++++++ .../controller/known-good/src/style.css | 99 +++++ .../known-good/tests/behavior.node.js | 20 + .../known-good/tsconfig.server.json | 13 + .../controller/known-good/vite.config.ts | 7 + .../controller/oracle-manifest.json | 26 ++ .../rivals/confidence-only-qualification.ts | 12 + .../rivals/destructive-reasonless-override.ts | 12 + .../controller/rivals/discarded-provenance.ts | 12 + .../rivals/external-runtime-request.ts | 12 + .../controller/rivals/in-memory-only-state.ts | 12 + .../rivals/non-dominant-suppression.ts | 12 + .../controller/rivals/overbroad-export.ts | 12 + .../rivals/provider-failure-laundering.ts | 12 + .../controller/rivals/unapproved-research.ts | 12 + .../public-contract.json | 63 +++ .../cases/prospect-research-workspace/spec.md | 35 ++ 55 files changed, 3081 insertions(+), 55 deletions(-) create mode 100644 .changeset/tricky-poems-do.md create mode 100644 src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.test.ts create mode 100644 src/dev/execution-comparison/prospect-research-workspace-oracle.ts create mode 100644 src/dev/execution-comparison/prospect-research-workspace-oracle/journey-runner.ts create mode 100644 src/dev/execution-comparison/prospect-research-workspace-oracle/journeys.ts create mode 100644 src/dev/execution-comparison/prospect-research-workspace-oracle/lifecycle.ts create mode 100644 src/dev/execution-comparison/prospect-research-workspace-oracle/reference.ts create mode 100644 src/dev/execution-comparison/prospect-research-workspace-oracle/runner.ts create mode 100644 src/dev/execution-comparison/prospect-research-workspace-oracle/sqlite-evidence.ts create mode 100644 src/dev/execution-comparison/prospect-research-workspace-oracle/types.ts create mode 100644 testing/comparisons/missions/prospect-research-workspace.md create mode 100644 testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json create mode 100644 testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md create mode 100644 testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/provider-failure.json create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/index.html create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/package.json create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/behavior.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/client.tsx create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/server.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/style.css create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/tests/behavior.node.js create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/tsconfig.server.json create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/vite.config.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/oracle-manifest.json create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/confidence-only-qualification.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/destructive-reasonless-override.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/discarded-provenance.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/external-runtime-request.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/in-memory-only-state.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/non-dominant-suppression.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/overbroad-export.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/provider-failure-laundering.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/unapproved-research.ts create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json create mode 100644 testing/execution-comparisons/cases/prospect-research-workspace/spec.md diff --git a/.changeset/tricky-poems-do.md b/.changeset/tricky-poems-do.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/tricky-poems-do.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/docs/praxis/comparison-runs.md b/docs/praxis/comparison-runs.md index 4bf71e99e..12dc7516c 100644 --- a/docs/praxis/comparison-runs.md +++ b/docs/praxis/comparison-runs.md @@ -129,6 +129,14 @@ audience-safe requirement ledger, and the bounded validity-first report. This historical witness predates Alpha 10 provenance capture. Keep it as evidence and an example, but do not backfill `provenance.json` or publish it through the current publication skill. +FE-1253 adds `prospect-research-workspace` as a separate second greenfield case. Its fixed public +stack is React + Node.js + TypeScript + SQLite, with deterministic server-side Pi/Clay-compatible +fixtures and denied runtime network. The mission path is +`testing/comparisons/missions/prospect-research-workspace.md`. Its compiled full-stack +browser/HTTP/SQLite/export oracle is calibrated against a known-good implementation and focused wrong +rivals. No elicitation or execution provider turn is valid until strict greenfield Claude isolation is +present and witnessed on the parent stack. + ## Agent recipe: drive, then join evidence On an overlay-capable host, drive the real TUI with the pinned project-local `pi-interactive-shell` package. In a sandbox or headless environment, use `npm run tui-driver` as the fallback. Follow the bounded observation, named-key input, and deterministic cleanup protocol in [Manual Testing](manual-testing.md). diff --git a/memory/PLAN.md b/memory/PLAN.md index 9a5472079..23107964d 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -119,6 +119,7 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand - `executor-slice-admission-parity` ([FE-1240](https://linear.app/hash/issue/FE-1240/prevent-invalid-scoped-slices-from-reaching-execution)) — **complete on `ka/fe-1240-slice-admission-parity`, restacked on `next` after FE-1239 landed:** incomplete scoped-slice worker context is rejected during deterministic plan admission, exact findings enter bounded repair before `slice_execute`, and the execution boundary remains fail-closed. Definition below. - `executor-plan-coherence` ([FE-1250](https://linear.app/hash/issue/FE-1250/build-coherent-execution-plans)) — **implementation complete on `ka/fe-1250-coherent-execution-plans`, based on FE-1240 via `next`:** frontier-verified multi-slice epics now require one ordinary terminal member over every sibling, planner conduct establishes shared foundations, and workers preserve cumulative public contracts for the canonical harness. Fast verification passes; next is the separately authorized unchanged Petri comparison rerun. No new browser gate, durable plan kind, or executor lifecycle phase. Definition below. - `comparison-publication-workflow` ([FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports)) — **active on `ka/fe-1251-comparison-publication`, stacked on FE-1250:** capture immutable controller and release provenance before comparison lanes start, then explicitly publish retained validity-first reports into the canonical Notion database through a guarded idempotent skill. Definition below. +- `prospect-research-workspace-e2e` ([FE-1253](https://linear.app/hash/issue/FE-1253/add-prospect-research-end-to-end-comparison-case)) — **active on `ka/fe-1253-prospect-research-e2e`, stacked on FE-1241:** the frozen packets and opaque Brunch handoff are materialized, and the independent browser/HTTP/SQLite/export oracle is calibrated against a known-good full stack plus focused wrong rivals. Next: restore and witness strict greenfield Claude isolation on the landed parent before any scored provider turn. - **Carved from FE-1167 (2026-07-13):** the Execute-mode evidence sub-list — Execute entry beats on thin vs rich seeds (assessment honesty: Ask on thin, Proceed on rich) and the FE-1107/KA residue (close-or-narrow, demo/walkthrough session via `TESTING_PLAN.md`, post-KA plan pass). The former sticky-posture question is no longer KA residue: FE-1187 `remediation-4` owns the persistent Specify elicitation-style audit/SPEC revision, and its Continue lexical audit owns the old `continue` ambiguity. Full context in the archived FE-1167 definition (`docs/archive/PLAN_HISTORY.md`). - `planning-process-model` — **moved to the KA stream 2026-07-13; reshaped by D126-L**: the durable scope handoff is settled, so this item now owns only plan projection and epistemic-horizon questions beyond committed scopes. Definition below. - **[1.x data-model handoff owed by FE-1187](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md):** a concise colleague-facing note must distinguish current canon from directional vocabulary before the consolidated outer checkpoint. Current: `{milestone, frontier, scope}` with executor-derived slices; basis (`explicit | implicit`) orthogonal to settlement (`advisory | settled`); no persisted readiness grade or spec-global elicitation-gap table; active-branch Pi JSONL reads; one CommandExecutor mutation authority with spec-local LSN/change log; no new projected `vv_obligation` (legacy rows remain readable). Directional only: explain stored `thesis` as pitch/concept without renaming it yet, and avoid new coupling to spec-local `term` while its possible workspace lift remains future work. Persisted judgment-shaped reconciliation needs remain current, but are YAGNI-suspect: add no kinds/consumers/orchestration dependency without fresh evidence; re-evaluate derivation/removal when KA work first needs that table. Delivered note: [`Brunch 1.x data-model handoff`](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md); it does not block the deterministic TUI queue. @@ -456,6 +457,25 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Boundary:** no hidden-oracle exposure, inferred browser command, new plan-plane node, candidate command surface, execution-time plan repair, or FE-1241 comparison-framework change. - **Current execution pointer:** none; the two-card scope is consumed. Re-enter only for the unchanged frozen Petri comparison rerun. +### prospect-research-workspace-e2e + +- **Name:** Add prospect research full-stack end-to-end comparison case +- **Linear / branch:** [FE-1253](https://linear.app/hash/issue/FE-1253/add-prospect-research-end-to-end-comparison-case); `ka/fe-1253-prospect-research-e2e`, stacked on FE-1241 with no parent issue. +- **Kind:** structural evaluation expansion — a second greenfield whole-application case with a full-stack lifecycle and independent browser/API/SQLite oracle. +- **Certainty:** proving. +- **Status:** active 2026-07-22. The content-addressed mission/baseline/registry/public packet, opaque Brunch execution seed, and closed compiled oracle are materialized. The exact npm lifecycle and seven independent claim-linked journeys pass the known-good full stack; eight focused semantic rivals plus an external-request rival prove sensitivity and deterministic cleanup. No scored provider turn exists yet because strict greenfield Claude isolation remains stop-the-line. +- **Objective:** prove that FE-1239's exact elicitation → immutable handoff → crossed Brunch/Claude execution → requirement-ledger path can build and assess a local full-stack prospect research product rather than only a static client application. +- **Lights up:** private prospect mission + public mechanical baseline → two independently approved specifications → exact crossed 2×2 execution → deterministic research/qualification fixtures → browser, HTTP, and SQLite evidence → requirement-level report. +- **Stabilizes:** a compile-time second greenfield profile; fixed React + Node.js + TypeScript + SQLite delivery; `npm test` / `npm run build` / `npm start`; environment-addressed port/database/fixture inputs; runtime-network denial; server-side fixture adapters; and opaque exact-spec Brunch seeding without Petri-specific parsing. +- **Depends on:** FE-1241 landing/restack for the finalized case registry and strict solution-isolation substrate. The reviewed greenfield Claude isolation gap is stop-the-line: no scored FE-1253 lane may launch until strict MCP/tool/sandbox/controller-root/network policy is restored and witnessed on the actual parent. +- **Boundary:** fresh empty Git repositories only; one fixed TypeScript stack but open build tool, Node framework, ORM, router, component library, CSS, and internal architecture. Manually initiated prospect research, evidence-backed qualification, deduplication/provenance, suppression, review, audited override, approval, export, provider failure, and restart persistence are in. Fixture adapters are scored; live Pi/Clay service quality is not. +- **Acceptance:** all mission/baseline/registry/public/oracle bytes freeze before the first valid elicitation lane; exact approved specs cross unchanged into all four cells; the full-stack lifecycle starts from public commands and a fresh temporary SQLite database; one known-good fixture passes; focused rivals for unapproved research, confidence-only qualification, lost provenance, weak suppression, reasonless/destructive override, overbroad export, provider-failure laundering, and non-durable state fail; every check retains claim-linked evidence and later journeys run after earlier failures; existing Petri and brownfield case bytes/oracles remain unchanged. +- **Verification:** inner — exact public/study/oracle parsers, hash drift, claim coverage, opaque Brunch seed, and strict isolation admission. Middle — independent browser + API + SQLite journeys over fresh database/fixture state, paired with a tiny controller reference model and focused wrong rivals. Outer — two rigorous elicitation lanes and the closed 2×2 Brunch/Claude execution matrix, followed by validity-first reporting; no winner, reliability estimate, live-provider grade, or cross-case causal claim. +- **Cross-cutting obligations:** public addressability receives no elicitation credit; mission/reveal/fixtures/expected states remain outside targets; strict Claude and Brunch isolation precedes scored execution; runtime network stays denied; all failed/invalid attempts are retained; Brunch stops at `promotion_prepared` and never lands. +- **Explicitly out:** sequencing, sending, follow-ups, mailbox/reply handling, unsubscribe/bounce automation, scheduler, CRM sync, live Clay/Pi credentials, production deployment, Cursor/Codex lanes, arbitrary manifest commands/plugins, `ExecutionAttempt` widening, automatic winner, and reliability repetitions. +- **Traceability:** D70-L, D134-L, D139-L; I67-L; FE-1210/FE-1230/FE-1232/FE-1239/FE-1241; [`docs/praxis/comparison-runs.md`](../docs/praxis/comparison-runs.md). +- **Current execution pointer:** none. The full-stack-oracle card is implemented; next scope the strict greenfield isolation witness required before scored provider execution. + ### comparison-reporting-skills - **Name:** Report comparison evidence @@ -647,8 +667,12 @@ KA stream: -[hard]-> brownfield-comparison-cases status: complete FE-1241 with learning-first pinned snapshots, deterministic case oracles, and publication-compatible execution runs lights_up: same-base brownfield elicitation -> exact handoff -> crossed execution -> case oracle - cases: minimal-petri-net-editor = sole greenfield reference | FE-1201-derived brunch backend | PR #9051 petrinaut frontend - excludes: Clay | more greenfields | repetitions | aggregate winner | ExecutionAttempt widening + cases: minimal-petri-net-editor = FE-1241 greenfield reference | FE-1201-derived brunch backend | PR #9051 petrinaut frontend + excludes: Clay | more greenfields inside FE-1241 | repetitions | aggregate winner | ExecutionAttempt widening + -[hard]-> prospect-research-workspace-e2e (FE-1253) + status: active; full-stack oracle calibrated, scored lanes wait on strict greenfield Claude isolation + lights_up: prospect mission -> exact elicited specs -> crossed execution -> browser/API/SQLite evidence + excludes: outreach delivery | live provider grading | repetitions | aggregate winner # executor-run-environment (FE-1166) resolved 2026-07-15: policy merged (PR #302), # live remainder folded into executor-plan-synthesis (FE-1197), card consumed. diff --git a/memory/SPEC.md b/memory/SPEC.md index 99b33e056..7fcdc4f41 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -317,6 +317,8 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D98-L | Operational mode remains the only top-level role/authority state: live product modes are `specify` / Specify and `execute` / Execute, each 1:1 with its foreground agent. FE-1187 narrowly amends the former “no conduct bias as runtime state” rule: Specify additionally carries one user-changeable `elicitation_style` (`interrogate | disambiguate | propose`) that changes elicitor process only, never authority, capability, agent identity, or target graph plane. This does not revive the retired generic strategy/lens/method axes, an Enhance mode, persisted lens selection, or capability routing as mutable state. Supersedes: D98-L's blanket prohibition on every secondary session axis while preserving its two-mode and axis-retirement conclusions. | [`src/.pi/extensions/TOPOLOGY.md`](../src/.pi/extensions/TOPOLOGY.md), [`src/session/TOPOLOGY.md`](../src/session/TOPOLOGY.md) | active — materialized 2026-07-16 (FE-1187) | | D100-L | `project` is a distinct first-level live Specify-mode skill home for cross-plane derivation, not a `generate` sub-mode. `generate` fans out alternatives within a target plane from context; `project` starts from accepted upstream graph anchors and derives downstream plane candidates/drafts plus connecting edge intent. It uses the existing structured-exchange offer/terminal seams (`present_candidates`, `present_review_set`, and their declared `ask` continuations per D116-L) and hands exact graph expression back to `map` / review-set commitment; it adds no product tool, exchange schema family, or direct graph-write path. Depends on: D95-L, D96-L, D97-L, I51-L. | [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md), [`src/agents/subagents/TOPOLOGY.md`](../src/agents/subagents/TOPOLOGY.md) | active | +| D139-L | The prospect research workspace is the second admitted greenfield end-to-end case (2026-07-22, FE-1253), separate from FE-1241's two brownfield replays. Its public execution contract fixes one npm repository with a React/TypeScript frontend, Node.js/TypeScript backend, SQLite persistence, `npm test` / `npm run build` / `npm start`, environment-addressed port/database/fixture paths, a health endpoint, accessible mechanical controls, package-registry-only install, and denied runtime network. Pi-compatible qualification and Clay-compatible research are server-side interfaces exercised only through controller-owned deterministic local fixtures; live provider quality is not scored. Product semantics such as evidence sufficiency, deduplication provenance, suppression precedence, override history, export eligibility, failure honesty, and restart durability remain reveal-gated and controller-oracle-owned. The case reuses FE-1239's exact handoff, fixed 2×2 matrix, failure retention, and validity-first reporting without arbitrary manifest commands/plugins or `ExecutionAttempt` widening. Depends on: D70-L, D134-L, FE-1239, FE-1241. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `prospect-research-workspace-e2e`; `testing/{comparisons,end-to-end-comparisons,execution-comparisons}` case homes | active — contract and independent full-stack oracle calibrated; scored lanes remain isolation-gated | + ### Critical Invariants @@ -694,6 +696,8 @@ For agent-as-user evaluation, the primary behavioral claim is **consequential-fa **Brownfield execution comparison assessment (updated 2026-07-23).** Observability is **partial**: pinned trees, diffs, build/test output, controller verdicts, temporary-host Git state, browser DOM/accessibility state, request/abort evidence, interventions, cleanup, and retained attempts are text-native; visual hierarchy and provider conduct remain external. Reproducibility is **high for deterministic mechanics and unproven for campaigns**: frozen Brunch/Petrinaut packets, remote-free pinned snapshots, case-owned recipes/oracles, the fake optimizer, and contrastive rivals are stable, while generated implementation and model conduct vary. Controllability is **partial**: Brunch host landing uses disposable repositories and an independent Git model; Petrinaut uses focused builds and same-origin browser checks after lane termination. Lightweight tool restrictions prevent accidental contamination but do not establish adversarial isolation; reports must preserve that limitation. +**FE-1253 prospect research full-stack assessment (2026-07-22).** Observability is **high for structural behavior**: browser accessibility, HTTP responses, SQLite rows, downloaded export bytes, bounded process output, and browser runtime requests are text-native; whether qualification rationales are persuasive remains qualitative. Reproducibility is **high mechanically**: each claim starts the exact public lifecycle with a fresh port, temporary database, copied deterministic fixture, and browser context; a known-good React/Node/TypeScript/SQLite implementation passes while focused wrong rivals prove sensitivity. Controllability is **high for the calibrated mechanical oracle and partial for the campaign**: lifecycle, local fixtures, independent journeys, reference-model comparison, cleanup, and runtime-request rejection are agent-controlled, while provider conduct and strict greenfield Claude isolation remain external gates. Strict Claude solution isolation is a stop-the-line prerequisite for scored lanes, not an accepted blind spot. + **Standalone-web tracer assessment (2026-07-14).** Observability is **partial**: JSONL, target-addressed RPC, semantic event frames, and accessible DOM/text states make structural session behavior observable; stream cadence, visual hierarchy, and error feel remain manual. Reproducibility is **high** for this tracer: paired temporary production boots use the deterministic faux provider and an existing JSONL session, not a live model or a static transcript golden. Controllability is **partial**: the middle loop controls the browser journey and cache/overlay loss, while a workbench walkthrough owns the bounded UX verdict. The normalizer may remove only declared nondeterministic ids/timestamps; it must not mask Brunch binding/runtime/exchange differences. **Combined trajectory/evaluation assessment.** The completed interactive TUI driver raises presentation-path controllability but does not by itself close causal attribution, real-provider reproducibility, or semantic ground truth. The first evaluation frontier should address those gaps with a minimal text-native legibility envelope over existing Pi lifecycle events, provider-payload introspection, Pi JSONL, TUI observations, and graph readback — not with OTel adoption. Its run/scenario/report contract must be product-neutral enough for later Claude Code or Cursor adapters, while Brunch-only trajectory enrichment remains optional diagnostic evidence. Campaigns are automated through a controlled user actor; deterministic checks own structural facts, and a small human-labeled set calibrates and audits semantic judgments. @@ -777,6 +781,9 @@ Dev-loop artifacts route to gitignored `.fixtures/scratch///`, res | Middle | **Brunch black-box `/brunch:land` journey plus independent Git model** | The controller creates disposable host/run/review repositories, resumes a settled candidate session under `PI_OFFLINE=1`, drives only the public TUI command, snapshots Git and run metadata before confirmation, then compares post-apply state with a controller-owned full-range reference model. This proves no pre-confirm mutation, complete multi-slice landing rather than final-commit-only application, mode-aware brownfield integration and greenfield materialization, bookkeeping exclusion, honest failure, and terminal `landed` without importing candidate host-landing modules or permitting a provider turn. | | Middle | **Petrinaut standalone fake-provider browser and accessibility oracle** | In the full pinned HASH checkout, controller-prepared focused Petrinaut packages launch the public `/optimization` website route. A deterministic loopback fake optimizer proves capability-present Optimizations visibility, scenario-first configuration, fixed/optimized parameter selection, one metric plus direction, request construction, streamed trials/best-so-far/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and reachable accessible controls without requiring the real optimizer service. | | Outer | **Brownfield live/qualitative evidence** | One fully provisioned HASH `/processes/draft` host/iframe smoke checks capability relay, optimizer proxying, iframe isolation, and adapter reality; one optional real-optimizer Petrinaut smoke checks live response compatibility. Identity-masked visual review judges Petrinaut hierarchy, clarity, and progress legibility; identity-masked code review judges both outputs. These are non-gating findings, are `not_assessable` when unavailable, and cannot override or repair common mechanical failures. | +| Inner | **FE-1253 prospect full-stack case contracts** | Exact parsers and content hashes freeze the private mission, public mechanical baseline, reveal registry, React/Node/TypeScript/SQLite public packet, fixture files, closed oracle manifest, budgets, and fixed 2×2 axes. Brunch preserves each elicited specification as one opaque settled handoff instead of applying Petri-specific parsing. | +| Middle | **FE-1253 independent browser/API/SQLite differential** | A code-owned runner starts each candidate with a fresh database and local fixture, then compares accessible UI, HTTP results, SQLite state, and export bytes with a tiny controller reference model. Independent journeys cover approval-before-run, evidence qualification, deduplication provenance, suppression across rerun, reasoned override history, approved-only export, provider-failure honesty, and restart persistence; focused wrong rivals prove sensitivity. | +| Outer | **FE-1253 crossed prospect workspace campaign** | Two rigorous elicitation lanes produce exact approved specifications; each crosses into Brunch and Claude execution. All four cells retain strict isolation, target-visible conduct, unchanged handoffs, common oracle evidence, failures, cleanup, and a validity-first requirement ledger. The result supports bounded within-case contrasts only, not a winner, reliability estimate, live-provider grade, or cross-case causal claim. | | Middle | **FE-1187 controlled provider conduct gate — paused** | On explicit re-entry, reconcile the extractor/oracle against the landed mixed-settlement contract, then run three fresh normalized-ingest samples. All must use free-text digest feedback, one bounded questionnaire when several questions exist, no combinatorial options, an honestly assigned per-node/per-edge review proposal when review is used, exact atomic settlement preservation, and no post-review mutation that completes or rewrites the approved proposal. Direct advisory mutation without review remains valid. Reports retain provider/model stamps and deterministic conduct markers; the stopped 2026-07-17 run is diagnostic and counts 0/3. | | Inner | **FE-1187 Impact Ledger golden + word-wrap-tolerant render-honesty** | Golden/inline snapshots at narrow/normal/wide widths lock populated-section-only canonical order, absence of empty heading/`None` pairs, per-node/per-edge settlement visibility, elision, `refs:` row shape, and the `obligation` fallback label for the borderless Impact Ledger renderer (D27-L/D131-L). `missingRenderedDetailsLeaves` is extended to reassemble `table`'s word-wrapped physical lines back into logical cells before leaf-presence checking, so a value silently split across wrapped rows cannot pass as "rendered" by accident. | | Middle | **FE-1187 Impact Ledger differential reference extractor** | A deliberately naive reference extractor (flat node/edge/term inventory, no styling or grouping) is compared against the ledger's code/connection inventory over the witnessed fixture plus hand-authored edge fixtures (empty group, single-node group, mixed-settlement group, term-only group, max-refs group). Proves item/status inventory completeness and empty-group omission independent of the "real" renderer's own logic. | diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index 207dca50f..4d74ccc97 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -31,7 +31,13 @@ It does not own published CLI behavior, public RPC contracts, database imports f Petrinaut preparation selected by D136-L is split across the historical-replay operation and oracle. `execution-comparison/operator-cli.ts` resolves every `pinned_git` contract into the shared operation; Petrinaut then runs the one immutable install (`corepack yarn install --immutable --mode=skip-build`) before either lane. Install failure or tracked-source mutation removes only the newly owned target. After lane termination, `execution-comparison/petrinaut-optimization-oracle.ts` runs the closed focused builds, deterministic loopback optimizer responses, and black-box browser journeys against `/optimization`. The compiled sequence includes `@hashintel/refractive` after design-system/core/optimizer prerequisites and before Petrinaut UI. Source-backed mechanical addresses and contrastive rivals remain frozen in the case. Recalibrate parent-fails/reference-passes only when the pinned source, public contract, or oracle changes; routine attempts do not duplicate the HASH install or reference checkout. -`execution-comparison-operator.ts` dispatches both brownfield oracles and the retained Petri browser oracle through a closed compile-time registry and includes every implementation module in the immutable oracle-pack hash; manifests cannot select paths, commands, or plugins at runtime. +`execution-comparison-operator.ts` dispatches both brownfield oracles and the Petri and prospect greenfield oracles through a closed compile-time registry and includes every implementation module in the immutable oracle-pack hash; manifests cannot select paths, commands, or plugins at runtime. + +## Prospect Research Greenfield Case + +D139-L admits `prospect-research-workspace-v1` as the second greenfield profile. Its private mission lives under `testing/comparisons/missions/`; its disclosed full-stack lifecycle baseline and reveal registry live under `testing/end-to-end-comparisons/cases/prospect-research-workspace/`; and its public execution packet plus controller fixtures/manifest live under the matching `testing/execution-comparisons/` case home. The public contract fixes React + Node.js + TypeScript + SQLite, npm lifecycle commands, health/addressability, environment-selected local fixtures, and denied runtime network while leaving framework internals open. Brunch seeds the exact target-authored specification opaquely rather than passing it through the Petri section parser. + +`execution-comparison/prospect-research-workspace-oracle.ts` is the public controller entry point. Its private subtree owns the exact npm test/build/start lifecycle, fresh process/database/fixture isolation, accessible browser actions, direct same-origin HTTP and SQLite/export evidence, a tiny independent fixture model, claim-linked journeys, runtime-request detection, and deterministic browser/process cleanup. The controller case home carries the known-good React/Node/TypeScript/SQLite full stack and focused wrong rivals; all participate in the immutable pack. Strict Claude solution isolation remains a stop-the-line prerequisite for every scored greenfield lane. ## Launcher Surface diff --git a/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts b/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts index cd4a171ac..8be6e0112 100644 --- a/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts @@ -22,25 +22,41 @@ const petrinautStudy = join( repositoryRoot, 'testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json', ); +const prospectStudy = join( + repositoryRoot, + 'testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json', +); +const prospectExecution = join( + repositoryRoot, + 'testing/execution-comparisons/cases/prospect-research-workspace', +); const brunchExecution = join(repositoryRoot, 'testing/execution-comparisons/cases/brunch-host-landing'); const petrinautExecution = join(repositoryRoot, 'testing/execution-comparisons/cases/petrinaut-optimization'); const execFileAsync = promisify(execFile); describe('compiled end-to-end comparison case profiles', () => { - it('loads the unchanged Petri case and exactly two pinned brownfield cases', async () => { - const [petri, brunch, petrinaut, brunchPacket, petrinautPacket] = await Promise.all([ - loadEndToEndStudyContract({ repositoryRoot, contractPath: petriStudy }), - loadEndToEndStudyContract({ repositoryRoot, contractPath: brunchStudy }), - loadEndToEndStudyContract({ repositoryRoot, contractPath: petrinautStudy }), - loadPublicCasePacket(brunchExecution), - loadPublicCasePacket(petrinautExecution), - ]); + it('loads two greenfield cases and exactly two pinned brownfield cases', async () => { + const [petri, prospect, brunch, petrinaut, prospectPacket, brunchPacket, petrinautPacket] = + await Promise.all([ + loadEndToEndStudyContract({ repositoryRoot, contractPath: petriStudy }), + loadEndToEndStudyContract({ repositoryRoot, contractPath: prospectStudy }), + loadEndToEndStudyContract({ repositoryRoot, contractPath: brunchStudy }), + loadEndToEndStudyContract({ repositoryRoot, contractPath: petrinautStudy }), + loadPublicCasePacket(prospectExecution), + loadPublicCasePacket(brunchExecution), + loadPublicCasePacket(petrinautExecution), + ]); expect(petri.contract).toMatchObject({ id: 'minimal-petri-net-editor-e2e-v1', caseId: 'minimal-petri-net-editor-v1', oracle: { id: 'minimal-petri-net-editor-oracles-v2' }, }); + expect(prospect.contract).toMatchObject({ + id: 'prospect-research-workspace-e2e-v1', + caseId: 'prospect-research-workspace-v1', + oracle: { id: 'prospect-research-workspace-oracles-v1' }, + }); expect(brunch.contract).toMatchObject({ id: 'brunch-host-landing-e2e-v1', caseId: 'brunch-host-landing-v1', @@ -70,6 +86,13 @@ describe('compiled end-to-end comparison case profiles', () => { parentTree: brunch.contract.source?.parentTree, }, }); + expect(prospectPacket.contract.case).toMatchObject({ + product: 'prospect_research_workspace', + mode: 'greenfield', + scope: 'whole_application', + surface: 'full_stack', + repository: { substrate: 'empty_dir', base: 'fresh-empty-commit' }, + }); expect(petrinautPacket.contract.case).toMatchObject({ product: 'petrinaut', mode: 'brownfield', diff --git a/src/dev/end-to-end-comparison/__tests__/study-contract.test.ts b/src/dev/end-to-end-comparison/__tests__/study-contract.test.ts index 590a5933d..fcf4e0e1e 100644 --- a/src/dev/end-to-end-comparison/__tests__/study-contract.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/study-contract.test.ts @@ -75,6 +75,27 @@ describe('end-to-end study contract', () => { expect(loaded.contractSha256).toMatch(/^sha256:[a-f0-9]{64}$/u); }); + it('loads the content-addressed prospect research study', async () => { + const repositoryRoot = fileURLToPath(new URL('../../../../', import.meta.url)); + const loaded = await loadEndToEndStudyContract({ + repositoryRoot, + contractPath: fileURLToPath( + new URL( + '../../../../testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json', + import.meta.url, + ), + ), + }); + + expect(loaded.contract).toMatchObject({ + id: 'prospect-research-workspace-e2e-v1', + caseId: 'prospect-research-workspace-v1', + oracle: { id: 'prospect-research-workspace-oracles-v1' }, + specSources: ['brunch_spec', 'claude_spec'], + executors: ['brunch', 'claude_code'], + }); + }); + it('accepts the frozen two-by-two study shape', () => { expect(parseEndToEndStudyContract(study())).toEqual(study()); }); diff --git a/src/dev/end-to-end-comparison/study-contract.ts b/src/dev/end-to-end-comparison/study-contract.ts index 9ecedff0d..2a07b0971 100644 --- a/src/dev/end-to-end-comparison/study-contract.ts +++ b/src/dev/end-to-end-comparison/study-contract.ts @@ -82,6 +82,10 @@ export function parseEndToEndStudyContract(value: unknown): EndToEndStudyContrac value['caseId'] === 'minimal-petri-net-editor-v1' && oracle['id'] === 'minimal-petri-net-editor-oracles-v2' && source === undefined) || + (value['id'] === 'prospect-research-workspace-e2e-v1' && + value['caseId'] === 'prospect-research-workspace-v1' && + oracle['id'] === 'prospect-research-workspace-oracles-v1' && + source === undefined) || (value['id'] === 'petrinaut-optimization-e2e-v1' && value['caseId'] === 'petrinaut-optimization-v1' && oracle['id'] === 'petrinaut-optimization-oracles-v1' && diff --git a/src/dev/execution-comparison-operator.ts b/src/dev/execution-comparison-operator.ts index f60f643b0..24d397adf 100644 --- a/src/dev/execution-comparison-operator.ts +++ b/src/dev/execution-comparison-operator.ts @@ -28,18 +28,33 @@ import { runPetrinautOptimizationOracle, type PetrinautOptimizationOracleReport, } from './execution-comparison/petrinaut-optimization-oracle.js'; +import { + runProspectResearchWorkspaceOracle, + type ProspectResearchWorkspaceOracleReport, +} from './execution-comparison/prospect-research-workspace-oracle.js'; type CompiledOracleId = | 'minimal-petri-net-editor-oracles-v2' | 'brunch-host-landing-oracles-v1' - | 'petrinaut-optimization-oracles-v1'; + | 'petrinaut-optimization-oracles-v1' + | 'prospect-research-workspace-oracles-v1'; + +type CompiledOracleReport = + | BrowserOracleReport + | HostLandingOracleReport + | PetrinautOptimizationOracleReport + | ProspectResearchWorkspaceOracleReport; interface CompiledOracle { readonly implementationFiles: readonly string[]; readonly run: (input: { readonly appDir: string; readonly caseDir: string; - }) => Promise; + }) => Promise; +} + +interface ResolvedCompiledOracle extends CompiledOracle { + readonly id: CompiledOracleId; } const COMPILED_ORACLES: Readonly> = { @@ -92,24 +107,59 @@ const COMPILED_ORACLES: Readonly> = { run: async ({ appDir, caseDir }) => await runPetrinautOptimizationOracle({ candidateRoot: appDir, caseDir }), }, + 'prospect-research-workspace-oracles-v1': { + implementationFiles: [ + fileURLToPath(new URL('./execution-comparison/prospect-research-workspace-oracle.ts', import.meta.url)), + fileURLToPath( + new URL('./execution-comparison/prospect-research-workspace-oracle/runner.ts', import.meta.url), + ), + fileURLToPath( + new URL('./execution-comparison/prospect-research-workspace-oracle/journeys.ts', import.meta.url), + ), + fileURLToPath( + new URL( + './execution-comparison/prospect-research-workspace-oracle/journey-runner.ts', + import.meta.url, + ), + ), + fileURLToPath( + new URL('./execution-comparison/prospect-research-workspace-oracle/lifecycle.ts', import.meta.url), + ), + fileURLToPath( + new URL('./execution-comparison/prospect-research-workspace-oracle/reference.ts', import.meta.url), + ), + fileURLToPath( + new URL( + './execution-comparison/prospect-research-workspace-oracle/sqlite-evidence.ts', + import.meta.url, + ), + ), + fileURLToPath( + new URL('./execution-comparison/prospect-research-workspace-oracle/types.ts', import.meta.url), + ), + ], + run: async ({ appDir, caseDir }) => + await runProspectResearchWorkspaceOracle({ candidateRoot: appDir, caseDir }), + }, }; -export function resolveCompiledExecutionOracle(id: string): CompiledOracle { +export function resolveCompiledExecutionOracle(id: string): ResolvedCompiledOracle { if ( id !== 'minimal-petri-net-editor-oracles-v2' && id !== 'brunch-host-landing-oracles-v1' && - id !== 'petrinaut-optimization-oracles-v1' + id !== 'petrinaut-optimization-oracles-v1' && + id !== 'prospect-research-workspace-oracles-v1' ) { throw new Error(`unknown compiled execution oracle id: ${id}`); } - return COMPILED_ORACLES[id]; + return { id, ...COMPILED_ORACLES[id] }; } export async function retainCompiledOracleReport(input: { readonly out: string; readonly oracleId: CompiledOracleId; readonly oraclePackSha256: string; - readonly report: BrowserOracleReport | HostLandingOracleReport | PetrinautOptimizationOracleReport; + readonly report: CompiledOracleReport; }): Promise<{ readonly out: string; readonly status: 'passed' | 'failed' | 'assertion_failed' | 'setup_failed'; @@ -213,7 +263,7 @@ export async function runExecutionComparisonOperatorCli(args: readonly string[]) }); const retained = await retainCompiledOracleReport({ out, - oracleId: oraclePack.manifest.id, + oracleId: oracle.id, oraclePackSha256: oraclePack.packSha256, report, }); diff --git a/src/dev/execution-comparison/__tests__/brunch-lane.test.ts b/src/dev/execution-comparison/__tests__/brunch-lane.test.ts index 5b828f3eb..be48afb9c 100644 --- a/src/dev/execution-comparison/__tests__/brunch-lane.test.ts +++ b/src/dev/execution-comparison/__tests__/brunch-lane.test.ts @@ -27,6 +27,9 @@ const caseDir = fileURLToPath( const petrinautCaseDir = fileURLToPath( new URL('../../../../testing/execution-comparisons/cases/petrinaut-optimization/', import.meta.url), ); +const prospectCaseDir = fileURLToPath( + new URL('../../../../testing/execution-comparisons/cases/prospect-research-workspace/', import.meta.url), +); describe('Brunch execution comparison lane adapter', () => { it('projects the frozen public packet into one complete greenfield execution scope', async () => { @@ -136,6 +139,20 @@ describe('Brunch execution comparison lane adapter', () => { expect(() => assertExecuteProjectionPlanReady(projection)).not.toThrow(); }); + it('prepares the prospect workspace from the exact opaque specification', async () => { + const workspaceDir = await mkdtemp(join(tmpdir(), 'brunch-prospect-execution-')); + const specification = await readFile(join(prospectCaseDir, 'spec.md'), 'utf8'); + const prepared = await prepareBrunchExecutionWorkspace({ workspaceDir, caseDir: prospectCaseDir }); + const graph = queryGraph(createDb(join(workspaceDir, '.brunch', 'brunch-v1.db')), prepared.specId); + + expect(graph.nodes.find((node) => node.source === 'e2e-handoff [exact-spec]')).toMatchObject({ + kind: 'requirement', + body: specification, + }); + expect(graph.nodes.filter((node) => node.source === 'e2e-handoff [exact-spec]')).toHaveLength(1); + expect(JSON.stringify(graph)).not.toMatch(/Petri-net|static browser/iu); + }); + it('projects an opaque brownfield specification without Petri-specific execution wording', async () => { const packet = await loadPublicCasePacket(petrinautCaseDir); const specification = await readFile(join(petrinautCaseDir, 'spec.md'), 'utf8'); diff --git a/src/dev/execution-comparison/__tests__/case-contract.test.ts b/src/dev/execution-comparison/__tests__/case-contract.test.ts index f49809310..f6583bc1d 100644 --- a/src/dev/execution-comparison/__tests__/case-contract.test.ts +++ b/src/dev/execution-comparison/__tests__/case-contract.test.ts @@ -11,9 +11,12 @@ import { isPetriControllerOracleManifest, loadControllerOraclePack } from '../or const caseDir = fileURLToPath( new URL('../../../../testing/execution-comparisons/cases/minimal-petri-net-editor/', import.meta.url), ); +const prospectCaseDir = fileURLToPath( + new URL('../../../../testing/execution-comparisons/cases/prospect-research-workspace/', import.meta.url), +); describe('execution comparison public case contract', () => { - it('accepts only the frozen greenfield and two exact brownfield profile variants', () => { + it('accepts only the two frozen greenfield and two exact brownfield profile variants', () => { const brunchContract = { schemaVersion: 1, case: { @@ -136,6 +139,56 @@ describe('execution comparison public case contract', () => { } }); + it('freezes the full-stack prospect workspace lifecycle and addressability', async () => { + const packet = await loadPublicCasePacket(prospectCaseDir); + + expect(packet.contract).toMatchObject({ + case: { + id: 'prospect-research-workspace-v1', + product: 'prospect_research_workspace', + mode: 'greenfield', + scope: 'whole_application', + surface: 'full_stack', + repository: { substrate: 'empty_dir', base: 'fresh-empty-commit' }, + }, + delivery: { + test: { command: 'npm', args: ['test'] }, + build: { command: 'npm', args: ['run', 'build'] }, + start: { command: 'npm', args: ['start'] }, + environment: ['PORT', 'DATABASE_PATH', 'RESEARCH_FIXTURE_PATH'], + runtimeNetwork: 'forbidden', + }, + acceptance: { healthPath: '/api/health', executionTerminal: 'promotion_prepared' }, + }); + + for (const mutation of [ + { case: { ...packet.contract.case, surface: 'frontend' } }, + { + delivery: { + ...packet.contract.delivery, + start: { command: 'sh', args: ['server.sh'] }, + }, + }, + { + delivery: { + ...packet.contract.delivery, + environment: ['PORT', 'DATABASE_PATH'], + }, + }, + { acceptance: { healthPath: '/health', executionTerminal: 'promotion_prepared' } }, + { + accessibility: { + ...(packet.contract as { accessibility: Record }).accessibility, + application: { role: 'application', name: 'Campaign machine' }, + }, + }, + ]) { + expect(() => parsePublicCaseContract({ ...packet.contract, ...mutation })).toThrow( + 'invalid fixed public execution contract', + ); + } + }); + it('freezes only the approved specification and public contract', async () => { const packet = await loadPublicCasePacket(caseDir); diff --git a/src/dev/execution-comparison/__tests__/operator-cli.test.ts b/src/dev/execution-comparison/__tests__/operator-cli.test.ts index 4b65c528b..af1e398cd 100644 --- a/src/dev/execution-comparison/__tests__/operator-cli.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-cli.test.ts @@ -27,6 +27,10 @@ describe('execution comparison operator case selection', () => { caseId: 'petrinaut-optimization-v1', directoryId: 'petrinaut-optimization', }, + { + caseId: 'prospect-research-workspace-v1', + directoryId: 'prospect-research-workspace', + }, ]); await expect(resolveExecutionCase('minimal-petri-net-editor', casesRoot)).resolves.toMatchObject({ caseId: 'minimal-petri-net-editor-v1', @@ -37,6 +41,10 @@ describe('execution comparison operator case selection', () => { caseId: 'petrinaut-optimization-v1', directoryId: 'petrinaut-optimization', }); + await expect(resolveExecutionCase('prospect-research-workspace-v1', casesRoot)).resolves.toMatchObject({ + caseId: 'prospect-research-workspace-v1', + directoryId: 'prospect-research-workspace', + }); }); it.each(['/tmp/minimal-petri-net-editor', '../minimal-petri-net-editor', 'controller', 'x/controller/y'])( diff --git a/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts b/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts index 58f1e1fe9..c854e3f33 100644 --- a/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts +++ b/src/dev/execution-comparison/__tests__/operator-oracle-dispatch.test.ts @@ -23,10 +23,11 @@ describe('execution comparison compiled oracle dispatch', () => { expect(EXECUTION_COMPARISON_SHARED_FRAMING).not.toContain('do not add a backend'); }); - it('selects only the three compiled implementations and rejects unknown ids before launch', () => { + it('selects only the four compiled implementations and rejects unknown ids before launch', () => { const petri = resolveCompiledExecutionOracle('minimal-petri-net-editor-oracles-v2'); const brunch = resolveCompiledExecutionOracle('brunch-host-landing-oracles-v1'); const petrinaut = resolveCompiledExecutionOracle('petrinaut-optimization-oracles-v1'); + const prospect = resolveCompiledExecutionOracle('prospect-research-workspace-oracles-v1'); expect(petri.implementationFiles).toEqual( expect.arrayContaining([expect.stringContaining('browser-oracle.ts')]), @@ -43,6 +44,14 @@ describe('execution comparison compiled oracle dispatch', () => { expect.stringContaining('petrinaut-optimization-oracle/fake-optimizer.ts'), ]), ); + expect(prospect.implementationFiles).toEqual( + expect.arrayContaining([ + expect.stringContaining('prospect-research-workspace-oracle.ts'), + expect.stringContaining('prospect-research-workspace-oracle/lifecycle.ts'), + expect.stringContaining('prospect-research-workspace-oracle/reference.ts'), + expect.stringContaining('prospect-research-workspace-oracle/sqlite-evidence.ts'), + ]), + ); expect(() => resolveCompiledExecutionOracle('runtime-plugin')).toThrow( 'unknown compiled execution oracle id', ); diff --git a/src/dev/execution-comparison/__tests__/oracle-pack.test.ts b/src/dev/execution-comparison/__tests__/oracle-pack.test.ts index 774814e1b..e4d8ad7fa 100644 --- a/src/dev/execution-comparison/__tests__/oracle-pack.test.ts +++ b/src/dev/execution-comparison/__tests__/oracle-pack.test.ts @@ -4,9 +4,11 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; +import { resolveCompiledExecutionOracle } from '../../execution-comparison-operator.js'; import { assertOracleClaimCoverage, loadControllerOracleManifest, + loadControllerOraclePack, parseControllerOracleManifest, } from '../oracle-pack.js'; @@ -23,24 +25,42 @@ const petrinautRequirementsPath = fileURLToPath( import.meta.url, ), ); +const prospectRequirementsPath = fileURLToPath( + new URL( + '../../../../testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json', + import.meta.url, + ), +); describe('compiled controller oracle manifests', () => { - it('accepts exactly three compiled variants with complete brownfield claim coverage', async () => { - const [petri, brunch, petrinaut, registry, petrinautRegistry] = await Promise.all([ - loadControllerOracleManifest(join(casesRoot, 'minimal-petri-net-editor')), - loadControllerOracleManifest(join(casesRoot, 'brunch-host-landing')), - loadControllerOracleManifest(join(casesRoot, 'petrinaut-optimization')), - readFile(requirementsPath, 'utf8').then( - (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, - ), - readFile(petrinautRequirementsPath, 'utf8').then( - (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, - ), - ]); + it('accepts exactly four compiled variants with complete non-Petri claim coverage', async () => { + const [petri, prospect, brunch, petrinaut, registry, petrinautRegistry, prospectRegistry] = + await Promise.all([ + loadControllerOracleManifest(join(casesRoot, 'minimal-petri-net-editor')), + loadControllerOracleManifest(join(casesRoot, 'prospect-research-workspace')), + loadControllerOracleManifest(join(casesRoot, 'brunch-host-landing')), + loadControllerOracleManifest(join(casesRoot, 'petrinaut-optimization')), + readFile(requirementsPath, 'utf8').then( + (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, + ), + readFile(petrinautRequirementsPath, 'utf8').then( + (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, + ), + readFile(prospectRequirementsPath, 'utf8').then( + (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, + ), + ]); expect(petri.id).toBe('minimal-petri-net-editor-oracles-v2'); + expect(prospect.id).toBe('prospect-research-workspace-oracles-v1'); expect(brunch.id).toBe('brunch-host-landing-oracles-v1'); expect(petrinaut.id).toBe('petrinaut-optimization-oracles-v1'); + expect(() => + assertOracleClaimCoverage( + prospect, + prospectRegistry.rows.map(({ id }) => id), + ), + ).not.toThrow(); expect(() => assertOracleClaimCoverage( brunch, @@ -55,6 +75,39 @@ describe('compiled controller oracle manifests', () => { ).not.toThrow(); expect(JSON.stringify(brunch)).not.toMatch(/manifestPath|command|plugin|implementationPath/u); expect(JSON.stringify(petrinaut)).not.toMatch(/manifestPath|command|plugin|implementationPath/u); + expect(JSON.stringify(prospect)).not.toMatch(/manifestPath|command|plugin|implementationPath/u); + const prospectPack = await loadControllerOraclePack({ + caseDir: join(casesRoot, 'prospect-research-workspace'), + implementationFiles: resolveCompiledExecutionOracle('prospect-research-workspace-oracles-v1') + .implementationFiles, + }); + expect(prospectPack.files.map(({ path }) => path)).toEqual( + expect.arrayContaining([ + 'controller/fixtures/provider-failure.json', + 'controller/fixtures/research-batch.json', + 'controller/known-good/src/client.tsx', + 'controller/known-good/src/server.ts', + 'controller/oracle-manifest.json', + 'controller/rivals/confidence-only-qualification.ts', + 'controller/rivals/destructive-reasonless-override.ts', + 'controller/rivals/discarded-provenance.ts', + 'controller/rivals/external-runtime-request.ts', + 'controller/rivals/in-memory-only-state.ts', + 'controller/rivals/non-dominant-suppression.ts', + 'controller/rivals/overbroad-export.ts', + 'controller/rivals/provider-failure-laundering.ts', + 'controller/rivals/unapproved-research.ts', + 'implementation/journeys.ts', + 'implementation/journey-runner.ts', + 'implementation/lifecycle.ts', + 'implementation/prospect-research-workspace-oracle.ts', + 'implementation/reference.ts', + 'implementation/runner.ts', + 'implementation/sqlite-evidence.ts', + 'implementation/types.ts', + ]), + ); + expect(prospectPack.packSha256).toMatch(/^sha256:[a-f0-9]{64}$/u); expect(() => parseControllerOracleManifest({ ...petrinaut, diff --git a/src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.test.ts b/src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.test.ts new file mode 100644 index 000000000..6faf75192 --- /dev/null +++ b/src/dev/execution-comparison/__tests__/prospect-research-workspace-oracle.test.ts @@ -0,0 +1,160 @@ +import { cp, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { resolveCompiledExecutionOracle } from '../../execution-comparison-operator.js'; +import { + runProspectResearchWorkspaceOracle, + type ProspectResearchWorkspaceOracleReport, +} from '../prospect-research-workspace-oracle.js'; +import { runManagedProspectJourney } from '../prospect-research-workspace-oracle/journey-runner.js'; + +const caseDir = fileURLToPath( + new URL('../../../../testing/execution-comparisons/cases/prospect-research-workspace/', import.meta.url), +); +const knownGood = join(caseDir, 'controller', 'known-good'); +const rivalsDir = join(caseDir, 'controller', 'rivals'); +const rootNodeModules = fileURLToPath(new URL('../../../../node_modules', import.meta.url)); +const temporaryRoots: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryRoots.splice(0).map(async (path) => await rm(path, { recursive: true, force: true })), + ); +}); + +describe('compiled prospect research workspace oracle', () => { + it('passes the exact public lifecycle and every independent claim against the known-good full stack', async () => { + const report = await runProspectResearchWorkspaceOracle({ candidateRoot: knownGood, caseDir }); + + expect(report.status).toBe('passed'); + expect(report.commands.map(({ id, command, args, status }) => ({ id, command, args, status }))).toEqual([ + { id: 'test', command: 'npm', args: ['test'], status: 'passed' }, + { id: 'build', command: 'npm', args: ['run', 'build'], status: 'passed' }, + ]); + expect(report.checks).toHaveLength(7); + expect(report.checks.every(({ status }) => status === 'passed')).toBe(true); + expect(report.checks.every(({ cleanup }) => cleanup.processStopped && cleanup.browserClosed)).toBe(true); + expect(report.externalRuntimeRequests).toEqual([]); + expect(report.checks.flatMap(({ evidence }) => evidence.map(({ source }) => source))).toEqual( + expect.arrayContaining(['browser', 'http', 'sqlite', 'export']), + ); + }, 120_000); + + it.each([ + ['unapproved-research', 'project-approval-and-research'], + ['confidence-only-qualification', 'qualification-and-deduplication'], + ['discarded-provenance', 'qualification-and-deduplication'], + ['non-dominant-suppression', 'suppression-and-rerun'], + ['destructive-reasonless-override', 'override-and-export'], + ['overbroad-export', 'override-and-export'], + ['provider-failure-laundering', 'provider-failure'], + ['in-memory-only-state', 'restart-persistence'], + ] as const)( + '%s rival fails its owning claim while later claims remain assessable', + async (rival, owner) => { + const candidateRoot = await materializeRival(rival); + const report = await runProspectResearchWorkspaceOracle({ candidateRoot, caseDir }); + + expect(check(report, owner).status).toBe('assertion_failed'); + expect(report.checks).toHaveLength(7); + expect(report.checks.every(({ cleanup }) => cleanup.processStopped && cleanup.browserClosed)).toBe( + true, + ); + const ownerIndex = report.checks.findIndex(({ id }) => id === owner); + expect(report.checks.slice(ownerIndex + 1).every(({ status }) => status !== 'setup_failed')).toBe(true); + }, + 120_000, + ); + + it('rejects an external browser runtime request and still cleans up the process and browser', async () => { + const candidateRoot = await materializeRival('external-runtime-request'); + const report = await runProspectResearchWorkspaceOracle({ candidateRoot, caseDir }); + + expect(report.status).toBe('assertion_failed'); + expect(report.externalRuntimeRequests).toContain('https://runtime.invalid/prospect-oracle'); + expect(report.checks.every(({ cleanup }) => cleanup.processStopped && cleanup.browserClosed)).toBe(true); + }, 120_000); + + it('registers every private helper in the closed compiled dispatch', () => { + const oracle = resolveCompiledExecutionOracle('prospect-research-workspace-oracles-v1'); + expect(oracle.implementationFiles.map((path) => path.split('/').at(-1))).toEqual([ + 'prospect-research-workspace-oracle.ts', + 'runner.ts', + 'journeys.ts', + 'journey-runner.ts', + 'lifecycle.ts', + 'reference.ts', + 'sqlite-evidence.ts', + 'types.ts', + ]); + expect(() => resolveCompiledExecutionOracle('prospect-runtime-plugin')).toThrow( + 'unknown compiled execution oracle id', + ); + }); + + it.each([ + ['success', async () => {}, async () => {}, 'passed'], + [ + 'setup failure', + async () => { + throw new Error('setup failed'); + }, + async () => {}, + 'setup_failed', + ], + [ + 'assertion failure', + async () => {}, + async () => { + throw new Error('assertion failed'); + }, + 'assertion_failed', + ], + ['timeout', async () => {}, async () => await new Promise(() => {}), 'assertion_failed'], + ] as const)( + '%s deterministically stops process and browser state', + async (_name, setup, assertion, status) => { + const cleanup = { processStopped: false, browserClosed: false }; + const result = await runManagedProspectJourney({ + open: async () => cleanup, + setup, + assert: assertion, + close: async (context) => { + context.processStopped = true; + context.browserClosed = true; + }, + timeoutMs: 5, + }); + + expect(result.status).toBe(status); + expect(cleanup).toEqual({ processStopped: true, browserClosed: true }); + expect(result.message.length).toBeLessThan(1_000); + }, + ); +}); + +async function materializeRival(name: string): Promise { + const root = await mkdtemp(join(tmpdir(), `brunch-prospect-${name}-`)); + temporaryRoots.push(root); + await cp(knownGood, root, { recursive: true }); + await symlink(rootNodeModules, join(root, 'node_modules'), 'dir'); + await writeFile( + join(root, 'src', 'behavior.ts'), + await readFile(join(rivalsDir, `${name}.ts`), 'utf8'), + 'utf8', + ); + return root; +} + +function check( + report: ProspectResearchWorkspaceOracleReport, + id: ProspectResearchWorkspaceOracleReport['checks'][number]['id'], +) { + const selected = report.checks.find((candidate) => candidate.id === id); + if (selected === undefined) throw new Error(`missing check ${id}`); + return selected; +} diff --git a/src/dev/execution-comparison/brunch-lane.ts b/src/dev/execution-comparison/brunch-lane.ts index 8a62c4db5..162c622a8 100644 --- a/src/dev/execution-comparison/brunch-lane.ts +++ b/src/dev/execution-comparison/brunch-lane.ts @@ -10,9 +10,11 @@ import { } from '../../graph/seed-fixtures.js'; import { isBrowserExecutionCaseContract, + isProspectResearchWorkspaceExecutionCaseContract, loadPublicCasePacket, type BrowserExecutionCasePublicContract, type ExecutionCasePublicContract, + type ProspectResearchWorkspaceExecutionCasePublicContract, type PublicCasePacket, } from './case-contract.js'; @@ -47,17 +49,20 @@ export async function prepareBrunchExecutionWorkspace(input: { } const packet = await loadPublicCasePacket(input.caseDir); - if (!isBrowserExecutionCaseContract(packet.contract)) { - throw new Error('legacy Brunch execution seed supports only the Petri browser case'); + if ( + !isBrowserExecutionCaseContract(packet.contract) && + !isProspectResearchWorkspaceExecutionCaseContract(packet.contract) + ) { + throw new Error('greenfield Brunch execution seed received a brownfield contract'); } const specification = await readFile(join(input.caseDir, packet.contract.case.specification), 'utf8'); const executor = await openWorkspaceCommandExecutor(input.workspaceDir); - const seeded = seedFixture( - executor, - input.specificationMode === 'opaque' + const fixture = isProspectResearchWorkspaceExecutionCaseContract(packet.contract) + ? buildOpaqueProspectResearchExecutionSeed({ specification, contract: packet.contract }) + : input.specificationMode === 'opaque' ? buildOpaqueBrunchExecutionSeed({ specification, contract: packet.contract }) - : buildBrunchExecutionSeed({ specification, contract: packet.contract }), - ); + : buildBrunchExecutionSeed({ specification, contract: packet.contract }); + const seeded = seedFixture(executor, fixture); const publicDir = join(input.workspaceDir, '.brunch', 'execution-comparison', 'public'); await mkdir(publicDir, { recursive: true }); await writeFile(join(publicDir, 'spec.md'), specification, { encoding: 'utf8', flag: 'wx' }); @@ -163,10 +168,81 @@ export function buildOpaqueBrownfieldExecutionSeed(input: { if (isBrowserExecutionCaseContract(input.contract) || input.contract.case.mode !== 'brownfield') { throw new Error('opaque brownfield seed requires a brownfield public contract'); } + return buildOpaqueExecutionSeed({ + specification: input.specification, + slug: input.contract.case.id, + name: `Approved ${input.contract.case.product} brownfield execution`, + frontierTitle: 'Deliver the approved brownfield change', + frontierBody: 'Implement the approved feature in the existing target repository.', + scopeTitle: 'Implement and verify the approved brownfield change', + scopeBody: + 'Execute the frozen specification as one coherent change while preserving the surrounding repository.', + moduleTitle: 'Brownfield feature implementation', + moduleBody: 'The implementation location and internal design remain the executor’s responsibility.', + criterionTitle: 'Approved brownfield behavior is satisfied', + criterionBody: + 'The completed change satisfies the frozen approved specification in the existing repository.', + checkTitle: 'Repository-local verification passes', + checkBody: + 'Use the prepared repository’s own focused build and test surfaces to verify the approved change.', + methodTitle: 'Prepared-target verification', + methodBody: + 'Run only repository-local verification available inside the prepared target under the execution isolation policy.', + }); +} + +export function buildOpaqueProspectResearchExecutionSeed(input: { + readonly specification: string; + readonly contract: ProspectResearchWorkspaceExecutionCasePublicContract; +}): SeedFixture { + return buildOpaqueExecutionSeed({ + specification: input.specification, + slug: input.contract.case.id, + name: 'Approved prospect research workspace execution', + frontierTitle: 'Deliver the prospect research workspace', + frontierBody: 'Implement the complete approved greenfield full-stack application and its own tests.', + scopeTitle: 'Implement and verify the approved prospect research workspace', + scopeBody: + 'Build the frozen specification as one coherent React, Node.js, TypeScript, and SQLite application.', + moduleTitle: 'Full-stack prospect research application', + moduleBody: + 'The React frontend, Node.js backend, SQLite store, and fixture-backed adapters form one local application.', + criterionTitle: 'Approved prospect research behavior is satisfied', + criterionBody: + 'The completed application satisfies the frozen specification and public full-stack contract.', + checkTitle: 'Public package test and build commands pass', + checkBody: 'npm test and npm run build both exit zero before promotion is prepared.', + methodTitle: 'Full-stack execution harness', + methodBody: [ + 'Use only the package-owned commands declared by the public execution packet.', + 'execute.setup: npm install', + 'execute.build: npm run build', + 'execute.verify: npm test', + ].join('\n'), + }); +} + +function buildOpaqueExecutionSeed(input: { + readonly specification: string; + readonly slug: string; + readonly name: string; + readonly frontierTitle: string; + readonly frontierBody: string; + readonly scopeTitle: string; + readonly scopeBody: string; + readonly moduleTitle: string; + readonly moduleBody: string; + readonly criterionTitle: string; + readonly criterionBody: string; + readonly checkTitle: string; + readonly checkBody: string; + readonly methodTitle: string; + readonly methodBody: string; +}): SeedFixture { return { spec: { - slug: input.contract.case.id, - name: `Approved ${input.contract.case.product} brownfield execution`, + slug: input.slug, + name: input.name, kind: 'feature', }, nodes: [ @@ -184,8 +260,8 @@ export function buildOpaqueBrownfieldExecutionSeed(input: { local_id: 2, plane: 'plan', kind: 'frontier', - title: 'Deliver the approved brownfield change', - body: 'Implement the approved feature in the existing target repository.', + title: input.frontierTitle, + body: input.frontierBody, basis: 'explicit', settlement: 'settled', source: 'execution-adapter [frontier]', @@ -194,8 +270,8 @@ export function buildOpaqueBrownfieldExecutionSeed(input: { local_id: 3, plane: 'plan', kind: 'scope', - title: 'Implement and verify the approved brownfield change', - body: 'Execute the frozen specification as one coherent change while preserving the surrounding repository.', + title: input.scopeTitle, + body: input.scopeBody, basis: 'explicit', settlement: 'settled', source: 'execution-adapter [scope]', @@ -204,8 +280,8 @@ export function buildOpaqueBrownfieldExecutionSeed(input: { local_id: 4, plane: 'design', kind: 'module', - title: 'Brownfield feature implementation', - body: 'The implementation location and internal design remain the executor’s responsibility.', + title: input.moduleTitle, + body: input.moduleBody, basis: 'explicit', settlement: 'settled', source: 'execution-adapter [module]', @@ -214,8 +290,8 @@ export function buildOpaqueBrownfieldExecutionSeed(input: { local_id: 5, plane: 'intent', kind: 'criterion', - title: 'Approved brownfield behavior is satisfied', - body: 'The completed change satisfies the frozen approved specification in the existing repository.', + title: input.criterionTitle, + body: input.criterionBody, basis: 'explicit', settlement: 'settled', source: 'execution-adapter [criterion]', @@ -224,8 +300,8 @@ export function buildOpaqueBrownfieldExecutionSeed(input: { local_id: 6, plane: 'oracle', kind: 'check', - title: 'Repository-local verification passes', - body: 'Use the prepared repository’s own focused build and test surfaces to verify the approved change.', + title: input.checkTitle, + body: input.checkBody, basis: 'explicit', settlement: 'settled', source: 'execution-adapter [check]', @@ -234,8 +310,8 @@ export function buildOpaqueBrownfieldExecutionSeed(input: { local_id: 7, plane: 'oracle', kind: 'vv_method', - title: 'Prepared-target verification', - body: 'Run only repository-local verification available inside the prepared target under the execution isolation policy.', + title: input.methodTitle, + body: input.methodBody, basis: 'explicit', settlement: 'settled', source: 'execution-adapter [vv_method]', diff --git a/src/dev/execution-comparison/case-contract.ts b/src/dev/execution-comparison/case-contract.ts index 3d2369bc2..4528c4d49 100644 --- a/src/dev/execution-comparison/case-contract.ts +++ b/src/dev/execution-comparison/case-contract.ts @@ -39,6 +39,55 @@ export interface BrowserExecutionCasePublicContract { readonly rules: readonly string[]; } +export interface ProspectResearchWorkspaceExecutionCasePublicContract { + readonly schemaVersion: 1; + readonly case: { + readonly id: 'prospect-research-workspace-v1'; + readonly specification: 'spec.md'; + readonly specificationSha256: string; + readonly provider: 'anthropic'; + readonly model: 'claude-opus-4-8'; + readonly product: 'prospect_research_workspace'; + readonly mode: 'greenfield'; + readonly scope: 'whole_application'; + readonly surface: 'full_stack'; + readonly repository: { + readonly substrate: 'empty_dir'; + readonly base: 'fresh-empty-commit'; + }; + }; + readonly budgets: { + readonly elapsedMinutes: 120; + readonly mechanicalInterventions: 2; + readonly substantiveHumanInterventions: 0; + }; + readonly delivery: { + readonly test: CommandContract; + readonly build: CommandContract; + readonly start: CommandContract; + readonly environment: readonly ['PORT', 'DATABASE_PATH', 'RESEARCH_FIXTURE_PATH']; + readonly runtimeNetwork: 'forbidden'; + readonly dependencyInstallNetwork: 'package-registry-only'; + }; + readonly acceptance: { + readonly healthPath: '/api/health'; + readonly executionTerminal: 'promotion_prepared'; + }; + readonly accessibility: { + readonly application: AccessibleNameContract; + readonly projects: AccessibleNameContract; + readonly queue: AccessibleNameContract; + readonly fields: readonly AccessibleNameContract[]; + readonly controls: readonly AccessibleNameContract[]; + readonly prospectPattern: { + readonly role: 'button'; + readonly namePattern: '^Prospect: .+ at .+$'; + }; + readonly feedbackRoles: readonly ['status', 'alert']; + }; + readonly rules: readonly string[]; +} + export interface BrunchHostLandingExecutionCasePublicContract { readonly schemaVersion: 1; readonly case: { @@ -139,6 +188,7 @@ export interface PetrinautOptimizationExecutionCasePublicContract { export type ExecutionCasePublicContract = | BrowserExecutionCasePublicContract + | ProspectResearchWorkspaceExecutionCasePublicContract | BrunchHostLandingExecutionCasePublicContract | PetrinautOptimizationExecutionCasePublicContract; @@ -149,6 +199,12 @@ export type PinnedExecutionCasePublicContract = Extract< export type PinnedExecutionCaseId = PinnedExecutionCasePublicContract['case']['id']; +export function isPinnedExecutionCaseContract( + value: ExecutionCasePublicContract, +): value is PinnedExecutionCasePublicContract { + return value.case.repository.substrate === 'pinned_git'; +} + export function isBrowserExecutionCaseContract( value: ExecutionCasePublicContract, ): value is BrowserExecutionCasePublicContract { @@ -161,6 +217,12 @@ export function isPetrinautOptimizationExecutionCaseContract( return value.case.id === 'petrinaut-optimization-v1'; } +export function isProspectResearchWorkspaceExecutionCaseContract( + value: ExecutionCasePublicContract, +): value is ProspectResearchWorkspaceExecutionCasePublicContract { + return value.case.id === 'prospect-research-workspace-v1'; +} + export interface CommandContract { readonly command: string; readonly args: readonly string[]; @@ -196,6 +258,24 @@ const ACCESSIBLE_NAME_PATTERNS = { transition: '^Transition: .+ \\((enabled|disabled)\\)$', arc: '^Arc: .+ to .+$', } as const satisfies Record<'place' | 'transition' | 'arc', AccessibleNamePattern>; +const PROSPECT_ENVIRONMENT = ['PORT', 'DATABASE_PATH', 'RESEARCH_FIXTURE_PATH'] as const; +const PROSPECT_FIELDS = ['Project name', 'Ideal customer profile', 'Decision reason'] as const; +const PROSPECT_CONTROLS = [ + 'Create project', + 'Approve project', + 'Run research', + 'Approve prospect', + 'Suppress prospect', + 'Override qualification', + 'Export approved prospects', +] as const; +const PROSPECT_RULES = [ + 'Work only in the target repository.', + 'Do not inspect controller paths or seek hidden comparison material.', + 'Use only the fixture-backed server adapters selected by the environment.', + 'Do not add sequencing, sending, mailbox, reply-classification, scheduler, or CRM behavior.', + 'Stop at promotion_prepared after npm test and npm run build pass.', +] as const; export async function loadPublicCasePacket(caseDir: string): Promise { const contractRaw = await readFile(join(caseDir, 'public-contract.json'), 'utf8'); @@ -230,9 +310,103 @@ export function parsePublicCaseContract(value: unknown): ExecutionCasePublicCont if (record(value['case']) && value['case']['id'] === 'petrinaut-optimization-v1') { return parsePetrinautOptimizationContract(value); } + if (record(value['case']) && value['case']['id'] === 'prospect-research-workspace-v1') { + return parseProspectResearchWorkspaceContract(value); + } return parsePetriBrowserContract(value); } +function parseProspectResearchWorkspaceContract( + value: Record, +): ProspectResearchWorkspaceExecutionCasePublicContract { + const caseValue = requiredRecord(value, 'case'); + const repository = requiredRecord(caseValue, 'repository'); + const budgets = requiredRecord(value, 'budgets'); + const delivery = requiredRecord(value, 'delivery'); + const acceptance = requiredRecord(value, 'acceptance'); + const accessibility = requiredRecord(value, 'accessibility'); + const prospectPattern = requiredRecord(accessibility, 'prospectPattern'); + if ( + !exactKeys(value, [ + 'schemaVersion', + 'case', + 'budgets', + 'delivery', + 'acceptance', + 'accessibility', + 'rules', + ]) || + !exactKeys(caseValue, [ + 'id', + 'specification', + 'specificationSha256', + 'provider', + 'model', + 'product', + 'mode', + 'scope', + 'surface', + 'repository', + ]) || + !exactKeys(repository, ['substrate', 'base']) || + !exactKeys(budgets, ['elapsedMinutes', 'mechanicalInterventions', 'substantiveHumanInterventions']) || + !exactKeys(delivery, [ + 'test', + 'build', + 'start', + 'environment', + 'runtimeNetwork', + 'dependencyInstallNetwork', + ]) || + !exactKeys(acceptance, ['healthPath', 'executionTerminal']) || + !exactKeys(accessibility, [ + 'application', + 'projects', + 'queue', + 'fields', + 'controls', + 'prospectPattern', + 'feedbackRoles', + ]) || + value['schemaVersion'] !== 1 || + caseValue['id'] !== 'prospect-research-workspace-v1' || + caseValue['specification'] !== 'spec.md' || + !sha256HexValue(caseValue['specificationSha256']) || + caseValue['provider'] !== 'anthropic' || + caseValue['model'] !== 'claude-opus-4-8' || + caseValue['product'] !== 'prospect_research_workspace' || + caseValue['mode'] !== 'greenfield' || + caseValue['scope'] !== 'whole_application' || + caseValue['surface'] !== 'full_stack' || + repository['substrate'] !== 'empty_dir' || + repository['base'] !== 'fresh-empty-commit' || + budgets['elapsedMinutes'] !== 120 || + budgets['mechanicalInterventions'] !== 2 || + budgets['substantiveHumanInterventions'] !== 0 || + !command(delivery['test'], 'npm', ['test']) || + !command(delivery['build'], 'npm', ['run', 'build']) || + !command(delivery['start'], 'npm', ['start']) || + !exactStrings(delivery['environment'], PROSPECT_ENVIRONMENT) || + delivery['runtimeNetwork'] !== 'forbidden' || + delivery['dependencyInstallNetwork'] !== 'package-registry-only' || + acceptance['healthPath'] !== '/api/health' || + acceptance['executionTerminal'] !== 'promotion_prepared' || + !accessibleName(accessibility['application'], 'application', 'Prospect research workspace') || + !accessibleName(accessibility['projects'], 'heading', 'Research projects') || + !accessibleName(accessibility['queue'], 'region', 'Prospect queue') || + !exactAccessibleNames(accessibility['fields'], 'textbox', PROSPECT_FIELDS) || + !exactAccessibleNames(accessibility['controls'], 'button', PROSPECT_CONTROLS) || + !exactKeys(prospectPattern, ['role', 'namePattern']) || + prospectPattern['role'] !== 'button' || + prospectPattern['namePattern'] !== '^Prospect: .+ at .+$' || + !feedbackRoles(accessibility['feedbackRoles']) || + !exactStrings(value['rules'], PROSPECT_RULES) + ) { + invalid(); + } + return value as unknown as ProspectResearchWorkspaceExecutionCasePublicContract; +} + function parsePetriBrowserContract(value: Record): BrowserExecutionCasePublicContract { const caseValue = requiredRecord(value, 'case'); const repository = requiredRecord(caseValue, 'repository'); @@ -535,6 +709,18 @@ function accessibleNameArray(value: unknown): value is AccessibleNameContract[] return Array.isArray(value) && value.length > 0 && value.every((item) => accessibleName(item)); } +function exactAccessibleNames( + value: unknown, + role: string, + names: readonly string[], +): value is AccessibleNameContract[] { + return ( + Array.isArray(value) && + value.length === names.length && + value.every((item, index) => accessibleName(item, role, names[index])) + ); +} + function feedbackRoles(value: unknown): boolean { return ( Array.isArray(value) && @@ -575,6 +761,14 @@ function exactKeys(value: Record, expected: readonly string[]): return actual.length === wanted.length && actual.every((key, index) => key === wanted[index]); } +function exactStrings(value: unknown, expected: readonly string[]): boolean { + return ( + Array.isArray(value) && + value.length === expected.length && + value.every((item, index) => item === expected[index]) + ); +} + function sha256Hex(value: string): string { return createHash('sha256').update(value).digest('hex'); } diff --git a/src/dev/execution-comparison/historical-replay-target.ts b/src/dev/execution-comparison/historical-replay-target.ts index 07f8c6b69..81d5fef80 100644 --- a/src/dev/execution-comparison/historical-replay-target.ts +++ b/src/dev/execution-comparison/historical-replay-target.ts @@ -21,7 +21,7 @@ import { import { assertControllerIsolation } from '../end-to-end-comparison/study-contract.js'; import { seedBrownfieldBrunchExecutionWorkspace } from './brunch-lane.js'; import { - isBrowserExecutionCaseContract, + isPinnedExecutionCaseContract, type PinnedExecutionCaseId, type PublicCasePacket, } from './case-contract.js'; @@ -174,7 +174,7 @@ export async function prepareHistoricalReplayTarget( try { await validateInput(input); const contract = input.selectedCase.packet.contract; - if (isBrowserExecutionCaseContract(contract) || contract.case.repository.substrate !== 'pinned_git') { + if (!isPinnedExecutionCaseContract(contract)) { throw new Error('historical replay preparation requires a pinned brownfield case'); } const forbiddenRoots = uniqueRoots([ diff --git a/src/dev/execution-comparison/oracle-pack.ts b/src/dev/execution-comparison/oracle-pack.ts index be7ef6273..5e6c9e092 100644 --- a/src/dev/execution-comparison/oracle-pack.ts +++ b/src/dev/execution-comparison/oracle-pack.ts @@ -53,8 +53,30 @@ export interface PetrinautOptimizationControllerOracleManifest { readonly replacementRule: string; } +export interface ProspectResearchWorkspaceControllerOracleManifest { + readonly schemaVersion: 1; + readonly id: 'prospect-research-workspace-oracles-v1'; + readonly publicCaseId: 'prospect-research-workspace-v1'; + readonly runnerVersion: 'prospect-research-full-stack-v1'; + readonly fixtureVersion: 'prospect-research-fixtures-v1'; + readonly checks: readonly { + readonly id: + | 'startup-and-health' + | 'project-approval-and-research' + | 'qualification-and-deduplication' + | 'suppression-and-rerun' + | 'override-and-export' + | 'provider-failure' + | 'restart-persistence'; + readonly claims: readonly string[]; + }[]; + readonly validityRules: readonly string[]; + readonly replacementRule: string; +} + export type ControllerOracleManifest = | PetriControllerOracleManifest + | ProspectResearchWorkspaceControllerOracleManifest | BrunchHostLandingControllerOracleManifest | PetrinautOptimizationControllerOracleManifest; @@ -70,6 +92,12 @@ export function isPetrinautOptimizationControllerOracleManifest( return value.id === 'petrinaut-optimization-oracles-v1'; } +export function isProspectResearchWorkspaceControllerOracleManifest( + value: ControllerOracleManifest, +): value is ProspectResearchWorkspaceControllerOracleManifest { + return value.id === 'prospect-research-workspace-oracles-v1'; +} + export interface ControllerOraclePack { readonly manifest: ControllerOracleManifest; readonly files: readonly { @@ -130,6 +158,9 @@ export function parseControllerOracleManifest(value: unknown): ControllerOracleM if (value['id'] === 'petrinaut-optimization-oracles-v1') { return parsePetrinautOptimizationManifest(value); } + if (value['id'] === 'prospect-research-workspace-oracles-v1') { + return parseProspectResearchWorkspaceManifest(value); + } return parsePetriManifest(value); } @@ -224,6 +255,67 @@ const PETRINAUT_OPTIMIZATION_CHECKS = [ 'cancel-and-abort', 'private-origin-secrecy', ] as const; +const PROSPECT_RESEARCH_CHECKS = [ + 'startup-and-health', + 'project-approval-and-research', + 'qualification-and-deduplication', + 'suppression-and-rerun', + 'override-and-export', + 'provider-failure', + 'restart-persistence', +] as const; + +function parseProspectResearchWorkspaceManifest( + value: Record, +): ProspectResearchWorkspaceControllerOracleManifest { + const checks = value['checks']; + const rules = value['validityRules']; + if ( + !exactKeys(value, [ + 'schemaVersion', + 'id', + 'publicCaseId', + 'runnerVersion', + 'fixtureVersion', + 'checks', + 'validityRules', + 'replacementRule', + ]) || + value['schemaVersion'] !== 1 || + value['id'] !== 'prospect-research-workspace-oracles-v1' || + value['publicCaseId'] !== 'prospect-research-workspace-v1' || + value['runnerVersion'] !== 'prospect-research-full-stack-v1' || + value['fixtureVersion'] !== 'prospect-research-fixtures-v1' || + !Array.isArray(checks) || + checks.length !== PROSPECT_RESEARCH_CHECKS.length || + !checks.every( + (check, index) => + record(check) && + exactKeys(check, ['id', 'claims']) && + check['id'] === PROSPECT_RESEARCH_CHECKS[index] && + nonemptyStrings(check['claims']), + ) || + !nonemptyStrings(rules) || + !prospectValidityRules(rules as string[]) || + !nonempty(value['replacementRule']) + ) { + invalid(); + } + return value as unknown as ProspectResearchWorkspaceControllerOracleManifest; +} + +function prospectValidityRules(rules: readonly string[]): boolean { + const joined = rules.join('\n'); + return ( + joined.includes('Only spec.md and public-contract.json') && + joined.includes('Controller fixtures') && + joined.includes('package-registry-only') && + joined.includes('runtime network is denied') && + joined.includes('promotion_prepared') && + joined.includes('/brunch:land') && + joined.includes('landed') + ); +} function parsePetrinautOptimizationManifest( value: Record, diff --git a/src/dev/execution-comparison/prospect-research-workspace-oracle.ts b/src/dev/execution-comparison/prospect-research-workspace-oracle.ts new file mode 100644 index 000000000..8070e9595 --- /dev/null +++ b/src/dev/execution-comparison/prospect-research-workspace-oracle.ts @@ -0,0 +1,5 @@ +export { runProspectResearchWorkspaceOracle } from './prospect-research-workspace-oracle/runner.js'; +export type { + ProspectResearchWorkspaceOracleCheck, + ProspectResearchWorkspaceOracleReport, +} from './prospect-research-workspace-oracle/types.js'; diff --git a/src/dev/execution-comparison/prospect-research-workspace-oracle/journey-runner.ts b/src/dev/execution-comparison/prospect-research-workspace-oracle/journey-runner.ts new file mode 100644 index 000000000..de2fd9acb --- /dev/null +++ b/src/dev/execution-comparison/prospect-research-workspace-oracle/journey-runner.ts @@ -0,0 +1,65 @@ +export interface ManagedJourneyResult { + readonly status: 'passed' | 'setup_failed' | 'assertion_failed'; + readonly message: string; + readonly context?: Context; +} + +export async function runManagedProspectJourney(input: { + readonly open: () => Promise; + readonly setup: (context: Context) => Promise; + readonly assert: (context: Context) => Promise; + readonly close: (context: Context) => Promise; + readonly timeoutMs: number; +}): Promise> { + let context: Context | undefined; + let status: ManagedJourneyResult['status'] = 'passed'; + let message = 'all independent assertions passed'; + try { + context = await input.open(); + try { + await withTimeout(input.setup(context), input.timeoutMs, 'journey setup timed out'); + } catch (error) { + status = 'setup_failed'; + message = errorMessage(error); + } + if (status === 'passed') { + try { + await withTimeout(input.assert(context), input.timeoutMs, 'journey assertion timed out'); + } catch (error) { + status = 'assertion_failed'; + message = errorMessage(error); + } + } + } catch (error) { + status = 'setup_failed'; + message = errorMessage(error); + } finally { + if (context !== undefined) { + try { + await input.close(context); + } catch (error) { + status = 'setup_failed'; + message = `${message}; cleanup failed: ${errorMessage(error)}`; + } + } + } + return { status, message, ...(context === undefined ? {} : { context }) }; +} + +async function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/dev/execution-comparison/prospect-research-workspace-oracle/journeys.ts b/src/dev/execution-comparison/prospect-research-workspace-oracle/journeys.ts new file mode 100644 index 000000000..815724c80 --- /dev/null +++ b/src/dev/execution-comparison/prospect-research-workspace-oracle/journeys.ts @@ -0,0 +1,379 @@ +import { join } from 'node:path'; + +import type { Page } from 'playwright-core'; + +import type { ProspectResearchWorkspaceExecutionCasePublicContract } from '../case-contract.js'; +import type { ProspectResearchWorkspaceControllerOracleManifest } from '../oracle-pack.js'; +import type { ProspectJourneyEnvironment } from './lifecycle.js'; +import { expectedResearchState } from './reference.js'; +import { rows, snapshotSqlite } from './sqlite-evidence.js'; +import type { ProspectResearchCheckId } from './types.js'; + +interface AppState { + readonly projects: readonly { readonly id: number; readonly approved: boolean }[]; + readonly runs: readonly { readonly status: string; readonly error: string | null }[]; + readonly prospects: readonly { + readonly id: number; + readonly person: string; + readonly company: string; + readonly email: string; + readonly automatedStatus: string; + readonly currentStatus: string; + readonly suppressed: boolean; + readonly approved: boolean; + readonly sources: readonly string[]; + }[]; + readonly decisions: readonly { + readonly kind: string; + readonly previousStatus: string | null; + readonly nextStatus: string; + readonly reason: string | null; + }[]; +} + +type Journey = (environment: ProspectJourneyEnvironment) => Promise; + +export function prospectJourneyDefinitions(input: { + readonly caseDir: string; + readonly contract: ProspectResearchWorkspaceExecutionCasePublicContract; + readonly manifest: ProspectResearchWorkspaceControllerOracleManifest; +}): ReadonlyMap { + const researchFixture = join(input.caseDir, 'controller', 'fixtures', 'research-batch.json'); + return new Map([ + [ + 'startup-and-health', + async (environment) => { + const { page } = environment; + await requireCount( + page.getByRole('application', { name: input.contract.accessibility.application.name, exact: true }), + 1, + 'application shell', + ); + await requireCount( + page.getByRole('heading', { name: input.contract.accessibility.projects.name, exact: true }), + 1, + 'projects heading', + ); + await requireCount( + page.getByRole('region', { name: input.contract.accessibility.queue.name, exact: true }), + 1, + 'prospect queue', + ); + for (const field of input.contract.accessibility.fields) { + await requireCount( + page.getByRole(field.role as 'textbox', { name: field.name, exact: true }), + 1, + field.name, + ); + } + for (const control of input.contract.accessibility.controls) { + await requireCount( + page.getByRole(control.role as 'button', { name: control.name, exact: true }), + 1, + control.name, + ); + } + const state = await httpState(environment); + assert(state.projects.length === 0, 'fresh HTTP state contains projects'); + const sqlite = snapshotSqlite(environment.databasePath); + assert(rows(sqlite, 'projects').length === 0, 'fresh SQLite state contains projects'); + environment.evidence.push( + { source: 'browser', detail: 'accessible React shell and all declared controls are reachable' }, + { source: 'sqlite', detail: 'fresh durable store contains zero projects' }, + ); + assertNoExternalRequests(environment); + }, + ], + [ + 'project-approval-and-research', + async (environment) => { + await createProject(environment.page); + const unapproved = await action(environment.page, 'Run research', '/api/projects/', 'research'); + assert(unapproved.status === 409, `unapproved research returned ${unapproved.status}`); + let state = await httpState(environment); + assert(state.runs.length === 0, 'unapproved research created a run'); + await action(environment.page, 'Approve project', '/approve'); + await action(environment.page, 'Run research', '/research'); + state = await httpState(environment); + assert( + state.runs.length === 1 && state.runs[0]?.status === 'completed', + 'approved run not completed once', + ); + const sqlite = snapshotSqlite(environment.databasePath); + assert(rows(sqlite, 'runs').length === 1, 'SQLite did not retain exactly one run'); + environment.evidence.push( + { + source: 'browser', + detail: 'research was refused before approval and completed after explicit approval', + }, + { source: 'sqlite', detail: 'one completed manual run retained durably' }, + ); + assertNoExternalRequests(environment); + }, + ], + [ + 'qualification-and-deduplication', + async (environment) => { + await createApprovedResearch(environment.page); + const state = await httpState(environment); + const expected = await expectedResearchState(researchFixture); + assert( + state.prospects.length === expected.length, + 'HTTP prospect identities differ from reference model', + ); + for (const prospect of expected) { + const actual = state.prospects.find(({ email }) => email === prospect.email); + assert(actual !== undefined, `missing normalized prospect ${prospect.email}`); + assert( + actual.automatedStatus === prospect.automatedStatus, + `${prospect.email} qualification is not evidence-backed`, + ); + assert( + sameStrings(actual.sources, prospect.sources), + `${prospect.email} provenance was not retained`, + ); + } + const sqlite = snapshotSqlite(environment.databasePath); + const ari = rows(sqlite, 'prospects').filter(({ email }) => email === 'ari@arc.example'); + assert(ari.length === 1, 'SQLite contains duplicate Ari identities'); + assert( + rows(sqlite, 'provenance').filter(({ prospect_id }) => prospect_id === ari[0]?.['id']).length === 2, + 'SQLite discarded one Ari provenance row', + ); + environment.evidence.push( + { + source: 'http', + detail: 'state API agrees with the controller qualification/deduplication model', + }, + { source: 'sqlite', detail: 'one normalized Ari identity retains two independent provenance rows' }, + ); + assertNoExternalRequests(environment); + }, + ], + [ + 'suppression-and-rerun', + async (environment) => { + await createApprovedResearch(environment.page); + await selectProspect(environment.page, 'Sam Reed', 'Muted Systems'); + await environment.page + .getByRole('textbox', { name: 'Decision reason', exact: true }) + .fill('Existing customer'); + await action(environment.page, 'Suppress prospect', '/suppress'); + await action(environment.page, 'Run research', '/research'); + const state = await httpState(environment); + const sam = requiredProspect(state, 'sam@muted.example'); + assert(sam.suppressed, 'suppression did not dominate the later research run'); + const exported = await exportApproved(environment.page); + assert(!exported.some(({ email }) => email === sam.email), 'suppressed prospect entered export'); + const sqlite = snapshotSqlite(environment.databasePath); + assert(rows(sqlite, 'suppressions').length === 1, 'SQLite suppression history missing'); + assert(rows(sqlite, 'runs').length === 2, 'rerun was not retained independently'); + environment.evidence.push( + { source: 'browser', detail: 'selected prospect remained suppressed after a second manual run' }, + { source: 'sqlite', detail: 'suppression and both runs remain durable' }, + { source: 'export', detail: 'suppressed identity is absent from downloaded JSON' }, + ); + assertNoExternalRequests(environment); + }, + ], + [ + 'override-and-export', + async (environment) => { + await createApprovedResearch(environment.page); + await selectProspect(environment.page, 'Noor Vale', 'Arc Labs'); + const reasonless = await action(environment.page, 'Override qualification', '/override'); + assert(reasonless.status === 400, `reasonless override returned ${reasonless.status}`); + await environment.page + .getByRole('textbox', { name: 'Decision reason', exact: true }) + .fill('Role confirmed by operator'); + await action(environment.page, 'Override qualification', '/override'); + await selectProspect(environment.page, 'Ari Lane', 'Arc Labs'); + await action(environment.page, 'Approve prospect', '/approve'); + const state = await httpState(environment); + const noor = requiredProspect(state, 'noor@arc.example'); + assert( + noor.currentStatus === 'qualified' && !noor.approved, + 'reasoned override or approval boundary is wrong', + ); + const overrides = state.decisions.filter(({ kind }) => kind === 'override'); + assert(overrides.length === 1, 'reasonless override mutated audit history'); + assert( + overrides[0]?.previousStatus === 'needs_review' && + overrides[0].nextStatus === 'qualified' && + overrides[0].reason === 'Role confirmed by operator', + 'override did not preserve prior decision and reason', + ); + const exported = await exportApproved(environment.page); + assert( + exported.length === 1 && exported[0]?.email === 'ari@arc.example', + 'export contains prospects outside the explicitly approved non-suppressed subset', + ); + const sqlite = snapshotSqlite(environment.databasePath); + assert( + rows(sqlite, 'decisions').some( + ({ kind, previous_status, next_status, reason }) => + kind === 'override' && + previous_status === 'needs_review' && + next_status === 'qualified' && + reason === 'Role confirmed by operator', + ), + 'SQLite override audit is incomplete', + ); + environment.evidence.push( + { source: 'sqlite', detail: 'reasoned override retains previous and next automated decisions' }, + { source: 'export', detail: 'download contains only explicitly approved Ari Lane' }, + ); + assertNoExternalRequests(environment); + }, + ], + [ + 'provider-failure', + async (environment) => { + await createProject(environment.page); + await action(environment.page, 'Approve project', '/approve'); + const response = await action(environment.page, 'Run research', '/research'); + assert(response.status === 503, `provider failure returned ${response.status}`); + const state = await httpState(environment); + assert( + state.runs.length === 1 && state.runs[0]?.status === 'failed' && state.runs[0].error !== null, + 'provider failure was not retained honestly', + ); + assert(state.prospects.length === 0, 'provider failure created prospect decisions'); + const sqlite = snapshotSqlite(environment.databasePath); + assert(rows(sqlite, 'prospects').length === 0, 'provider failure corrupted prior prospect state'); + assert(rows(sqlite, 'decisions').length === 0, 'provider failure was laundered into rejection'); + environment.evidence.push( + { source: 'http', detail: 'provider failure remains a distinct 503 response and failed run' }, + { source: 'sqlite', detail: 'failure produced no prospect or rejection decision rows' }, + ); + assertNoExternalRequests(environment); + }, + ], + [ + 'restart-persistence', + async (environment) => { + await createApprovedResearch(environment.page); + await selectProspect(environment.page, 'Noor Vale', 'Arc Labs'); + await environment.page + .getByRole('textbox', { name: 'Decision reason', exact: true }) + .fill('Durable operator decision'); + await action(environment.page, 'Override qualification', '/override'); + await selectProspect(environment.page, 'Ari Lane', 'Arc Labs'); + await action(environment.page, 'Approve prospect', '/approve'); + const before = await httpState(environment); + const beforeExport = await exportApproved(environment.page); + await environment.restart(); + const after = await httpState(environment); + const afterExport = await exportApproved(environment.page); + assert(JSON.stringify(after) === JSON.stringify(before), 'restart changed durable API state'); + assert( + JSON.stringify(afterExport) === JSON.stringify(beforeExport), + 'restart changed export eligibility', + ); + const sqlite = snapshotSqlite(environment.databasePath); + assert(rows(sqlite, 'projects').length === 1, 'project missing after restart'); + assert( + rows(sqlite, 'decisions').some(({ kind }) => kind === 'override'), + 'override history missing after restart', + ); + environment.evidence.push( + { + source: 'browser', + detail: 'fresh browser context rehydrated the complete workspace after restart', + }, + { source: 'http', detail: 'pre/post restart state API bytes are equivalent' }, + { + source: 'sqlite', + detail: 'project, prospects, decisions, and approval remain in the same database', + }, + { source: 'export', detail: 'approved-only export is stable across restart' }, + ); + assertNoExternalRequests(environment); + }, + ], + ]); +} + +export function fixtureForCheck(caseDir: string, id: ProspectResearchCheckId): string { + const name = id === 'provider-failure' ? 'provider-failure.json' : 'research-batch.json'; + return join(caseDir, 'controller', 'fixtures', name); +} + +async function createProject(page: Page): Promise { + await page.getByRole('textbox', { name: 'Project name', exact: true }).fill('Founder-led B2B'); + await page + .getByRole('textbox', { name: 'Ideal customer profile', exact: true }) + .fill('B2B workflow software with a growth leader'); + await action(page, 'Create project', '/api/projects'); +} + +async function createApprovedResearch(page: Page): Promise { + await createProject(page); + await action(page, 'Approve project', '/approve'); + await action(page, 'Run research', '/research'); +} + +async function action( + page: Page, + buttonName: string, + ...urlParts: string[] +): Promise<{ readonly status: number }> { + const responsePromise = page.waitForResponse( + (response) => + response.request().method() === 'POST' && urlParts.every((part) => response.url().includes(part)), + ); + await page.getByRole('button', { name: buttonName, exact: true }).click(); + const response = await responsePromise; + return { status: response.status() }; +} + +async function selectProspect(page: Page, person: string, company: string): Promise { + await page.getByRole('button', { name: `Prospect: ${person} at ${company}`, exact: true }).click(); +} + +async function httpState(environment: ProspectJourneyEnvironment): Promise { + const response = await fetch(`${environment.origin}/api/state`); + assert(response.ok, `GET /api/state returned ${response.status}`); + environment.evidence.push({ source: 'http', detail: 'controller read same-origin /api/state directly' }); + return (await response.json()) as AppState; +} + +async function exportApproved(page: Page): Promise { + const downloadPromise = page.waitForEvent('download'); + await page.getByRole('button', { name: 'Export approved prospects', exact: true }).click(); + const download = await downloadPromise; + const stream = await download.createReadStream(); + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as { email: string }[]; +} + +function requiredProspect(state: AppState, email: string): AppState['prospects'][number] { + const prospect = state.prospects.find((candidate) => candidate.email === email); + if (prospect === undefined) throw new Error(`missing prospect ${email}`); + return prospect; +} + +function assertNoExternalRequests(environment: ProspectJourneyEnvironment): void { + assert( + environment.externalRuntimeRequests.length === 0, + `external runtime requests: ${environment.externalRuntimeRequests.join(', ')}`, + ); +} + +async function requireCount( + locator: ReturnType, + expected: number, + label: string, +): Promise { + const actual = await locator.count(); + assert(actual === expected, `${label}: expected ${expected}, received ${actual}`); +} + +function sameStrings(left: readonly string[], right: readonly string[]): boolean { + return JSON.stringify([...left].sort()) === JSON.stringify([...right].sort()); +} + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} diff --git a/src/dev/execution-comparison/prospect-research-workspace-oracle/lifecycle.ts b/src/dev/execution-comparison/prospect-research-workspace-oracle/lifecycle.ts new file mode 100644 index 000000000..39fba2c17 --- /dev/null +++ b/src/dev/execution-comparison/prospect-research-workspace-oracle/lifecycle.ts @@ -0,0 +1,241 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { access, copyFile, mkdtemp, rm } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { chromium, type Browser, type BrowserContext, type Page } from 'playwright-core'; + +import type { ProspectResearchWorkspaceOracleCheck } from './types.js'; + +const START_TIMEOUT_MS = 10_000; +const MAX_PROCESS_EVIDENCE_BYTES = 64 * 1024; + +export interface ProspectJourneyEnvironment { + readonly candidateRoot: string; + readonly databasePath: string; + readonly fixturePath: string; + readonly evidence: ProspectResearchWorkspaceOracleCheck['evidence'][number][]; + readonly externalRuntimeRequests: string[]; + readonly cleanup: { processStopped: boolean; browserClosed: boolean }; + origin: string; + page: Page; + restart: () => Promise; + close: () => Promise; +} + +export async function openProspectJourneyEnvironment(input: { + readonly candidateRoot: string; + readonly fixtureSource: string; +}): Promise { + const temporaryRoot = await mkdtemp(join(tmpdir(), 'brunch-prospect-oracle-')); + const databasePath = join(temporaryRoot, 'workspace.sqlite'); + const fixturePath = join(temporaryRoot, 'research-fixture.json'); + await copyFile(input.fixtureSource, fixturePath); + const evidence: ProspectResearchWorkspaceOracleCheck['evidence'][number][] = []; + const externalRuntimeRequests: string[] = []; + const cleanup = { processStopped: true, browserClosed: true }; + let candidate: ChildProcess | undefined; + let browser: Browser | undefined; + let context: BrowserContext | undefined; + let page: Page | undefined; + let origin = ''; + let closed = false; + + const stop = async (): Promise => { + if (context !== undefined) { + try { + await context.close(); + } finally { + context = undefined; + } + } + if (browser !== undefined) { + try { + await browser.close(); + } finally { + browser = undefined; + cleanup.browserClosed = true; + } + } + if (candidate !== undefined) { + cleanup.processStopped = await stopChild(candidate); + candidate = undefined; + } + }; + + const start = async (): Promise => { + const port = await availablePort(); + origin = `http://127.0.0.1:${port}`; + const processChunks: string[] = []; + cleanup.processStopped = false; + evidence.push({ + source: 'process', + detail: 'exact npm start launched with fresh PORT, DATABASE_PATH, and RESEARCH_FIXTURE_PATH', + }); + candidate = spawn('npm', ['start'], { + cwd: input.candidateRoot, + env: { + ...process.env, + PORT: String(port), + DATABASE_PATH: databasePath, + RESEARCH_FIXTURE_PATH: fixturePath, + }, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'pipe'], + }); + const retainProcessChunk = (chunk: Buffer): void => { + const retained = processChunks.join('').length; + if (retained < MAX_PROCESS_EVIDENCE_BYTES) { + processChunks.push(chunk.toString('utf8').slice(0, MAX_PROCESS_EVIDENCE_BYTES - retained)); + } + }; + candidate.stdout?.on('data', retainProcessChunk); + candidate.stderr?.on('data', retainProcessChunk); + await waitForHealth(`${origin}/api/health`, candidate, processChunks); + evidence.push({ source: 'http', detail: `GET /api/health returned ready on fresh PORT ${port}` }); + + browser = await chromium.launch({ executablePath: await resolveChromeExecutable(), headless: true }); + cleanup.browserClosed = false; + context = await browser.newContext({ acceptDownloads: true }); + await context.route('**/*', async (route) => { + const url = route.request().url(); + if (!url.startsWith('data:') && !url.startsWith('blob:') && new URL(url).origin !== origin) { + externalRuntimeRequests.push(url); + await route.abort('blockedbyclient'); + return; + } + await route.continue(); + }); + page = await context.newPage(); + page.setDefaultTimeout(5_000); + await page.goto(origin, { waitUntil: 'networkidle' }); + }; + + try { + await start(); + } catch (error) { + await stop(); + await rm(temporaryRoot, { recursive: true, force: true }); + throw error; + } + + const environment: ProspectJourneyEnvironment = { + candidateRoot: input.candidateRoot, + databasePath, + fixturePath, + evidence, + externalRuntimeRequests, + cleanup, + origin, + page: page!, + restart: async () => { + await stop(); + await start(); + environment.origin = origin; + environment.page = page!; + }, + close: async () => { + if (closed) return; + closed = true; + try { + await stop(); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } + }, + }; + return environment; +} + +async function waitForHealth(url: string, child: ChildProcess, output: readonly string[]): Promise { + const started = Date.now(); + while (Date.now() - started < START_TIMEOUT_MS) { + if (child.exitCode !== null) { + throw new Error(`npm start exited ${child.exitCode}: ${output.join('')}`); + } + try { + const response = await fetch(url, { signal: AbortSignal.timeout(500) }); + if (response.ok) { + const value = (await response.json()) as unknown; + if (record(value) && value['status'] === 'ready') return; + } + } catch { + // The exact public start command is still becoming ready. + } + await delay(50); + } + throw new Error(`npm start health timeout: ${output.join('')}`); +} + +async function stopChild(child: ChildProcess): Promise { + if (child.exitCode !== null) return true; + signalChildTree(child, 'SIGTERM'); + await Promise.race([onceExit(child), delay(2_000)]); + if (child.exitCode === null) { + signalChildTree(child, 'SIGKILL'); + await Promise.race([onceExit(child), delay(2_000)]); + } + return child.exitCode !== null || child.signalCode !== null; +} + +function signalChildTree(child: ChildProcess, signal: NodeJS.Signals): void { + if (process.platform !== 'win32' && child.pid !== undefined) { + try { + process.kill(-child.pid, signal); + return; + } catch { + // The detached process group may already be gone. + } + } + child.kill(signal); +} + +async function onceExit(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolve) => child.once('exit', () => resolve())); +} + +async function availablePort(): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('fresh PORT reservation failed'); + await new Promise((resolve, reject) => { + server.close((error) => (error === undefined ? resolve() : reject(error))); + }); + return address.port; +} + +async function resolveChromeExecutable(): Promise { + const candidates = [ + process.env['BRUNCH_CHROME_PATH'], + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/usr/bin/google-chrome', + '/usr/bin/google-chrome-stable', + '/usr/bin/chromium', + ].filter((candidate): candidate is string => candidate !== undefined && candidate.length > 0); + for (const candidate of candidates) { + try { + await access(candidate); + return candidate; + } catch { + // Continue through the closed platform candidates. + } + } + throw new Error('no Chrome executable found; set BRUNCH_CHROME_PATH'); +} + +function delay(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/dev/execution-comparison/prospect-research-workspace-oracle/reference.ts b/src/dev/execution-comparison/prospect-research-workspace-oracle/reference.ts new file mode 100644 index 000000000..68e3d8443 --- /dev/null +++ b/src/dev/execution-comparison/prospect-research-workspace-oracle/reference.ts @@ -0,0 +1,60 @@ +import { readFile } from 'node:fs/promises'; + +interface ResearchFixture { + readonly companies: readonly { + readonly id: string; + readonly name: string; + readonly domain: string; + readonly fitEvidence?: string; + }[]; + readonly prospects: readonly { + readonly source: string; + readonly person: string; + readonly email: string; + readonly role?: string; + readonly companyId: string; + readonly roleEvidence?: string; + readonly confidence: number; + }[]; +} + +export interface ExpectedProspect { + readonly person: string; + readonly company: string; + readonly email: string; + readonly automatedStatus: 'qualified' | 'needs_review'; + readonly sources: readonly string[]; +} + +export async function expectedResearchState(path: string): Promise { + const fixture = parseFixture(JSON.parse(await readFile(path, 'utf8')) as unknown); + const companies = new Map(fixture.companies.map((company) => [company.id, company])); + const prospects = new Map(); + for (const candidate of fixture.prospects) { + const company = companies.get(candidate.companyId); + if (company === undefined) throw new Error(`fixture references unknown company ${candidate.companyId}`); + const key = candidate.email.toLowerCase(); + const prior = prospects.get(key); + const automatedStatus = + candidate.role && candidate.roleEvidence && company.fitEvidence ? 'qualified' : 'needs_review'; + prospects.set(key, { + person: candidate.person, + company: company.name, + email: key, + automatedStatus, + sources: [...new Set([...(prior?.sources ?? []), candidate.source])].sort(), + }); + } + return [...prospects.values()].sort((left, right) => left.email.localeCompare(right.email)); +} + +function parseFixture(value: unknown): ResearchFixture { + if (!record(value) || !Array.isArray(value['companies']) || !Array.isArray(value['prospects'])) { + throw new Error('invalid deterministic research fixture'); + } + return value as unknown as ResearchFixture; +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} diff --git a/src/dev/execution-comparison/prospect-research-workspace-oracle/runner.ts b/src/dev/execution-comparison/prospect-research-workspace-oracle/runner.ts new file mode 100644 index 000000000..d32900da5 --- /dev/null +++ b/src/dev/execution-comparison/prospect-research-workspace-oracle/runner.ts @@ -0,0 +1,107 @@ +import { runCommand } from '../../../app/command-runner.js'; +import { isProspectResearchWorkspaceExecutionCaseContract, loadPublicCasePacket } from '../case-contract.js'; +import { + isProspectResearchWorkspaceControllerOracleManifest, + loadControllerOracleManifest, +} from '../oracle-pack.js'; +import { runManagedProspectJourney } from './journey-runner.js'; +import { fixtureForCheck, prospectJourneyDefinitions } from './journeys.js'; +import { openProspectJourneyEnvironment } from './lifecycle.js'; +import type { ProspectResearchWorkspaceOracleCheck, ProspectResearchWorkspaceOracleReport } from './types.js'; + +const COMMAND_TIMEOUT_MS = 10 * 60_000; +const JOURNEY_TIMEOUT_MS = 30_000; + +export async function runProspectResearchWorkspaceOracle(input: { + readonly candidateRoot: string; + readonly caseDir: string; +}): Promise { + const [packet, manifest] = await Promise.all([ + loadPublicCasePacket(input.caseDir), + loadControllerOracleManifest(input.caseDir), + ]); + if ( + !isProspectResearchWorkspaceExecutionCaseContract(packet.contract) || + !isProspectResearchWorkspaceControllerOracleManifest(manifest) + ) { + throw new Error('prospect research oracle received a different compiled case'); + } + + const commands: ProspectResearchWorkspaceOracleReport['commands'][number][] = []; + for (const step of [ + { id: 'test', command: 'npm', args: ['test'] }, + { id: 'build', command: 'npm', args: ['run', 'build'] }, + ] as const) { + const result = await runCommand(step.command, step.args, { + cwd: input.candidateRoot, + timeoutMs: COMMAND_TIMEOUT_MS, + maxOutputBytes: 256 * 1024, + }); + commands.push({ + ...step, + status: result.exitCode === 0 ? 'passed' : 'failed', + exitCode: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + }); + if (result.exitCode !== 0) { + return { + schemaVersion: 1, + caseId: packet.contract.case.id, + oracleId: manifest.id, + status: 'setup_failed', + commands, + checks: [], + externalRuntimeRequests: [], + setupFailure: `${step.command} ${step.args.join(' ')} exited ${result.exitCode}`, + }; + } + } + + const definitions = prospectJourneyDefinitions({ + caseDir: input.caseDir, + contract: packet.contract, + manifest, + }); + const checks: ProspectResearchWorkspaceOracleCheck[] = []; + for (const declared of manifest.checks) { + const definition = definitions.get(declared.id); + if (definition === undefined) throw new Error(`missing prospect journey implementation: ${declared.id}`); + const result = await runManagedProspectJourney({ + open: async () => + await openProspectJourneyEnvironment({ + candidateRoot: input.candidateRoot, + fixtureSource: fixtureForCheck(input.caseDir, declared.id), + }), + setup: async () => {}, + assert: definition, + close: async (environment) => await environment.close(), + timeoutMs: JOURNEY_TIMEOUT_MS, + }); + const environment = result.context; + checks.push({ + id: declared.id, + claims: declared.claims, + status: result.status, + message: result.message, + evidence: environment?.evidence ?? [], + externalRuntimeRequests: environment?.externalRuntimeRequests ?? [], + cleanup: environment?.cleanup ?? { processStopped: true, browserClosed: true }, + }); + } + + const externalRuntimeRequests = [...new Set(checks.flatMap((check) => check.externalRuntimeRequests))]; + return { + schemaVersion: 1, + caseId: packet.contract.case.id, + oracleId: manifest.id, + status: checks.some(({ status }) => status === 'setup_failed') + ? 'setup_failed' + : checks.some(({ status }) => status === 'assertion_failed') + ? 'assertion_failed' + : 'passed', + commands, + checks, + externalRuntimeRequests, + }; +} diff --git a/src/dev/execution-comparison/prospect-research-workspace-oracle/sqlite-evidence.ts b/src/dev/execution-comparison/prospect-research-workspace-oracle/sqlite-evidence.ts new file mode 100644 index 000000000..1ab46aae2 --- /dev/null +++ b/src/dev/execution-comparison/prospect-research-workspace-oracle/sqlite-evidence.ts @@ -0,0 +1,36 @@ +import Database from 'better-sqlite3'; + +export interface SqliteSnapshot { + readonly tables: Readonly[]>>; +} + +export function snapshotSqlite(path: string): SqliteSnapshot { + const database = new Database(path, { readonly: true, fileMustExist: true }); + try { + const names = database + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + .all() as { name: string }[]; + return { + tables: Object.fromEntries( + names.map(({ name }) => [ + name, + database.prepare(`SELECT * FROM ${quoteIdentifier(name)}`).all() as Record[], + ]), + ), + }; + } finally { + database.close(); + } +} + +export function rows(snapshot: SqliteSnapshot, table: string): readonly Record[] { + const selected = snapshot.tables[table]; + if (selected === undefined) throw new Error(`SQLite evidence missing table ${table}`); + return selected; +} + +function quoteIdentifier(value: string): string { + return `"${value.replaceAll('"', '""')}"`; +} diff --git a/src/dev/execution-comparison/prospect-research-workspace-oracle/types.ts b/src/dev/execution-comparison/prospect-research-workspace-oracle/types.ts new file mode 100644 index 000000000..17aa55ac5 --- /dev/null +++ b/src/dev/execution-comparison/prospect-research-workspace-oracle/types.ts @@ -0,0 +1,45 @@ +export type ProspectResearchCheckId = + | 'startup-and-health' + | 'project-approval-and-research' + | 'qualification-and-deduplication' + | 'suppression-and-rerun' + | 'override-and-export' + | 'provider-failure' + | 'restart-persistence'; + +export type ProspectEvidenceSource = 'browser' | 'http' | 'sqlite' | 'export' | 'process'; + +export interface ProspectResearchWorkspaceOracleCheck { + readonly id: ProspectResearchCheckId; + readonly claims: readonly string[]; + readonly status: 'passed' | 'setup_failed' | 'assertion_failed'; + readonly message: string; + readonly evidence: readonly { + readonly source: ProspectEvidenceSource; + readonly detail: string; + }[]; + readonly externalRuntimeRequests: readonly string[]; + readonly cleanup: { + readonly processStopped: boolean; + readonly browserClosed: boolean; + }; +} + +export interface ProspectResearchWorkspaceOracleReport { + readonly schemaVersion: 1; + readonly caseId: 'prospect-research-workspace-v1'; + readonly oracleId: 'prospect-research-workspace-oracles-v1'; + readonly status: 'passed' | 'setup_failed' | 'assertion_failed'; + readonly commands: readonly { + readonly id: 'test' | 'build'; + readonly command: 'npm'; + readonly args: readonly string[]; + readonly status: 'passed' | 'failed'; + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; + }[]; + readonly checks: readonly ProspectResearchWorkspaceOracleCheck[]; + readonly externalRuntimeRequests: readonly string[]; + readonly setupFailure?: string; +} diff --git a/testing/comparisons/missions/README.md b/testing/comparisons/missions/README.md index c882cdbc2..709b7a254 100644 --- a/testing/comparisons/missions/README.md +++ b/testing/comparisons/missions/README.md @@ -11,6 +11,8 @@ Current library: elicitation and end-to-end origin mission - [`brunch-host-landing.md`](brunch-host-landing.md) — Brunch host-landing brownfield mission - [`petrinaut-optimization.md`](petrinaut-optimization.md) — Petrinaut optimization brownfield mission +- [`prospect-research-workspace.md`](prospect-research-workspace.md) — full-stack prospect + research elicitation and end-to-end origin mission This `README.md` is the directory's reserved control file, not a mission. `/compare-specs` must exclude it from mission listing, resolution, revision, and creation. diff --git a/testing/comparisons/missions/prospect-research-workspace.md b/testing/comparisons/missions/prospect-research-workspace.md new file mode 100644 index 000000000..ac15cd53f --- /dev/null +++ b/testing/comparisons/missions/prospect-research-workspace.md @@ -0,0 +1,48 @@ +# Prospect research workspace + +## Objective and opening request + +The operator wants a review-ready product specification for a local full-stack workspace that turns an approved ideal-customer profile into a trustworthy prospect list. + +Open naturally with: + +> We want to build a prospect research workspace for a founder or growth operator. Help me work through the product and technical decisions and produce a review-ready specification for the full-stack application. + +## Product context and priorities + +The operator defines a research project, ideal-customer profile, target roles, qualification criteria, and exclusions. After approval, a manually started research run obtains candidate companies and people through Clay-compatible research and Pi-compatible qualification adapters. The product normalizes identities, detects duplicates, applies suppressions, records evidence-backed qualification decisions, and lets the operator review and export approved prospects. + +The priority is trust in the prospect queue. Every automated decision must remain explainable from retained evidence, and human corrections must remain auditable. + +## Constraints and known facts + +- The implementation is one npm repository with a React and TypeScript frontend, a Node.js and TypeScript backend, and SQLite persistence. +- Pi and Clay integrations sit behind server-side adapters. Comparison and local development use deterministic fixture adapters with no credentials or runtime network. +- Research runs are manually initiated. There is no scheduler or recurring campaign loop. +- The product does not generate outreach, sequence contacts, send messages, ingest mail, classify replies, synchronize a CRM, or autonomously hand prospects to an outreach system. +- A prospect is qualified only when required role, company-fit, and source-evidence fields are present. Provider confidence is not evidence. +- A matching person, company, domain, or email suppression takes precedence over qualification and export, including on later research runs. +- Deduplication retains merged source provenance rather than discarding the contributing records. +- An operator override requires a reason and retains the previous automated decision in audit history. +- Only explicitly operator-approved prospects may be exported. +- Provider failures and partial results remain distinguishable from rejected prospects. + +## Uncertainties + +The operator has not chosen the React build tool, Node server framework, ORM, router, component library, or CSS system. Detailed queue layout, filter presentation, duplicate-resolution interaction, export format, and the exact boundary between automatic qualification and `needs_review` should be settled through the specification conversation. + +When information is not present here and has not been decided in conversation, say that it is unknown or undecided rather than inventing a fact. + +## Decision latitude + +Ask about consequential product behavior, especially project approval, evidence sufficiency, duplicate handling, suppression, override authority, export, failure recovery, and restart persistence. Explain meaningful tradeoffs and recommend a coherent minimal direction. + +The operator may accept sensible implementation details within the fixed stack and may choose among well-explained recommendations. Do not weaken the known trust and safety constraints merely to simplify the build. Leave genuinely unresolved consequential choices explicit. + +## Conversational and disclosure posture + +Answer directly from the facts above, but do not volunteer a requirements dump at the start. Let consequential facts emerge in response to focused questions. Be decisive after a clear recommendation is established, candid about unknowns, and resistant to adding outreach or campaign-delivery features outside the prospect-research boundary. + +## Requested document + +Ask for the completed specification to be written as `prospect-research-workspace-spec.md`. It should give a product and implementation team a coherent account of the user workflow, fixed technical boundary, domain model, qualification and suppression semantics, accessible UI and API surface, failure behavior, and verification approach. It must be build-ready without prescribing libraries that remain deliberately open. diff --git a/testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json b/testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json new file mode 100644 index 000000000..525c9c3fb --- /dev/null +++ b/testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json @@ -0,0 +1,117 @@ +{ + "schemaVersion": 1, + "caseId": "prospect-research-workspace-v1", + "rows": [ + { + "id": "PR1", + "publicConcern": "Application startup", + "origin": "public_baseline", + "publicWording": "The production application starts through the declared command and exposes a ready health endpoint.", + "controller": { + "wording": "The Node service, React application, and SQLite store become ready without uncaught errors or external runtime requests.", + "revealPolicy": "The shared baseline discloses delivery, health, and runtime-network mechanics.", + "expectedState": "Health is successful, the accessible shell is present, and a fresh database has no projects.", + "fixtureRefs": [] + } + }, + { + "id": "PR2", + "publicConcern": "Approved manual research", + "origin": "public_baseline", + "publicWording": "An operator creates and approves a project before manually starting fixture-backed research.", + "controller": { + "wording": "Research is unavailable before approval and one manual run imports the deterministic fixture once.", + "revealPolicy": "The mission and shared baseline disclose project approval and manual run controls.", + "expectedState": "The unapproved rival cannot run; approval followed by research creates one completed run.", + "fixtureRefs": [ + "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json" + ] + } + }, + { + "id": "PR3", + "publicConcern": "Evidence-backed qualification", + "origin": "controller_only", + "controller": { + "wording": "Qualification requires role, company-fit, and source evidence; provider confidence alone is insufficient.", + "revealPolicy": "Reveal exact evidence sufficiency only after a qualifying qualification or trust question.", + "expectedState": "The complete prospect qualifies and the high-confidence evidence-missing prospect needs review.", + "fixtureRefs": [ + "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json" + ] + } + }, + { + "id": "PR4", + "publicConcern": "Identity and provenance", + "origin": "controller_only", + "controller": { + "wording": "Duplicate people and companies merge into stable identities while retaining every contributing source.", + "revealPolicy": "Reveal provenance retention only after a qualifying deduplication or identity question.", + "expectedState": "Two records for the same person produce one prospect with two source references.", + "fixtureRefs": [ + "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json" + ] + } + }, + { + "id": "PR5", + "publicConcern": "Suppression precedence", + "origin": "controller_only", + "controller": { + "wording": "Matching person, company, domain, or email suppression dominates qualification, approval, export, and later research runs.", + "revealPolicy": "Reveal precedence and persistence only after a qualifying exclusions or suppression question.", + "expectedState": "An otherwise qualified suppressed-domain prospect remains suppressed after the fixture is rerun and never exports.", + "fixtureRefs": [ + "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json" + ] + } + }, + { + "id": "PR6", + "publicConcern": "Audited operator override", + "origin": "controller_only", + "controller": { + "wording": "An override requires a reason and retains the previous automated decision in immutable audit history.", + "revealPolicy": "Reveal exact audit retention only after a qualifying override or accountability question.", + "expectedState": "A reasonless override fails; a reasoned override changes current status while preserving both decisions.", + "fixtureRefs": [] + } + }, + { + "id": "PR7", + "publicConcern": "Approved export", + "origin": "controller_only", + "controller": { + "wording": "Export contains only explicitly operator-approved, non-suppressed prospects.", + "revealPolicy": "Reveal export eligibility only after a qualifying downstream-use or approval question.", + "expectedState": "Qualified but unapproved, needs-review, rejected, and suppressed prospects are absent from JSON export.", + "fixtureRefs": [] + } + }, + { + "id": "PR8", + "publicConcern": "Provider failure honesty", + "origin": "controller_only", + "controller": { + "wording": "Provider failure and partial results remain distinct from prospect rejection and do not corrupt prior state.", + "revealPolicy": "Reveal failure classification only after a qualifying provider-error or recovery question.", + "expectedState": "The failure fixture records a failed run, creates no rejection decisions, and leaves prior prospects unchanged.", + "fixtureRefs": [ + "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/provider-failure.json" + ] + } + }, + { + "id": "PR9", + "publicConcern": "Restart persistence", + "origin": "controller_only", + "controller": { + "wording": "Projects, prospects, suppressions, decisions, approvals, and audit history survive application restart through SQLite.", + "revealPolicy": "Reveal exact durable scope only after a qualifying persistence or recovery question.", + "expectedState": "Restarting against the same database reproduces the complete prior state and export eligibility.", + "fixtureRefs": [] + } + } + ] +} diff --git a/testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md b/testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md new file mode 100644 index 000000000..4ae72d122 --- /dev/null +++ b/testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md @@ -0,0 +1,34 @@ +# Shared full-stack execution and interoperability baseline + +This baseline is visible to both elicitation targets before they begin. It controls delivery and mechanical addressability so the same black-box browser, HTTP, and SQLite journeys can exercise independently produced applications. Requirements copied from this baseline are not elicitation gains. + +## Delivery + +- Build one npm repository from a fresh empty Git repository. +- Use a React and TypeScript frontend, a Node.js and TypeScript backend, and SQLite persistence. +- `npm test` runs the implementation's own tests. +- `npm run build` produces the production frontend and backend. +- `npm start` starts the production application using `PORT`, `DATABASE_PATH`, and `RESEARCH_FIXTURE_PATH` from the environment. +- `GET /api/health` returns a successful JSON response only when the application and SQLite store are ready. +- Dependency installation may use the package registry. The running application may make no external network requests. +- Pi-compatible qualification and Clay-compatible research enter only through server-side adapters. The scored application uses the local fixture selected by `RESEARCH_FIXTURE_PATH`. + +## Accessible application surface + +- Expose one `application` named `Prospect research workspace`. +- Expose a heading named `Research projects` and a region named `Prospect queue`. +- Expose textboxes named `Project name`, `Ideal customer profile`, and `Decision reason`. +- Expose buttons named `Create project`, `Approve project`, `Run research`, `Approve prospect`, `Suppress prospect`, `Override qualification`, and `Export approved prospects`. +- Expose prospect items as buttons named `Prospect: at `. +- Operation outcomes and failures use a `status` or `alert` role. + +## Mechanical interaction vocabulary + +- Create a project by filling `Project name` and `Ideal customer profile`, then activating `Create project`. +- Select a project and activate `Approve project` before activating `Run research`. +- Select a prospect through its accessible prospect-item button. +- Prospect actions operate on the selected prospect. +- An override or suppression supplies `Decision reason` before activating its action. +- Activate `Export approved prospects` and capture the downloaded JSON document. + +This baseline does not settle qualification evidence, duplicate identity, suppression precedence, override history, export eligibility, provider-failure semantics, or restart behavior. Those remain specification and controller-oracle concerns. diff --git a/testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json b/testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json new file mode 100644 index 000000000..8b2033ebd --- /dev/null +++ b/testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json @@ -0,0 +1,48 @@ +{ + "schemaVersion": 1, + "id": "prospect-research-workspace-e2e-v1", + "caseId": "prospect-research-workspace-v1", + "mission": { + "path": "testing/comparisons/missions/prospect-research-workspace.md", + "sha256": "sha256:ee5ec5da6aefe79287a797fa18630f8b454ae53ef1d91a9503098be5eea11090" + }, + "sharedBaseline": { + "path": "testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md", + "sha256": "sha256:8efe626c9b275b56da7577715ed2dc07046d93d9a3e445840cac510824ac9b82" + }, + "requirementRegistry": { + "path": "testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json", + "sha256": "sha256:20460d405d422c12ce1faa9e954614044b1aef6a0ac9d1bdbe81ef61b54360ab" + }, + "executionContractTemplate": { + "path": "testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json", + "sha256": "sha256:6ce5985ef06809c1622d50564d0ca5a63b517c8386931303f4be435eaafacd7b" + }, + "oracle": { + "id": "prospect-research-workspace-oracles-v1", + "manifestPath": "testing/execution-comparisons/cases/prospect-research-workspace/controller/oracle-manifest.json", + "manifestSha256": "sha256:83398c66a41516ee9fc985724aa07dd88f4f3bae350670b80f1854970c92e900" + }, + "budgets": { + "elicitation": { + "qualifyingQuestions": 10, + "targetTurns": 20, + "elapsedMinutes": 60, + "mechanicalInterventions": 2 + }, + "execution": { + "elapsedMinutes": 120, + "mechanicalInterventions": 2, + "substantiveHumanInterventions": 0 + } + }, + "actorRecipes": { + "elicitation": "agent-as-user-comparison/v1", + "execution": { + "brunch": "brunch-empty-dir/v1", + "claude_code": "claude-code-empty-dir/v1" + } + }, + "specSources": ["brunch_spec", "claude_spec"], + "executors": ["brunch", "claude_code"] +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/provider-failure.json b/testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/provider-failure.json new file mode 100644 index 000000000..22d81b112 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/provider-failure.json @@ -0,0 +1,6 @@ +{ + "error": { + "code": "provider_unavailable", + "message": "Deterministic research provider failure" + } +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json b/testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json new file mode 100644 index 000000000..653835963 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json @@ -0,0 +1,53 @@ +{ + "companies": [ + { + "id": "co-arc", + "name": "Arc Labs", + "domain": "arc.example", + "fitEvidence": "B2B workflow software, 42 employees" + }, + { + "id": "co-suppressed", + "name": "Muted Systems", + "domain": "muted.example", + "fitEvidence": "B2B workflow software, 30 employees" + } + ], + "prospects": [ + { + "source": "clay-a", + "person": "Ari Lane", + "email": "ari@arc.example", + "role": "VP Growth", + "companyId": "co-arc", + "roleEvidence": "Company team page", + "confidence": 0.84 + }, + { + "source": "clay-b", + "person": "Ari Lane", + "email": "ARI@arc.example", + "role": "VP Growth", + "companyId": "co-arc", + "roleEvidence": "Professional profile", + "confidence": 0.92 + }, + { + "source": "clay-c", + "person": "Noor Vale", + "email": "noor@arc.example", + "role": "Growth leader", + "companyId": "co-arc", + "confidence": 0.99 + }, + { + "source": "clay-d", + "person": "Sam Reed", + "email": "sam@muted.example", + "role": "VP Growth", + "companyId": "co-suppressed", + "roleEvidence": "Company team page", + "confidence": 0.95 + } + ] +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/index.html b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/index.html new file mode 100644 index 000000000..388ac56a5 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/index.html @@ -0,0 +1,12 @@ + + + + + + Prospect research workspace + + +
+ + + diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/package.json b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/package.json new file mode 100644 index 000000000..433df99b5 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/package.json @@ -0,0 +1,18 @@ +{ + "name": "prospect-research-workspace-known-good", + "private": true, + "type": "module", + "scripts": { + "test": "node --test tests/*.node.js", + "build": "vite build && tsc -p tsconfig.server.json", + "start": "node dist/server/server.js" + }, + "dependencies": { + "@vitejs/plugin-react": "^6.0.3", + "better-sqlite3": "12.11.1", + "react": "^19.2.7", + "react-dom": "^19.2.7", + "typescript": "^6.0.3", + "vite": "^8.1.3" + } +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/behavior.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/behavior.ts new file mode 100644 index 000000000..356cf7ad8 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/behavior.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: false, + retainProvenance: true, + suppressionDominates: true, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: true, + honestProviderFailure: true, + durable: true, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/client.tsx b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/client.tsx new file mode 100644 index 000000000..3eb460982 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/client.tsx @@ -0,0 +1,157 @@ +/// + +import React, { useEffect, useState } from 'react'; +import { createRoot } from 'react-dom/client'; + +import './style.css'; + +interface State { + readonly projects: readonly { readonly id: number; readonly name: string; readonly approved: boolean }[]; + readonly runs: readonly { readonly id: number; readonly status: string }[]; + readonly prospects: readonly { + readonly id: number; + readonly person: string; + readonly company: string; + readonly email: string; + readonly currentStatus: string; + readonly suppressed: boolean; + readonly approved: boolean; + readonly sources: readonly string[]; + }[]; + readonly externalRuntimeRequest: boolean; +} + +const emptyState: State = { + projects: [], + runs: [], + prospects: [], + externalRuntimeRequest: false, +}; + +function Workspace() { + const [state, setState] = useState(emptyState); + const [projectName, setProjectName] = useState(''); + const [icp, setIcp] = useState(''); + const [reason, setReason] = useState(''); + const [selected, setSelected] = useState(null); + const [status, setStatus] = useState('Ready'); + const [alert, setAlert] = useState(''); + + const refresh = async () => { + const response = await fetch('/api/state'); + const next = (await response.json()) as State; + setState(next); + setSelected((current) => current ?? next.prospects[0]?.id ?? null); + if (next.externalRuntimeRequest) { + void fetch('https://runtime.invalid/prospect-oracle').catch(() => undefined); + } + }; + + useEffect(() => { + void refresh(); + }, []); + + const post = async (path: string, body: Record = {}) => { + const response = await fetch(path, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }); + const value = (await response.json()) as { error?: string }; + if (response.ok) { + setAlert(''); + setStatus('Saved'); + } else { + setAlert(value.error ?? `Request failed (${response.status})`); + } + await refresh(); + }; + + const projectId = state.projects[0]?.id; + const selectedProspect = state.prospects.find(({ id }) => id === selected); + + return ( +
+
+

Local research desk

+

Research projects

+

Approve the brief, run deterministic research, then review the evidence trail.

+
+ +
+ + +
+ + + +
+
+ +
+
+

Prospect queue

+

{state.runs.length} research runs

+
+ {state.prospects.map((prospect) => ( + + ))} +
+
+ +
+

{status}

+ {alert ?

{alert}

: null} +
+ ); +} + +createRoot(document.getElementById('root')!).render(); diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/server.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/server.ts new file mode 100644 index 000000000..cadbfccb0 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/server.ts @@ -0,0 +1,403 @@ +import { readFile } from 'node:fs/promises'; +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { extname, join } from 'node:path'; + +import Database from 'better-sqlite3'; + +import { behavior } from './behavior.js'; + +const port = Number(requiredEnvironment('PORT')); +const databasePath = behavior.durable ? requiredEnvironment('DATABASE_PATH') : ':memory:'; +const fixturePath = requiredEnvironment('RESEARCH_FIXTURE_PATH'); +const clientRoot = join(process.cwd(), 'dist', 'client'); +const database = new Database(databasePath); + +database.pragma('journal_mode = WAL'); +database.exec(` + CREATE TABLE IF NOT EXISTS projects ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + icp TEXT NOT NULL, + approved INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE IF NOT EXISTS runs ( + id INTEGER PRIMARY KEY, + project_id INTEGER NOT NULL, + status TEXT NOT NULL, + error TEXT + ); + CREATE TABLE IF NOT EXISTS companies ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + domain TEXT NOT NULL, + fit_evidence TEXT + ); + CREATE TABLE IF NOT EXISTS prospects ( + id INTEGER PRIMARY KEY, + project_id INTEGER NOT NULL, + person TEXT NOT NULL, + email TEXT NOT NULL, + role TEXT, + company_id TEXT NOT NULL, + role_evidence TEXT, + confidence REAL NOT NULL, + automated_status TEXT NOT NULL, + current_status TEXT NOT NULL, + suppressed INTEGER NOT NULL DEFAULT 0, + approved INTEGER NOT NULL DEFAULT 0, + UNIQUE(project_id, email) + ); + CREATE TABLE IF NOT EXISTS provenance ( + prospect_id INTEGER NOT NULL, + source TEXT NOT NULL, + role_evidence TEXT, + PRIMARY KEY (prospect_id, source) + ); + CREATE TABLE IF NOT EXISTS decisions ( + id INTEGER PRIMARY KEY, + prospect_id INTEGER, + kind TEXT NOT NULL, + previous_status TEXT, + next_status TEXT NOT NULL, + reason TEXT + ); + CREATE TABLE IF NOT EXISTS suppressions ( + id INTEGER PRIMARY KEY, + kind TEXT NOT NULL, + value TEXT NOT NULL, + reason TEXT NOT NULL + ); +`); + +const server = createServer(async (request, response) => { + try { + await route(request, response); + } catch (error) { + json(response, 500, { error: error instanceof Error ? error.message : String(error) }); + } +}); + +server.listen(port, '127.0.0.1', () => process.stdout.write(`ready:${port}\n`)); +for (const signal of ['SIGTERM', 'SIGINT'] as const) { + process.on(signal, () => { + server.close(() => { + database.close(); + process.exit(0); + }); + }); +} + +async function route(request: IncomingMessage, response: ServerResponse): Promise { + const method = request.method ?? 'GET'; + const url = new URL(request.url ?? '/', `http://127.0.0.1:${port}`); + if (method === 'GET' && url.pathname === '/api/health') { + json(response, 200, { status: 'ready' }); + return; + } + if (method === 'GET' && url.pathname === '/api/state') { + json(response, 200, state()); + return; + } + if (method === 'POST' && url.pathname === '/api/projects') { + const body = await jsonBody(request); + const result = database + .prepare('INSERT INTO projects (name, icp) VALUES (?, ?)') + .run(requiredString(body, 'name'), requiredString(body, 'icp')); + json(response, 201, { id: Number(result.lastInsertRowid) }); + return; + } + const projectAction = url.pathname.match(/^\/api\/projects\/(\d+)\/(approve|research)$/u); + if (method === 'POST' && projectAction) { + const projectId = Number(projectAction[1]); + if (projectAction[2] === 'approve') { + database.prepare('UPDATE projects SET approved = 1 WHERE id = ?').run(projectId); + json(response, 200, { approved: true }); + return; + } + await research(response, projectId); + return; + } + const prospectAction = url.pathname.match(/^\/api\/prospects\/(\d+)\/(approve|suppress|override)$/u); + if (method === 'POST' && prospectAction) { + await mutateProspect(response, Number(prospectAction[1]), prospectAction[2]!, await jsonBody(request)); + return; + } + if (method === 'GET' && url.pathname === '/api/export') { + const where = behavior.approvedOnlyExport ? 'WHERE p.approved = 1 AND p.suppressed = 0' : ''; + const exported = database + .prepare( + `SELECT p.person, p.email, p.role, c.name AS company + FROM prospects p JOIN companies c ON c.id = p.company_id ${where} ORDER BY p.email`, + ) + .all(); + response.writeHead(200, { + 'content-type': 'application/json; charset=utf-8', + 'content-disposition': 'attachment; filename="approved-prospects.json"', + }); + response.end(`${JSON.stringify(exported, null, 2)}\n`); + return; + } + await staticFile(url.pathname, response); +} + +async function research(response: ServerResponse, projectId: number): Promise { + const project = database.prepare('SELECT * FROM projects WHERE id = ?').get(projectId) as + | { approved: number } + | undefined; + if (project === undefined) { + json(response, 404, { error: 'project not found' }); + return; + } + if (!project.approved && !behavior.allowUnapprovedResearch) { + json(response, 409, { error: 'project approval required' }); + return; + } + const run = database.prepare("INSERT INTO runs (project_id, status) VALUES (?, 'running')").run(projectId); + const fixture = JSON.parse(await readFile(fixturePath, 'utf8')) as Record; + if (record(fixture['error'])) { + const rawMessage = fixture['error']['message']; + const message = typeof rawMessage === 'string' ? rawMessage : 'provider failure'; + if (behavior.honestProviderFailure) { + database + .prepare("UPDATE runs SET status = 'failed', error = ? WHERE id = ?") + .run(message, run.lastInsertRowid); + json(response, 503, { error: message }); + return; + } + database.prepare("UPDATE runs SET status = 'completed' WHERE id = ?").run(run.lastInsertRowid); + database + .prepare( + "INSERT INTO decisions (prospect_id, kind, previous_status, next_status, reason) VALUES (NULL, 'automated', NULL, 'rejected', 'provider unavailable')", + ) + .run(); + json(response, 200, { imported: 0 }); + return; + } + + const companies = requiredArray(fixture, 'companies'); + for (const company of companies) { + database + .prepare( + `INSERT INTO companies (id, name, domain, fit_evidence) VALUES (?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET name = excluded.name, domain = excluded.domain, + fit_evidence = excluded.fit_evidence`, + ) + .run(company['id'], company['name'], company['domain'], company['fitEvidence'] ?? null); + } + const companyById = new Map(companies.map((company) => [String(company['id']), company])); + for (const candidate of requiredArray(fixture, 'prospects')) { + const email = String(candidate['email']).toLowerCase(); + const company = companyById.get(String(candidate['companyId'])); + if (company === undefined) throw new Error('candidate references unknown company'); + const evidenceComplete = Boolean( + candidate['role'] && candidate['roleEvidence'] && company['fitEvidence'], + ); + const automatedStatus = + evidenceComplete || (behavior.confidenceOnlyQualification && Number(candidate['confidence']) >= 0.9) + ? 'qualified' + : 'needs_review'; + let prospect = database + .prepare('SELECT id, suppressed FROM prospects WHERE project_id = ? AND email = ?') + .get(projectId, email) as { id: number; suppressed: number } | undefined; + if (prospect === undefined) { + const inserted = database + .prepare( + `INSERT INTO prospects + (project_id, person, email, role, company_id, role_evidence, confidence, automated_status, current_status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + projectId, + candidate['person'], + email, + candidate['role'] ?? null, + candidate['companyId'], + candidate['roleEvidence'] ?? null, + candidate['confidence'], + automatedStatus, + automatedStatus, + ); + prospect = { id: Number(inserted.lastInsertRowid), suppressed: 0 }; + database + .prepare( + "INSERT INTO decisions (prospect_id, kind, previous_status, next_status) VALUES (?, 'automated', NULL, ?)", + ) + .run(prospect.id, automatedStatus); + } else { + database + .prepare( + `UPDATE prospects SET person = ?, role = ?, company_id = ?, role_evidence = ?, confidence = ?, + automated_status = ?, current_status = ?, suppressed = ? WHERE id = ?`, + ) + .run( + candidate['person'], + candidate['role'] ?? null, + candidate['companyId'], + candidate['roleEvidence'] ?? null, + candidate['confidence'], + automatedStatus, + automatedStatus, + behavior.suppressionDominates ? prospect.suppressed : 0, + prospect.id, + ); + } + if (behavior.retainProvenance) { + database + .prepare('INSERT OR IGNORE INTO provenance (prospect_id, source, role_evidence) VALUES (?, ?, ?)') + .run(prospect.id, candidate['source'], candidate['roleEvidence'] ?? null); + } + } + database.prepare("UPDATE runs SET status = 'completed' WHERE id = ?").run(run.lastInsertRowid); + json(response, 200, { imported: requiredArray(fixture, 'prospects').length }); +} + +async function mutateProspect( + response: ServerResponse, + prospectId: number, + action: string, + body: Record, +): Promise { + if (action === 'approve') { + database.prepare('UPDATE prospects SET approved = 1 WHERE id = ?').run(prospectId); + json(response, 200, { approved: true }); + return; + } + const prospect = database.prepare('SELECT * FROM prospects WHERE id = ?').get(prospectId) as + | { email: string; current_status: string } + | undefined; + if (prospect === undefined) { + json(response, 404, { error: 'prospect not found' }); + return; + } + const reason = typeof body['reason'] === 'string' ? body['reason'].trim() : ''; + if (action === 'suppress') { + if (!reason) { + json(response, 400, { error: 'suppression reason required' }); + return; + } + database.prepare('UPDATE prospects SET suppressed = 1 WHERE id = ?').run(prospectId); + database + .prepare("INSERT INTO suppressions (kind, value, reason) VALUES ('email', ?, ?)") + .run(prospect.email, reason); + json(response, 200, { suppressed: true }); + return; + } + if (!reason && behavior.reasonRequired) { + json(response, 400, { error: 'override reason required' }); + return; + } + database.prepare("UPDATE prospects SET current_status = 'qualified' WHERE id = ?").run(prospectId); + if (!behavior.preserveOverrideHistory) { + database.prepare('DELETE FROM decisions WHERE prospect_id = ?').run(prospectId); + } + database + .prepare( + "INSERT INTO decisions (prospect_id, kind, previous_status, next_status, reason) VALUES (?, 'override', ?, 'qualified', ?)", + ) + .run(prospectId, behavior.preserveOverrideHistory ? prospect.current_status : null, reason || null); + json(response, 200, { status: 'qualified' }); +} + +function state() { + const projects = database + .prepare('SELECT id, name, icp, approved FROM projects ORDER BY id') + .all() + .map((project) => ({ + ...(project as Record), + approved: Boolean((project as { approved: number }).approved), + })); + const runs = database + .prepare('SELECT id, project_id AS projectId, status, error FROM runs ORDER BY id') + .all(); + const prospects = database + .prepare( + `SELECT p.id, p.person, p.email, p.role, c.name AS company, + p.automated_status AS automatedStatus, p.current_status AS currentStatus, + p.suppressed, p.approved + FROM prospects p JOIN companies c ON c.id = p.company_id ORDER BY p.email`, + ) + .all() + .map((value) => { + const prospect = value as Record & { + id: number; + suppressed: number; + approved: number; + }; + return { + ...prospect, + suppressed: Boolean(prospect.suppressed), + approved: Boolean(prospect.approved), + sources: ( + database + .prepare('SELECT source FROM provenance WHERE prospect_id = ? ORDER BY source') + .all(prospect.id) as { + source: string; + }[] + ).map(({ source }) => source), + }; + }); + const decisions = database + .prepare( + 'SELECT kind, previous_status AS previousStatus, next_status AS nextStatus, reason FROM decisions ORDER BY id', + ) + .all(); + return { projects, runs, prospects, decisions, externalRuntimeRequest: behavior.externalRuntimeRequest }; +} + +async function staticFile(pathname: string, response: ServerResponse): Promise { + const relative = pathname === '/' ? 'index.html' : pathname.replace(/^\/+/, ''); + if (relative.includes('..')) { + response.writeHead(403).end('Forbidden'); + return; + } + try { + const path = join(clientRoot, relative); + const body = await readFile(path); + response.writeHead(200, { 'content-type': contentType(extname(path)) }).end(body); + } catch { + const body = await readFile(join(clientRoot, 'index.html')); + response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }).end(body); + } +} + +async function jsonBody(request: IncomingMessage): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + const value = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}') as unknown; + if (!record(value)) throw new Error('JSON body must be an object'); + return value; +} + +function json(response: ServerResponse, status: number, value: unknown): void { + response.writeHead(status, { 'content-type': 'application/json; charset=utf-8' }); + response.end(JSON.stringify(value)); +} + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function requiredString(value: Record, key: string): string { + const selected = value[key]; + if (typeof selected !== 'string' || !selected.trim()) throw new Error(`${key} is required`); + return selected.trim(); +} + +function requiredArray(value: Record, key: string): Record[] { + const selected = value[key]; + if (!Array.isArray(selected) || !selected.every(record)) throw new Error(`${key} must be an array`); + return selected; +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function contentType(extension: string): string { + if (extension === '.html') return 'text/html; charset=utf-8'; + if (extension === '.js') return 'text/javascript; charset=utf-8'; + if (extension === '.css') return 'text/css; charset=utf-8'; + return 'application/octet-stream'; +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/style.css b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/style.css new file mode 100644 index 000000000..1ddd809ff --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/src/style.css @@ -0,0 +1,99 @@ +:root { + font-family: Inter, ui-sans-serif, system-ui, sans-serif; + color: #18211d; + background: #edf0e8; +} + +body { + margin: 0; +} + +main { + width: min(1100px, calc(100% - 48px)); + margin: 0 auto; + padding: 48px 0; +} + +header { + border-bottom: 1px solid #abb5aa; + margin-bottom: 24px; +} + +.eyebrow { + color: #356349; + font-size: 12px; + font-weight: 700; + letter-spacing: 0.14em; + text-transform: uppercase; +} + +.project-card, +.queue { + display: grid; + gap: 18px; + padding: 22px; + border: 1px solid #abb5aa; + background: #f8faf5; +} + +.project-card { + grid-template-columns: 1fr 2fr; +} + +.queue { + grid-template-columns: 2fr 1fr; + margin-top: 18px; +} + +label, +.prospects, +.decision-actions { + display: grid; + gap: 8px; +} + +input, +button { + min-height: 38px; + border: 1px solid #7d8b80; + border-radius: 3px; + background: #fff; + color: inherit; +} + +button { + padding: 8px 12px; + text-align: left; + cursor: pointer; +} + +.actions { + grid-column: 1 / -1; + display: flex; + gap: 8px; +} + +.prospects button { + display: grid; + gap: 4px; +} + +.prospects .selected { + border-color: #195b3c; + background: #e0ede3; +} + +small { + color: #5c675f; +} + +@media (max-width: 700px) { + .project-card, + .queue { + grid-template-columns: 1fr; + } + + .actions { + flex-direction: column; + } +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/tests/behavior.node.js b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/tests/behavior.node.js new file mode 100644 index 000000000..381972052 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/tests/behavior.node.js @@ -0,0 +1,20 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { behavior } from '../src/behavior.ts'; + +void test('fixture behavior defines every scored switch explicitly', () => { + assert.deepEqual(Object.keys(behavior).sort(), [ + 'allowUnapprovedResearch', + 'approvedOnlyExport', + 'confidenceOnlyQualification', + 'durable', + 'externalRuntimeRequest', + 'honestProviderFailure', + 'preserveOverrideHistory', + 'reasonRequired', + 'retainProvenance', + 'suppressionDominates', + ]); + assert.ok(Object.values(behavior).every((value) => typeof value === 'boolean')); +}); diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/tsconfig.server.json b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/tsconfig.server.json new file mode 100644 index 000000000..08852da5f --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/tsconfig.server.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": "src", + "outDir": "dist/server", + "strict": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src/server.ts", "src/behavior.ts"] +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/vite.config.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/vite.config.ts new file mode 100644 index 000000000..66c6351df --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/known-good/vite.config.ts @@ -0,0 +1,7 @@ +import react from '@vitejs/plugin-react'; +import { defineConfig } from 'vite'; + +export default defineConfig({ + plugins: [react()], + build: { outDir: 'dist/client' }, +}); diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/oracle-manifest.json b/testing/execution-comparisons/cases/prospect-research-workspace/controller/oracle-manifest.json new file mode 100644 index 000000000..65d390ad1 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/oracle-manifest.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": 1, + "id": "prospect-research-workspace-oracles-v1", + "publicCaseId": "prospect-research-workspace-v1", + "runnerVersion": "prospect-research-full-stack-v1", + "fixtureVersion": "prospect-research-fixtures-v1", + "checks": [ + { "id": "startup-and-health", "claims": ["PR1"] }, + { "id": "project-approval-and-research", "claims": ["PR2"] }, + { "id": "qualification-and-deduplication", "claims": ["PR3", "PR4"] }, + { "id": "suppression-and-rerun", "claims": ["PR5"] }, + { "id": "override-and-export", "claims": ["PR6", "PR7"] }, + { "id": "provider-failure", "claims": ["PR8"] }, + { "id": "restart-persistence", "claims": ["PR9"] } + ], + "validityRules": [ + "The target repository starts at the frozen empty base commit.", + "Only spec.md and public-contract.json enter the target repository before launch.", + "Controller fixtures, expected states, and oracle implementation never enter the target repository or prompt.", + "Dependency installation is package-registry-only and candidate runtime network is denied.", + "The candidate receives fixture paths only when the controller starts the scored application after promotion_prepared.", + "No substantive human implementation advice or content is supplied.", + "A Brunch run stops at promotion_prepared and never invokes /brunch:land or records landed." + ], + "replacementRule": "Retain every attempt. Replace only provider, adapter, or mechanical invalidity under the same frozen packet; never replace poor output." +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/confidence-only-qualification.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/confidence-only-qualification.ts new file mode 100644 index 000000000..ff32b3f8f --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/confidence-only-qualification.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: true, + retainProvenance: true, + suppressionDominates: true, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: true, + honestProviderFailure: true, + durable: true, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/destructive-reasonless-override.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/destructive-reasonless-override.ts new file mode 100644 index 000000000..28468dc1e --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/destructive-reasonless-override.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: false, + retainProvenance: true, + suppressionDominates: true, + reasonRequired: false, + preserveOverrideHistory: false, + approvedOnlyExport: true, + honestProviderFailure: true, + durable: true, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/discarded-provenance.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/discarded-provenance.ts new file mode 100644 index 000000000..00c5c2471 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/discarded-provenance.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: false, + retainProvenance: false, + suppressionDominates: true, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: true, + honestProviderFailure: true, + durable: true, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/external-runtime-request.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/external-runtime-request.ts new file mode 100644 index 000000000..4a5a0ae9d --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/external-runtime-request.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: false, + retainProvenance: true, + suppressionDominates: true, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: true, + honestProviderFailure: true, + durable: true, + externalRuntimeRequest: true, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/in-memory-only-state.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/in-memory-only-state.ts new file mode 100644 index 000000000..ab8d9b4e9 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/in-memory-only-state.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: false, + retainProvenance: true, + suppressionDominates: true, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: true, + honestProviderFailure: true, + durable: false, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/non-dominant-suppression.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/non-dominant-suppression.ts new file mode 100644 index 000000000..139c29526 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/non-dominant-suppression.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: false, + retainProvenance: true, + suppressionDominates: false, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: true, + honestProviderFailure: true, + durable: true, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/overbroad-export.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/overbroad-export.ts new file mode 100644 index 000000000..86af4b946 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/overbroad-export.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: false, + retainProvenance: true, + suppressionDominates: true, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: false, + honestProviderFailure: true, + durable: true, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/provider-failure-laundering.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/provider-failure-laundering.ts new file mode 100644 index 000000000..9760150ec --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/provider-failure-laundering.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: false, + confidenceOnlyQualification: false, + retainProvenance: true, + suppressionDominates: true, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: true, + honestProviderFailure: false, + durable: true, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/unapproved-research.ts b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/unapproved-research.ts new file mode 100644 index 000000000..10578c7ae --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/controller/rivals/unapproved-research.ts @@ -0,0 +1,12 @@ +export const behavior = { + allowUnapprovedResearch: true, + confidenceOnlyQualification: false, + retainProvenance: true, + suppressionDominates: true, + reasonRequired: true, + preserveOverrideHistory: true, + approvedOnlyExport: true, + honestProviderFailure: true, + durable: true, + externalRuntimeRequest: false, +}; diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json b/testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json new file mode 100644 index 000000000..e49e82bc5 --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json @@ -0,0 +1,63 @@ +{ + "schemaVersion": 1, + "case": { + "id": "prospect-research-workspace-v1", + "specification": "spec.md", + "specificationSha256": "c31b9857949e5f15fb6c490b6d9127356e90c5c8bb20956a9e2080da97bf4547", + "provider": "anthropic", + "model": "claude-opus-4-8", + "product": "prospect_research_workspace", + "mode": "greenfield", + "scope": "whole_application", + "surface": "full_stack", + "repository": { + "substrate": "empty_dir", + "base": "fresh-empty-commit" + } + }, + "budgets": { + "elapsedMinutes": 120, + "mechanicalInterventions": 2, + "substantiveHumanInterventions": 0 + }, + "delivery": { + "test": { "command": "npm", "args": ["test"] }, + "build": { "command": "npm", "args": ["run", "build"] }, + "start": { "command": "npm", "args": ["start"] }, + "environment": ["PORT", "DATABASE_PATH", "RESEARCH_FIXTURE_PATH"], + "runtimeNetwork": "forbidden", + "dependencyInstallNetwork": "package-registry-only" + }, + "acceptance": { + "healthPath": "/api/health", + "executionTerminal": "promotion_prepared" + }, + "accessibility": { + "application": { "role": "application", "name": "Prospect research workspace" }, + "projects": { "role": "heading", "name": "Research projects" }, + "queue": { "role": "region", "name": "Prospect queue" }, + "fields": [ + { "role": "textbox", "name": "Project name" }, + { "role": "textbox", "name": "Ideal customer profile" }, + { "role": "textbox", "name": "Decision reason" } + ], + "controls": [ + { "role": "button", "name": "Create project" }, + { "role": "button", "name": "Approve project" }, + { "role": "button", "name": "Run research" }, + { "role": "button", "name": "Approve prospect" }, + { "role": "button", "name": "Suppress prospect" }, + { "role": "button", "name": "Override qualification" }, + { "role": "button", "name": "Export approved prospects" } + ], + "prospectPattern": { "role": "button", "namePattern": "^Prospect: .+ at .+$" }, + "feedbackRoles": ["status", "alert"] + }, + "rules": [ + "Work only in the target repository.", + "Do not inspect controller paths or seek hidden comparison material.", + "Use only the fixture-backed server adapters selected by the environment.", + "Do not add sequencing, sending, mailbox, reply-classification, scheduler, or CRM behavior.", + "Stop at promotion_prepared after npm test and npm run build pass." + ] +} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/spec.md b/testing/execution-comparisons/cases/prospect-research-workspace/spec.md new file mode 100644 index 000000000..07ec7210c --- /dev/null +++ b/testing/execution-comparisons/cases/prospect-research-workspace/spec.md @@ -0,0 +1,35 @@ +# Prospect Research Workspace + +## Product Summary + +Build a local full-stack workspace that turns an approved ideal-customer profile into a reviewable, evidence-backed prospect list. A founder or growth operator manually starts research, reviews qualification decisions, applies suppressions or reasoned overrides, approves selected prospects, and exports only that approved subset. + +## Product Boundary + +The application researches and qualifies prospects. It does not generate outreach, sequence contacts, send messages, ingest mail, classify replies, schedule recurring runs, synchronize a CRM, or autonomously hand prospects to another system. + +## Core Workflow + +1. The operator creates a research project with an ideal-customer profile, target roles, qualification criteria, and exclusions. +2. The operator approves the project before research can run. +3. A manual research run reads deterministic candidates from the configured Clay-compatible adapter. +4. The application normalizes people and companies, merges duplicates while retaining provenance, and applies suppressions. +5. The Pi-compatible qualification adapter assigns `qualified`, `needs_review`, or `rejected` from criterion-level evidence. +6. The operator reviews evidence and audit history, may suppress or override with a reason, and explicitly approves prospects. +7. Export contains only explicitly approved, non-suppressed prospects. + +## Qualification And Safety + +- Required role, company-fit, and source-evidence fields must be present before qualification. +- Provider confidence is metadata, not qualifying evidence. +- Person, company, domain, and email suppressions take precedence over qualification, approval, and later imports. +- Overrides preserve the prior automated decision and record the operator's reason. +- Provider failure or partial delivery never silently becomes prospect rejection. + +## Technical Boundary + +Use a React and TypeScript frontend, Node.js and TypeScript backend, SQLite durable store, and server-side Pi/Clay-compatible adapter interfaces. The scored mode uses deterministic local fixtures and performs no runtime network access. Build tool, server framework, ORM, router, component library, and CSS system remain implementation choices. + +## Success + +The prototype succeeds when the operator can approve one project, run fixture-backed research, inspect deduplicated and suppressed prospects, understand evidence-backed decisions, apply an audited override, approve a subset, export only that subset, observe provider failures honestly, and restart without losing durable state. From 8eb356d012a9174f06fefbd28b75ff35fd6f2e6b Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 22 Jul 2026 23:50:15 +0200 Subject: [PATCH 2/3] FE-1253: Add release note --- .changeset/tricky-poems-do.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.changeset/tricky-poems-do.md b/.changeset/tricky-poems-do.md index a845151cc..1e6a736cc 100644 --- a/.changeset/tricky-poems-do.md +++ b/.changeset/tricky-poems-do.md @@ -1,2 +1,5 @@ --- +"@hashintel/brunch": patch --- + +Add a full-stack prospect research end-to-end comparison case ([#369](https://github.com/hashintel/brunch/pull/369)). From c0c3e47e32ed1676fac9ad64a774771be12ed5b5 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 23 Jul 2026 02:39:36 +0200 Subject: [PATCH 3/3] FE-1253: Thin prospect research comparison --- .changeset/tricky-poems-do.md | 2 +- docs/praxis/comparison-runs.md | 13 +- memory/PLAN.md | 46 +++---- memory/SPEC.md | 7 +- src/dev/TOPOLOGY.md | 6 +- .../__tests__/case-profile.test.ts | 29 ++--- .../__tests__/study-contract.test.ts | 21 ---- .../end-to-end-comparison/study-contract.ts | 4 - .../__tests__/oracle-pack.test.ts | 40 ++---- src/dev/execution-comparison/case-contract.ts | 49 +++++++- src/dev/execution-comparison/oracle-pack.ts | 5 +- testing/comparisons/missions/README.md | 4 +- .../controller/requirement-registry.json | 117 ------------------ .../shared-baseline.md | 34 ----- .../study-contract.json | 48 ------- .../public-contract.json | 13 +- .../cases/prospect-research-workspace/spec.md | 8 +- 17 files changed, 128 insertions(+), 318 deletions(-) delete mode 100644 testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json delete mode 100644 testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md delete mode 100644 testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json diff --git a/.changeset/tricky-poems-do.md b/.changeset/tricky-poems-do.md index 1e6a736cc..4c8741b48 100644 --- a/.changeset/tricky-poems-do.md +++ b/.changeset/tricky-poems-do.md @@ -2,4 +2,4 @@ "@hashintel/brunch": patch --- -Add a full-stack prospect research end-to-end comparison case ([#369](https://github.com/hashintel/brunch/pull/369)). +Add a deterministic full-stack prospect research regression case ([#369](https://github.com/hashintel/brunch/pull/369)). diff --git a/docs/praxis/comparison-runs.md b/docs/praxis/comparison-runs.md index 12dc7516c..3a9708b0b 100644 --- a/docs/praxis/comparison-runs.md +++ b/docs/praxis/comparison-runs.md @@ -129,13 +129,12 @@ audience-safe requirement ledger, and the bounded validity-first report. This historical witness predates Alpha 10 provenance capture. Keep it as evidence and an example, but do not backfill `provenance.json` or publish it through the current publication skill. -FE-1253 adds `prospect-research-workspace` as a separate second greenfield case. Its fixed public -stack is React + Node.js + TypeScript + SQLite, with deterministic server-side Pi/Clay-compatible -fixtures and denied runtime network. The mission path is -`testing/comparisons/missions/prospect-research-workspace.md`. Its compiled full-stack -browser/HTTP/SQLite/export oracle is calibrated against a known-good implementation and focused wrong -rivals. No elicitation or execution provider turn is valid until strict greenfield Claude isolation is -present and witnessed on the parent stack. +FE-1253 adds `prospect-research-workspace` as a deterministic full-stack regression case, not a second +end-to-end campaign profile. Its fixed public stack is React + Node.js + TypeScript + SQLite, with +server-side Pi/Clay-compatible fixtures and denied runtime network. The saved mission remains available +for exploratory specification work, while the compiled browser/HTTP/SQLite/export oracle is calibrated +against a known-good implementation and focused wrong rivals. A future campaign may compose this case +only through a separately authorized study contract. ## Agent recipe: drive, then join evidence diff --git a/memory/PLAN.md b/memory/PLAN.md index 23107964d..9316c09e2 100644 --- a/memory/PLAN.md +++ b/memory/PLAN.md @@ -21,7 +21,7 @@ Brunch-next has delivered the original composition spine: the host, sealed Pi pr **Parallel lanes.** Group 3 agent-layer work is pickup-ready. `agent-control-plane-closure` is one earned coverage frontier over foreground and background prompt ingress: it absorbs the former `graph-assurance-conduct` and `subagent-skill-access` items, reconciles the stale eager-context fossils left by D58-L's load-on-demand cutover, and closes prompt replacement/resource-invocation contradictions before capture experiments amplify them. FE-1187 retains only the concern-grouped review renderer. FE-1208's reshaped `automation-observability-dx` closure landed on the restacked base; its invalid warrant pilots remain historical evidence, while the useful evaluator/report primitives are available to `capture-ledger-tracer`. Two evaluation use cases stay deliberately distinct: **seed-based intra-product testing** (Brunch on its own terms — seeds encode Brunch graph/spec state only Brunch understands) and **mission-driven agent-as-user cross-product comparison** (`agent-as-user-comparison` — competitor CLIs share no Brunch state, so the comparable artifact is the "ready" spec/plan document a mission produces). FE-1210 closed the rigorous technical tracer on 2026-07-17; the separately planned operator workflow (`operator-comparison-workflow`, FE-1215) makes saved missions, visible comparison-harness framing, conversational launch, and a readable free-form report approachable without weakening the retained regression/evaluation machinery. Its first operator-led run falsified the nested fresh-actor topology: the top-level project Pi session must act as the simulated user and drive one direct harness subshell at a time, with stock-Pi text interaction as the portable baseline; the corrected real witness is deferred to `saved-mission-comparison-witness`. The KA stream owns executor/orchestrator/Execute-mode work. FE-1192 and FE-1195 completed the attempt, isolated fan-in, durable parallel authority, and epic integration sequence; `executor-plan-coherence` now uses those settled seams to test whether multi-slice plans converge on one working result. Broader instrumentation remains trigger-gated under Later. -**End-to-end comparison lane.** FE-1239 materializes the handoff that the completed reporting procedure names but does not run: two fresh rigorous elicitation lanes produce exact approved specifications, each specification crosses an immutable content-addressed handoff into both Brunch and Claude execution, and the unchanged Petri-editor oracle joins all four outputs into requirement-level evidence. Shared interface requirements are disclosed before elicitation and remain controlled baseline, while private/reveal requirements stay controller-only. The first tracer supports bounded within-case contrasts only — no winner, reliability claim, or cross-case causal conclusion. FE-1241 keeps that Petri editor as the greenfield case and adds frozen Brunch/Petrinaut brownfield packets plus controller-owned oracles. All three missions can now enter the same execution-only workflow and publication contract. Brownfield preparation uses pinned remote-free snapshots and target-bounded tools as experimental hygiene, not adversarial benchmark proof; Clay remains outside the campaign. +**Comparison lane.** FE-1239 remains the sole rigorous greenfield 2×2 tracer: two independently approved Petri specifications cross exact handoffs into Brunch and Claude execution and one unchanged oracle joins all four outputs. FE-1241 keeps that Petri editor as the greenfield study profile and adds frozen Brunch/Petrinaut brownfield packets plus deterministic oracles under a learning-first workflow. FE-1253 is deliberately thinner: a saved exploratory prospect mission plus a deterministic React/Node/TypeScript/SQLite execution regression case, with no prospect study profile, requirement ledger, scored provider gate, or campaign claim. **Current seams.** Brunch ships on the `1.0.0-alpha.x` line. One-shot `ask` is the only interactive structured-exchange terminal; D125-L's live ask registry provides headless discovery/answering, while the transcript-backed pending projection remains a compatibility surface for live offer tools after the legacy `present_question` pending branch retired. Sweep classification remains fail-closed and compile-time anchored to the exchange-schema terminal names (D117-L), while the larger capture-conditional watermark question remains A40-L. @@ -119,7 +119,7 @@ Everything executor/orchestrator-shaped or Execute-mode-owned belongs to Kostand - `executor-slice-admission-parity` ([FE-1240](https://linear.app/hash/issue/FE-1240/prevent-invalid-scoped-slices-from-reaching-execution)) — **complete on `ka/fe-1240-slice-admission-parity`, restacked on `next` after FE-1239 landed:** incomplete scoped-slice worker context is rejected during deterministic plan admission, exact findings enter bounded repair before `slice_execute`, and the execution boundary remains fail-closed. Definition below. - `executor-plan-coherence` ([FE-1250](https://linear.app/hash/issue/FE-1250/build-coherent-execution-plans)) — **implementation complete on `ka/fe-1250-coherent-execution-plans`, based on FE-1240 via `next`:** frontier-verified multi-slice epics now require one ordinary terminal member over every sibling, planner conduct establishes shared foundations, and workers preserve cumulative public contracts for the canonical harness. Fast verification passes; next is the separately authorized unchanged Petri comparison rerun. No new browser gate, durable plan kind, or executor lifecycle phase. Definition below. - `comparison-publication-workflow` ([FE-1251](https://linear.app/hash/issue/FE-1251/publish-traceable-comparison-reports)) — **active on `ka/fe-1251-comparison-publication`, stacked on FE-1250:** capture immutable controller and release provenance before comparison lanes start, then explicitly publish retained validity-first reports into the canonical Notion database through a guarded idempotent skill. Definition below. -- `prospect-research-workspace-e2e` ([FE-1253](https://linear.app/hash/issue/FE-1253/add-prospect-research-end-to-end-comparison-case)) — **active on `ka/fe-1253-prospect-research-e2e`, stacked on FE-1241:** the frozen packets and opaque Brunch handoff are materialized, and the independent browser/HTTP/SQLite/export oracle is calibrated against a known-good full stack plus focused wrong rivals. Next: restore and witness strict greenfield Claude isolation on the landed parent before any scored provider turn. +- `prospect-research-workspace-regression` ([FE-1253](https://linear.app/hash/issue/FE-1253/thin-prospect-research-into-a-regression-case)) — **active on `ka/fe-1253-prospect-research-e2e`, stacked on FE-1241:** retain the exploratory mission and calibrated browser/HTTP/SQLite/export oracle while deleting the unexecuted prospect study profile, requirement-ledger expansion, and scored-provider completion gate. - **Carved from FE-1167 (2026-07-13):** the Execute-mode evidence sub-list — Execute entry beats on thin vs rich seeds (assessment honesty: Ask on thin, Proceed on rich) and the FE-1107/KA residue (close-or-narrow, demo/walkthrough session via `TESTING_PLAN.md`, post-KA plan pass). The former sticky-posture question is no longer KA residue: FE-1187 `remediation-4` owns the persistent Specify elicitation-style audit/SPEC revision, and its Continue lexical audit owns the old `continue` ambiguity. Full context in the archived FE-1167 definition (`docs/archive/PLAN_HISTORY.md`). - `planning-process-model` — **moved to the KA stream 2026-07-13; reshaped by D126-L**: the durable scope handoff is settled, so this item now owns only plan projection and epistemic-horizon questions beyond committed scopes. Definition below. - **[1.x data-model handoff owed by FE-1187](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md):** a concise colleague-facing note must distinguish current canon from directional vocabulary before the consolidated outer checkpoint. Current: `{milestone, frontier, scope}` with executor-derived slices; basis (`explicit | implicit`) orthogonal to settlement (`advisory | settled`); no persisted readiness grade or spec-global elicitation-gap table; active-branch Pi JSONL reads; one CommandExecutor mutation authority with spec-local LSN/change log; no new projected `vv_obligation` (legacy rows remain readable). Directional only: explain stored `thesis` as pitch/concept without renaming it yet, and avoid new coupling to spec-local `term` while its possible workspace lift remains future work. Persisted judgment-shaped reconciliation needs remain current, but are YAGNI-suspect: add no kinds/consumers/orchestration dependency without fresh evidence; re-evaluate derivation/removal when KA work first needs that table. Delivered note: [`Brunch 1.x data-model handoff`](../docs/architecture/BRUNCH_1X_DATA_MODEL_HANDOFF.md); it does not block the deterministic TUI queue. @@ -457,24 +457,24 @@ Instrumentation experiments and far-horizon items. Each re-enters only via re-qu - **Boundary:** no hidden-oracle exposure, inferred browser command, new plan-plane node, candidate command surface, execution-time plan repair, or FE-1241 comparison-framework change. - **Current execution pointer:** none; the two-card scope is consumed. Re-enter only for the unchanged frozen Petri comparison rerun. -### prospect-research-workspace-e2e +### prospect-research-workspace-regression -- **Name:** Add prospect research full-stack end-to-end comparison case -- **Linear / branch:** [FE-1253](https://linear.app/hash/issue/FE-1253/add-prospect-research-end-to-end-comparison-case); `ka/fe-1253-prospect-research-e2e`, stacked on FE-1241 with no parent issue. -- **Kind:** structural evaluation expansion — a second greenfield whole-application case with a full-stack lifecycle and independent browser/API/SQLite oracle. -- **Certainty:** proving. -- **Status:** active 2026-07-22. The content-addressed mission/baseline/registry/public packet, opaque Brunch execution seed, and closed compiled oracle are materialized. The exact npm lifecycle and seven independent claim-linked journeys pass the known-good full stack; eight focused semantic rivals plus an external-request rival prove sensitivity and deterministic cleanup. No scored provider turn exists yet because strict greenfield Claude isolation remains stop-the-line. -- **Objective:** prove that FE-1239's exact elicitation → immutable handoff → crossed Brunch/Claude execution → requirement-ledger path can build and assess a local full-stack prospect research product rather than only a static client application. -- **Lights up:** private prospect mission + public mechanical baseline → two independently approved specifications → exact crossed 2×2 execution → deterministic research/qualification fixtures → browser, HTTP, and SQLite evidence → requirement-level report. -- **Stabilizes:** a compile-time second greenfield profile; fixed React + Node.js + TypeScript + SQLite delivery; `npm test` / `npm run build` / `npm start`; environment-addressed port/database/fixture inputs; runtime-network denial; server-side fixture adapters; and opaque exact-spec Brunch seeding without Petri-specific parsing. -- **Depends on:** FE-1241 landing/restack for the finalized case registry and strict solution-isolation substrate. The reviewed greenfield Claude isolation gap is stop-the-line: no scored FE-1253 lane may launch until strict MCP/tool/sandbox/controller-root/network policy is restored and witnessed on the actual parent. -- **Boundary:** fresh empty Git repositories only; one fixed TypeScript stack but open build tool, Node framework, ORM, router, component library, CSS, and internal architecture. Manually initiated prospect research, evidence-backed qualification, deduplication/provenance, suppression, review, audited override, approval, export, provider failure, and restart persistence are in. Fixture adapters are scored; live Pi/Clay service quality is not. -- **Acceptance:** all mission/baseline/registry/public/oracle bytes freeze before the first valid elicitation lane; exact approved specs cross unchanged into all four cells; the full-stack lifecycle starts from public commands and a fresh temporary SQLite database; one known-good fixture passes; focused rivals for unapproved research, confidence-only qualification, lost provenance, weak suppression, reasonless/destructive override, overbroad export, provider-failure laundering, and non-durable state fail; every check retains claim-linked evidence and later journeys run after earlier failures; existing Petri and brownfield case bytes/oracles remain unchanged. -- **Verification:** inner — exact public/study/oracle parsers, hash drift, claim coverage, opaque Brunch seed, and strict isolation admission. Middle — independent browser + API + SQLite journeys over fresh database/fixture state, paired with a tiny controller reference model and focused wrong rivals. Outer — two rigorous elicitation lanes and the closed 2×2 Brunch/Claude execution matrix, followed by validity-first reporting; no winner, reliability estimate, live-provider grade, or cross-case causal claim. -- **Cross-cutting obligations:** public addressability receives no elicitation credit; mission/reveal/fixtures/expected states remain outside targets; strict Claude and Brunch isolation precedes scored execution; runtime network stays denied; all failed/invalid attempts are retained; Brunch stops at `promotion_prepared` and never lands. -- **Explicitly out:** sequencing, sending, follow-ups, mailbox/reply handling, unsubscribe/bounce automation, scheduler, CRM sync, live Clay/Pi credentials, production deployment, Cursor/Codex lanes, arbitrary manifest commands/plugins, `ExecutionAttempt` widening, automatic winner, and reliability repetitions. -- **Traceability:** D70-L, D134-L, D139-L; I67-L; FE-1210/FE-1230/FE-1232/FE-1239/FE-1241; [`docs/praxis/comparison-runs.md`](../docs/praxis/comparison-runs.md). -- **Current execution pointer:** none. The full-stack-oracle card is implemented; next scope the strict greenfield isolation witness required before scored provider execution. +- **Name:** Thin prospect research into a regression case +- **Linear / branch:** [FE-1253](https://linear.app/hash/issue/FE-1253/thin-prospect-research-into-a-regression-case); `ka/fe-1253-prospect-research-e2e`, stacked on FE-1241 with no parent issue. +- **Kind:** earned evaluation simplification — retain a deterministic full-stack regression oracle and retire unproven campaign expansion. +- **Certainty:** earned. +- **Status:** active 2026-07-23. The mission, public packet, opaque Brunch seed, closed compiled oracle, known-good full stack, and focused rivals remain. The prospect-specific end-to-end study profile and provider campaign gate are being retired before any scored lane exists. +- **Objective:** provide one deterministic implementation-level acceptance case for the prospect research workflow without making ordinary regression value depend on a rigorous 2×2 campaign. +- **Closes:** the accidental coupling between a useful full-stack oracle and an unexecuted prospect-specific campaign. +- **Deletes / retires:** the prospect end-to-end study contract, shared baseline, reveal registry, matrix registration, requirement-ledger obligation, scored-provider gate, and campaign claims. +- **Stabilizes:** the fixed React + Node.js + TypeScript + SQLite public packet; `npm test` / `npm run build` / `npm start`; fresh database and fixture isolation; runtime-network denial; independent browser/API/SQLite/export journeys; focused rivals; and opaque specification seeding. +- **Depends on:** FE-1241 for the finalized execution-case registry and oracle dispatch only. No provider or strict greenfield Claude-isolation witness blocks completion. +- **Boundary:** manually initiated prospect research, evidence-backed qualification, deduplication/provenance, suppression, review, audited override, approval, export, provider failure, and restart persistence are in. Outreach delivery, live Pi/Clay quality, campaign composition, scored provider lanes, reliability claims, and `ExecutionAttempt` widening are out. +- **Acceptance:** the known-good full stack passes; focused rivals for unapproved research, confidence-only qualification, lost provenance, weak suppression, reasonless/destructive override, overbroad export, provider-failure laundering, non-durable state, and external runtime requests fail their owning claims; the prospect case is absent from end-to-end study registration; existing Petri and brownfield study bytes/oracles remain unchanged. +- **Verification:** inner — exact public/oracle parsers, oracle-pack hash, claim coverage, and opaque Brunch seed. Middle — independent browser + API + SQLite journeys over fresh database/fixture state, paired with the controller reference model and focused rivals. No outer campaign evidence is required. +- **Cross-cutting obligations:** controller fixtures and expected states remain outside targets; runtime network stays denied; compiled oracle dispatch remains fail-closed; historical comparison evidence remains unchanged. +- **Traceability:** D70-L, D139-L; FE-1230/FE-1241; [`docs/praxis/comparison-runs.md`](../docs/praxis/comparison-runs.md). +- **Current execution pointer:** thin the materialized branch to this regression boundary, then close through deterministic verification. ### comparison-reporting-skills @@ -669,10 +669,10 @@ KA stream: lights_up: same-base brownfield elicitation -> exact handoff -> crossed execution -> case oracle cases: minimal-petri-net-editor = FE-1241 greenfield reference | FE-1201-derived brunch backend | PR #9051 petrinaut frontend excludes: Clay | more greenfields inside FE-1241 | repetitions | aggregate winner | ExecutionAttempt widening - -[hard]-> prospect-research-workspace-e2e (FE-1253) - status: active; full-stack oracle calibrated, scored lanes wait on strict greenfield Claude isolation - lights_up: prospect mission -> exact elicited specs -> crossed execution -> browser/API/SQLite evidence - excludes: outreach delivery | live provider grading | repetitions | aggregate winner + -[hard]-> prospect-research-workspace-regression (FE-1253) + status: active; deterministic full-stack oracle calibrated, campaign expansion retiring + closes: useful regression oracle coupled to an unexecuted 2x2 campaign + excludes: provider lanes | requirement ledger | outreach delivery | reliability claims # executor-run-environment (FE-1166) resolved 2026-07-15: policy merged (PR #302), # live remainder folded into executor-plan-synthesis (FE-1197), card consumed. diff --git a/memory/SPEC.md b/memory/SPEC.md index 7fcdc4f41..8283a737d 100644 --- a/memory/SPEC.md +++ b/memory/SPEC.md @@ -317,7 +317,7 @@ The POC's purpose is to prove three things: (a) that pi's coding-agent harness c | D98-L | Operational mode remains the only top-level role/authority state: live product modes are `specify` / Specify and `execute` / Execute, each 1:1 with its foreground agent. FE-1187 narrowly amends the former “no conduct bias as runtime state” rule: Specify additionally carries one user-changeable `elicitation_style` (`interrogate | disambiguate | propose`) that changes elicitor process only, never authority, capability, agent identity, or target graph plane. This does not revive the retired generic strategy/lens/method axes, an Enhance mode, persisted lens selection, or capability routing as mutable state. Supersedes: D98-L's blanket prohibition on every secondary session axis while preserving its two-mode and axis-retirement conclusions. | [`src/.pi/extensions/TOPOLOGY.md`](../src/.pi/extensions/TOPOLOGY.md), [`src/session/TOPOLOGY.md`](../src/session/TOPOLOGY.md) | active — materialized 2026-07-16 (FE-1187) | | D100-L | `project` is a distinct first-level live Specify-mode skill home for cross-plane derivation, not a `generate` sub-mode. `generate` fans out alternatives within a target plane from context; `project` starts from accepted upstream graph anchors and derives downstream plane candidates/drafts plus connecting edge intent. It uses the existing structured-exchange offer/terminal seams (`present_candidates`, `present_review_set`, and their declared `ask` continuations per D116-L) and hands exact graph expression back to `map` / review-set commitment; it adds no product tool, exchange schema family, or direct graph-write path. Depends on: D95-L, D96-L, D97-L, I51-L. | [`src/agents/skills/TOPOLOGY.md`](../src/agents/skills/TOPOLOGY.md), [`src/agents/subagents/TOPOLOGY.md`](../src/agents/subagents/TOPOLOGY.md) | active | -| D139-L | The prospect research workspace is the second admitted greenfield end-to-end case (2026-07-22, FE-1253), separate from FE-1241's two brownfield replays. Its public execution contract fixes one npm repository with a React/TypeScript frontend, Node.js/TypeScript backend, SQLite persistence, `npm test` / `npm run build` / `npm start`, environment-addressed port/database/fixture paths, a health endpoint, accessible mechanical controls, package-registry-only install, and denied runtime network. Pi-compatible qualification and Clay-compatible research are server-side interfaces exercised only through controller-owned deterministic local fixtures; live provider quality is not scored. Product semantics such as evidence sufficiency, deduplication provenance, suppression precedence, override history, export eligibility, failure honesty, and restart durability remain reveal-gated and controller-oracle-owned. The case reuses FE-1239's exact handoff, fixed 2×2 matrix, failure retention, and validity-first reporting without arbitrary manifest commands/plugins or `ExecutionAttempt` widening. Depends on: D70-L, D134-L, FE-1239, FE-1241. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `prospect-research-workspace-e2e`; `testing/{comparisons,end-to-end-comparisons,execution-comparisons}` case homes | active — contract and independent full-stack oracle calibrated; scored lanes remain isolation-gated | +| D139-L | FE-1253 admits the prospect research workspace as a deterministic greenfield execution regression case, not a second end-to-end campaign profile. Its public contract fixes one npm repository with a React/TypeScript frontend, Node.js/TypeScript backend, SQLite persistence, `npm test` / `npm run build` / `npm start`, environment-addressed port/database/fixture paths, a health endpoint, accessible mechanical controls, package-registry-only install, and denied runtime network. Pi-compatible qualification and Clay-compatible research are server-side interfaces exercised through controller-owned deterministic local fixtures; live provider quality is not scored. The saved mission remains available for exploratory specification work, while a future crossed campaign requires a separate explicit study contract. Supersedes FE-1253's unexecuted 2×2 campaign expansion. Depends on: D70-L, FE-1230, FE-1241. | [`src/dev/TOPOLOGY.md`](../src/dev/TOPOLOGY.md); PLAN `prospect-research-workspace-regression`; `testing/comparisons/missions/prospect-research-workspace.md`; `testing/execution-comparisons/cases/prospect-research-workspace/` | active — deterministic full-stack oracle calibrated; campaign expansion retired | ### Critical Invariants @@ -696,7 +696,7 @@ For agent-as-user evaluation, the primary behavioral claim is **consequential-fa **Brownfield execution comparison assessment (updated 2026-07-23).** Observability is **partial**: pinned trees, diffs, build/test output, controller verdicts, temporary-host Git state, browser DOM/accessibility state, request/abort evidence, interventions, cleanup, and retained attempts are text-native; visual hierarchy and provider conduct remain external. Reproducibility is **high for deterministic mechanics and unproven for campaigns**: frozen Brunch/Petrinaut packets, remote-free pinned snapshots, case-owned recipes/oracles, the fake optimizer, and contrastive rivals are stable, while generated implementation and model conduct vary. Controllability is **partial**: Brunch host landing uses disposable repositories and an independent Git model; Petrinaut uses focused builds and same-origin browser checks after lane termination. Lightweight tool restrictions prevent accidental contamination but do not establish adversarial isolation; reports must preserve that limitation. -**FE-1253 prospect research full-stack assessment (2026-07-22).** Observability is **high for structural behavior**: browser accessibility, HTTP responses, SQLite rows, downloaded export bytes, bounded process output, and browser runtime requests are text-native; whether qualification rationales are persuasive remains qualitative. Reproducibility is **high mechanically**: each claim starts the exact public lifecycle with a fresh port, temporary database, copied deterministic fixture, and browser context; a known-good React/Node/TypeScript/SQLite implementation passes while focused wrong rivals prove sensitivity. Controllability is **high for the calibrated mechanical oracle and partial for the campaign**: lifecycle, local fixtures, independent journeys, reference-model comparison, cleanup, and runtime-request rejection are agent-controlled, while provider conduct and strict greenfield Claude isolation remain external gates. Strict Claude solution isolation is a stop-the-line prerequisite for scored lanes, not an accepted blind spot. +**FE-1253 prospect research full-stack assessment (2026-07-22).** Observability is **high for structural behavior**: browser accessibility, HTTP responses, SQLite rows, downloaded export bytes, bounded process output, and browser runtime requests are text-native; whether qualification rationales are persuasive remains qualitative. Reproducibility is **high mechanically**: each claim starts the exact public lifecycle with a fresh port, temporary database, copied deterministic fixture, and browser context; a known-good React/Node/TypeScript/SQLite implementation passes while focused wrong rivals prove sensitivity. Controllability is **high for the deterministic regression case**: lifecycle, local fixtures, independent journeys, reference-model comparison, cleanup, and runtime-request rejection are agent-controlled. No provider campaign or comparative claim is part of FE-1253. **Standalone-web tracer assessment (2026-07-14).** Observability is **partial**: JSONL, target-addressed RPC, semantic event frames, and accessible DOM/text states make structural session behavior observable; stream cadence, visual hierarchy, and error feel remain manual. Reproducibility is **high** for this tracer: paired temporary production boots use the deterministic faux provider and an existing JSONL session, not a live model or a static transcript golden. Controllability is **partial**: the middle loop controls the browser journey and cache/overlay loss, while a workbench walkthrough owns the bounded UX verdict. The normalizer may remove only declared nondeterministic ids/timestamps; it must not mask Brunch binding/runtime/exchange differences. @@ -781,9 +781,8 @@ Dev-loop artifacts route to gitignored `.fixtures/scratch///`, res | Middle | **Brunch black-box `/brunch:land` journey plus independent Git model** | The controller creates disposable host/run/review repositories, resumes a settled candidate session under `PI_OFFLINE=1`, drives only the public TUI command, snapshots Git and run metadata before confirmation, then compares post-apply state with a controller-owned full-range reference model. This proves no pre-confirm mutation, complete multi-slice landing rather than final-commit-only application, mode-aware brownfield integration and greenfield materialization, bookkeeping exclusion, honest failure, and terminal `landed` without importing candidate host-landing modules or permitting a provider turn. | | Middle | **Petrinaut standalone fake-provider browser and accessibility oracle** | In the full pinned HASH checkout, controller-prepared focused Petrinaut packages launch the public `/optimization` website route. A deterministic loopback fake optimizer proves capability-present Optimizations visibility, scenario-first configuration, fixed/optimized parameter selection, one metric plus direction, request construction, streamed trials/best-so-far/completion/error/cancellation, upstream abort, private-origin secrecy behind the same-origin proxy, and reachable accessible controls without requiring the real optimizer service. | | Outer | **Brownfield live/qualitative evidence** | One fully provisioned HASH `/processes/draft` host/iframe smoke checks capability relay, optimizer proxying, iframe isolation, and adapter reality; one optional real-optimizer Petrinaut smoke checks live response compatibility. Identity-masked visual review judges Petrinaut hierarchy, clarity, and progress legibility; identity-masked code review judges both outputs. These are non-gating findings, are `not_assessable` when unavailable, and cannot override or repair common mechanical failures. | -| Inner | **FE-1253 prospect full-stack case contracts** | Exact parsers and content hashes freeze the private mission, public mechanical baseline, reveal registry, React/Node/TypeScript/SQLite public packet, fixture files, closed oracle manifest, budgets, and fixed 2×2 axes. Brunch preserves each elicited specification as one opaque settled handoff instead of applying Petri-specific parsing. | +| Inner | **FE-1253 prospect full-stack regression contract** | Exact parsers and oracle-pack hashes freeze the React/Node/TypeScript/SQLite public packet, fixture files, closed oracle manifest, and implementation modules. Brunch can preserve the specification as one opaque settled input instead of applying Petri-specific parsing. | | Middle | **FE-1253 independent browser/API/SQLite differential** | A code-owned runner starts each candidate with a fresh database and local fixture, then compares accessible UI, HTTP results, SQLite state, and export bytes with a tiny controller reference model. Independent journeys cover approval-before-run, evidence qualification, deduplication provenance, suppression across rerun, reasoned override history, approved-only export, provider-failure honesty, and restart persistence; focused wrong rivals prove sensitivity. | -| Outer | **FE-1253 crossed prospect workspace campaign** | Two rigorous elicitation lanes produce exact approved specifications; each crosses into Brunch and Claude execution. All four cells retain strict isolation, target-visible conduct, unchanged handoffs, common oracle evidence, failures, cleanup, and a validity-first requirement ledger. The result supports bounded within-case contrasts only, not a winner, reliability estimate, live-provider grade, or cross-case causal claim. | | Middle | **FE-1187 controlled provider conduct gate — paused** | On explicit re-entry, reconcile the extractor/oracle against the landed mixed-settlement contract, then run three fresh normalized-ingest samples. All must use free-text digest feedback, one bounded questionnaire when several questions exist, no combinatorial options, an honestly assigned per-node/per-edge review proposal when review is used, exact atomic settlement preservation, and no post-review mutation that completes or rewrites the approved proposal. Direct advisory mutation without review remains valid. Reports retain provider/model stamps and deterministic conduct markers; the stopped 2026-07-17 run is diagnostic and counts 0/3. | | Inner | **FE-1187 Impact Ledger golden + word-wrap-tolerant render-honesty** | Golden/inline snapshots at narrow/normal/wide widths lock populated-section-only canonical order, absence of empty heading/`None` pairs, per-node/per-edge settlement visibility, elision, `refs:` row shape, and the `obligation` fallback label for the borderless Impact Ledger renderer (D27-L/D131-L). `missingRenderedDetailsLeaves` is extended to reassemble `table`'s word-wrapped physical lines back into logical cells before leaf-presence checking, so a value silently split across wrapped rows cannot pass as "rendered" by accident. | | Middle | **FE-1187 Impact Ledger differential reference extractor** | A deliberately naive reference extractor (flat node/edge/term inventory, no styling or grouping) is compared against the ledger's code/connection inventory over the witnessed fixture plus hand-authored edge fixtures (empty group, single-node group, mixed-settlement group, term-only group, max-refs group). Proves item/status inventory completeness and empty-group omission independent of the "real" renderer's own logic. | diff --git a/src/dev/TOPOLOGY.md b/src/dev/TOPOLOGY.md index 4d74ccc97..7ee93fe86 100644 --- a/src/dev/TOPOLOGY.md +++ b/src/dev/TOPOLOGY.md @@ -33,11 +33,11 @@ Petrinaut preparation selected by D136-L is split across the historical-replay o `execution-comparison-operator.ts` dispatches both brownfield oracles and the Petri and prospect greenfield oracles through a closed compile-time registry and includes every implementation module in the immutable oracle-pack hash; manifests cannot select paths, commands, or plugins at runtime. -## Prospect Research Greenfield Case +## Prospect Research Regression Case -D139-L admits `prospect-research-workspace-v1` as the second greenfield profile. Its private mission lives under `testing/comparisons/missions/`; its disclosed full-stack lifecycle baseline and reveal registry live under `testing/end-to-end-comparisons/cases/prospect-research-workspace/`; and its public execution packet plus controller fixtures/manifest live under the matching `testing/execution-comparisons/` case home. The public contract fixes React + Node.js + TypeScript + SQLite, npm lifecycle commands, health/addressability, environment-selected local fixtures, and denied runtime network while leaving framework internals open. Brunch seeds the exact target-authored specification opaquely rather than passing it through the Petri section parser. +D139-L admits `prospect-research-workspace-v1` as a deterministic greenfield execution regression case, not an end-to-end study profile. Its exploratory private mission lives under `testing/comparisons/missions/`; its public execution packet plus controller fixtures and manifest live under the matching `testing/execution-comparisons/` case home. The public contract fixes React + Node.js + TypeScript + SQLite, npm lifecycle commands, health/addressability, environment-selected local fixtures, and denied runtime network while leaving framework internals open. Brunch can seed the exact specification opaquely rather than passing it through the Petri section parser. -`execution-comparison/prospect-research-workspace-oracle.ts` is the public controller entry point. Its private subtree owns the exact npm test/build/start lifecycle, fresh process/database/fixture isolation, accessible browser actions, direct same-origin HTTP and SQLite/export evidence, a tiny independent fixture model, claim-linked journeys, runtime-request detection, and deterministic browser/process cleanup. The controller case home carries the known-good React/Node/TypeScript/SQLite full stack and focused wrong rivals; all participate in the immutable pack. Strict Claude solution isolation remains a stop-the-line prerequisite for every scored greenfield lane. +`execution-comparison/prospect-research-workspace-oracle.ts` is the public controller entry point. Its private subtree owns the exact npm test/build/start lifecycle, fresh process/database/fixture isolation, accessible browser actions, direct same-origin HTTP and SQLite/export evidence, a tiny independent fixture model, claim-linked journeys, runtime-request detection, and deterministic browser/process cleanup. The controller case home carries the known-good React/Node/TypeScript/SQLite full stack and focused wrong rivals; all participate in the immutable pack. No scored provider lane or four-cell campaign is required to close this regression case. ## Launcher Surface diff --git a/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts b/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts index 8be6e0112..25232e235 100644 --- a/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/case-profile.test.ts @@ -22,10 +22,6 @@ const petrinautStudy = join( repositoryRoot, 'testing/end-to-end-comparisons/cases/petrinaut-optimization/study-contract.json', ); -const prospectStudy = join( - repositoryRoot, - 'testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json', -); const prospectExecution = join( repositoryRoot, 'testing/execution-comparisons/cases/prospect-research-workspace', @@ -35,28 +31,21 @@ const petrinautExecution = join(repositoryRoot, 'testing/execution-comparisons/c const execFileAsync = promisify(execFile); describe('compiled end-to-end comparison case profiles', () => { - it('loads two greenfield cases and exactly two pinned brownfield cases', async () => { - const [petri, prospect, brunch, petrinaut, prospectPacket, brunchPacket, petrinautPacket] = - await Promise.all([ - loadEndToEndStudyContract({ repositoryRoot, contractPath: petriStudy }), - loadEndToEndStudyContract({ repositoryRoot, contractPath: prospectStudy }), - loadEndToEndStudyContract({ repositoryRoot, contractPath: brunchStudy }), - loadEndToEndStudyContract({ repositoryRoot, contractPath: petrinautStudy }), - loadPublicCasePacket(prospectExecution), - loadPublicCasePacket(brunchExecution), - loadPublicCasePacket(petrinautExecution), - ]); + it('keeps prospect research as an execution case outside the end-to-end profiles', async () => { + const [petri, brunch, petrinaut, prospectPacket, brunchPacket, petrinautPacket] = await Promise.all([ + loadEndToEndStudyContract({ repositoryRoot, contractPath: petriStudy }), + loadEndToEndStudyContract({ repositoryRoot, contractPath: brunchStudy }), + loadEndToEndStudyContract({ repositoryRoot, contractPath: petrinautStudy }), + loadPublicCasePacket(prospectExecution), + loadPublicCasePacket(brunchExecution), + loadPublicCasePacket(petrinautExecution), + ]); expect(petri.contract).toMatchObject({ id: 'minimal-petri-net-editor-e2e-v1', caseId: 'minimal-petri-net-editor-v1', oracle: { id: 'minimal-petri-net-editor-oracles-v2' }, }); - expect(prospect.contract).toMatchObject({ - id: 'prospect-research-workspace-e2e-v1', - caseId: 'prospect-research-workspace-v1', - oracle: { id: 'prospect-research-workspace-oracles-v1' }, - }); expect(brunch.contract).toMatchObject({ id: 'brunch-host-landing-e2e-v1', caseId: 'brunch-host-landing-v1', diff --git a/src/dev/end-to-end-comparison/__tests__/study-contract.test.ts b/src/dev/end-to-end-comparison/__tests__/study-contract.test.ts index fcf4e0e1e..590a5933d 100644 --- a/src/dev/end-to-end-comparison/__tests__/study-contract.test.ts +++ b/src/dev/end-to-end-comparison/__tests__/study-contract.test.ts @@ -75,27 +75,6 @@ describe('end-to-end study contract', () => { expect(loaded.contractSha256).toMatch(/^sha256:[a-f0-9]{64}$/u); }); - it('loads the content-addressed prospect research study', async () => { - const repositoryRoot = fileURLToPath(new URL('../../../../', import.meta.url)); - const loaded = await loadEndToEndStudyContract({ - repositoryRoot, - contractPath: fileURLToPath( - new URL( - '../../../../testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json', - import.meta.url, - ), - ), - }); - - expect(loaded.contract).toMatchObject({ - id: 'prospect-research-workspace-e2e-v1', - caseId: 'prospect-research-workspace-v1', - oracle: { id: 'prospect-research-workspace-oracles-v1' }, - specSources: ['brunch_spec', 'claude_spec'], - executors: ['brunch', 'claude_code'], - }); - }); - it('accepts the frozen two-by-two study shape', () => { expect(parseEndToEndStudyContract(study())).toEqual(study()); }); diff --git a/src/dev/end-to-end-comparison/study-contract.ts b/src/dev/end-to-end-comparison/study-contract.ts index 2a07b0971..9ecedff0d 100644 --- a/src/dev/end-to-end-comparison/study-contract.ts +++ b/src/dev/end-to-end-comparison/study-contract.ts @@ -82,10 +82,6 @@ export function parseEndToEndStudyContract(value: unknown): EndToEndStudyContrac value['caseId'] === 'minimal-petri-net-editor-v1' && oracle['id'] === 'minimal-petri-net-editor-oracles-v2' && source === undefined) || - (value['id'] === 'prospect-research-workspace-e2e-v1' && - value['caseId'] === 'prospect-research-workspace-v1' && - oracle['id'] === 'prospect-research-workspace-oracles-v1' && - source === undefined) || (value['id'] === 'petrinaut-optimization-e2e-v1' && value['caseId'] === 'petrinaut-optimization-v1' && oracle['id'] === 'petrinaut-optimization-oracles-v1' && diff --git a/src/dev/execution-comparison/__tests__/oracle-pack.test.ts b/src/dev/execution-comparison/__tests__/oracle-pack.test.ts index e4d8ad7fa..c3335ee18 100644 --- a/src/dev/execution-comparison/__tests__/oracle-pack.test.ts +++ b/src/dev/execution-comparison/__tests__/oracle-pack.test.ts @@ -25,41 +25,27 @@ const petrinautRequirementsPath = fileURLToPath( import.meta.url, ), ); -const prospectRequirementsPath = fileURLToPath( - new URL( - '../../../../testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json', - import.meta.url, - ), -); - describe('compiled controller oracle manifests', () => { it('accepts exactly four compiled variants with complete non-Petri claim coverage', async () => { - const [petri, prospect, brunch, petrinaut, registry, petrinautRegistry, prospectRegistry] = - await Promise.all([ - loadControllerOracleManifest(join(casesRoot, 'minimal-petri-net-editor')), - loadControllerOracleManifest(join(casesRoot, 'prospect-research-workspace')), - loadControllerOracleManifest(join(casesRoot, 'brunch-host-landing')), - loadControllerOracleManifest(join(casesRoot, 'petrinaut-optimization')), - readFile(requirementsPath, 'utf8').then( - (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, - ), - readFile(petrinautRequirementsPath, 'utf8').then( - (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, - ), - readFile(prospectRequirementsPath, 'utf8').then( - (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, - ), - ]); + const [petri, prospect, brunch, petrinaut, registry, petrinautRegistry] = await Promise.all([ + loadControllerOracleManifest(join(casesRoot, 'minimal-petri-net-editor')), + loadControllerOracleManifest(join(casesRoot, 'prospect-research-workspace')), + loadControllerOracleManifest(join(casesRoot, 'brunch-host-landing')), + loadControllerOracleManifest(join(casesRoot, 'petrinaut-optimization')), + readFile(requirementsPath, 'utf8').then( + (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, + ), + readFile(petrinautRequirementsPath, 'utf8').then( + (raw) => JSON.parse(raw) as { rows: readonly { id: string }[] }, + ), + ]); expect(petri.id).toBe('minimal-petri-net-editor-oracles-v2'); expect(prospect.id).toBe('prospect-research-workspace-oracles-v1'); expect(brunch.id).toBe('brunch-host-landing-oracles-v1'); expect(petrinaut.id).toBe('petrinaut-optimization-oracles-v1'); expect(() => - assertOracleClaimCoverage( - prospect, - prospectRegistry.rows.map(({ id }) => id), - ), + assertOracleClaimCoverage(prospect, ['PR1', 'PR2', 'PR3', 'PR4', 'PR5', 'PR6', 'PR7', 'PR8', 'PR9']), ).not.toThrow(); expect(() => assertOracleClaimCoverage( diff --git a/src/dev/execution-comparison/case-contract.ts b/src/dev/execution-comparison/case-contract.ts index 4528c4d49..030179178 100644 --- a/src/dev/execution-comparison/case-contract.ts +++ b/src/dev/execution-comparison/case-contract.ts @@ -71,6 +71,24 @@ export interface ProspectResearchWorkspaceExecutionCasePublicContract { }; readonly acceptance: { readonly healthPath: '/api/health'; + readonly statePath: '/api/state'; + readonly projectsPath: '/api/projects'; + readonly projectActionPathPattern: '^/api/projects/[0-9]+/(approve|research)$'; + readonly prospectActionPathPattern: '^/api/prospects/[0-9]+/(approve|suppress|override)$'; + readonly exportPath: '/api/export'; + readonly responseStatuses: { + readonly unapprovedResearch: 409; + readonly reasonlessOverride: 400; + readonly providerFailure: 503; + }; + readonly sqliteTables: readonly [ + 'projects', + 'runs', + 'prospects', + 'provenance', + 'suppressions', + 'decisions', + ]; readonly executionTerminal: 'promotion_prepared'; }; readonly accessibility: { @@ -259,6 +277,14 @@ const ACCESSIBLE_NAME_PATTERNS = { arc: '^Arc: .+ to .+$', } as const satisfies Record<'place' | 'transition' | 'arc', AccessibleNamePattern>; const PROSPECT_ENVIRONMENT = ['PORT', 'DATABASE_PATH', 'RESEARCH_FIXTURE_PATH'] as const; +const PROSPECT_SQLITE_TABLES = [ + 'projects', + 'runs', + 'prospects', + 'provenance', + 'suppressions', + 'decisions', +] as const; const PROSPECT_FIELDS = ['Project name', 'Ideal customer profile', 'Decision reason'] as const; const PROSPECT_CONTROLS = [ 'Create project', @@ -324,6 +350,7 @@ function parseProspectResearchWorkspaceContract( const budgets = requiredRecord(value, 'budgets'); const delivery = requiredRecord(value, 'delivery'); const acceptance = requiredRecord(value, 'acceptance'); + const responseStatuses = requiredRecord(acceptance, 'responseStatuses'); const accessibility = requiredRecord(value, 'accessibility'); const prospectPattern = requiredRecord(accessibility, 'prospectPattern'); if ( @@ -358,7 +385,18 @@ function parseProspectResearchWorkspaceContract( 'runtimeNetwork', 'dependencyInstallNetwork', ]) || - !exactKeys(acceptance, ['healthPath', 'executionTerminal']) || + !exactKeys(acceptance, [ + 'healthPath', + 'statePath', + 'projectsPath', + 'projectActionPathPattern', + 'prospectActionPathPattern', + 'exportPath', + 'responseStatuses', + 'sqliteTables', + 'executionTerminal', + ]) || + !exactKeys(responseStatuses, ['unapprovedResearch', 'reasonlessOverride', 'providerFailure']) || !exactKeys(accessibility, [ 'application', 'projects', @@ -390,6 +428,15 @@ function parseProspectResearchWorkspaceContract( delivery['runtimeNetwork'] !== 'forbidden' || delivery['dependencyInstallNetwork'] !== 'package-registry-only' || acceptance['healthPath'] !== '/api/health' || + acceptance['statePath'] !== '/api/state' || + acceptance['projectsPath'] !== '/api/projects' || + acceptance['projectActionPathPattern'] !== '^/api/projects/[0-9]+/(approve|research)$' || + acceptance['prospectActionPathPattern'] !== '^/api/prospects/[0-9]+/(approve|suppress|override)$' || + acceptance['exportPath'] !== '/api/export' || + responseStatuses['unapprovedResearch'] !== 409 || + responseStatuses['reasonlessOverride'] !== 400 || + responseStatuses['providerFailure'] !== 503 || + !exactStrings(acceptance['sqliteTables'], PROSPECT_SQLITE_TABLES) || acceptance['executionTerminal'] !== 'promotion_prepared' || !accessibleName(accessibility['application'], 'application', 'Prospect research workspace') || !accessibleName(accessibility['projects'], 'heading', 'Research projects') || diff --git a/src/dev/execution-comparison/oracle-pack.ts b/src/dev/execution-comparison/oracle-pack.ts index 5e6c9e092..9b609f90e 100644 --- a/src/dev/execution-comparison/oracle-pack.ts +++ b/src/dev/execution-comparison/oracle-pack.ts @@ -264,6 +264,7 @@ const PROSPECT_RESEARCH_CHECKS = [ 'provider-failure', 'restart-persistence', ] as const; +const PROSPECT_RESEARCH_CLAIMS = ['PR1', 'PR2', 'PR3', 'PR4', 'PR5', 'PR6', 'PR7', 'PR8', 'PR9'] as const; function parseProspectResearchWorkspaceManifest( value: Record, @@ -301,7 +302,9 @@ function parseProspectResearchWorkspaceManifest( ) { invalid(); } - return value as unknown as ProspectResearchWorkspaceControllerOracleManifest; + const manifest = value as unknown as ProspectResearchWorkspaceControllerOracleManifest; + assertOracleClaimCoverage(manifest, PROSPECT_RESEARCH_CLAIMS); + return manifest; } function prospectValidityRules(rules: readonly string[]): boolean { diff --git a/testing/comparisons/missions/README.md b/testing/comparisons/missions/README.md index 709b7a254..57bcb4835 100644 --- a/testing/comparisons/missions/README.md +++ b/testing/comparisons/missions/README.md @@ -11,8 +11,8 @@ Current library: elicitation and end-to-end origin mission - [`brunch-host-landing.md`](brunch-host-landing.md) — Brunch host-landing brownfield mission - [`petrinaut-optimization.md`](petrinaut-optimization.md) — Petrinaut optimization brownfield mission -- [`prospect-research-workspace.md`](prospect-research-workspace.md) — full-stack prospect - research elicitation and end-to-end origin mission +- [`prospect-research-workspace.md`](prospect-research-workspace.md) — exploratory full-stack prospect + research mission with a separate deterministic execution regression case This `README.md` is the directory's reserved control file, not a mission. `/compare-specs` must exclude it from mission listing, resolution, revision, and creation. diff --git a/testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json b/testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json deleted file mode 100644 index 525c9c3fb..000000000 --- a/testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "schemaVersion": 1, - "caseId": "prospect-research-workspace-v1", - "rows": [ - { - "id": "PR1", - "publicConcern": "Application startup", - "origin": "public_baseline", - "publicWording": "The production application starts through the declared command and exposes a ready health endpoint.", - "controller": { - "wording": "The Node service, React application, and SQLite store become ready without uncaught errors or external runtime requests.", - "revealPolicy": "The shared baseline discloses delivery, health, and runtime-network mechanics.", - "expectedState": "Health is successful, the accessible shell is present, and a fresh database has no projects.", - "fixtureRefs": [] - } - }, - { - "id": "PR2", - "publicConcern": "Approved manual research", - "origin": "public_baseline", - "publicWording": "An operator creates and approves a project before manually starting fixture-backed research.", - "controller": { - "wording": "Research is unavailable before approval and one manual run imports the deterministic fixture once.", - "revealPolicy": "The mission and shared baseline disclose project approval and manual run controls.", - "expectedState": "The unapproved rival cannot run; approval followed by research creates one completed run.", - "fixtureRefs": [ - "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json" - ] - } - }, - { - "id": "PR3", - "publicConcern": "Evidence-backed qualification", - "origin": "controller_only", - "controller": { - "wording": "Qualification requires role, company-fit, and source evidence; provider confidence alone is insufficient.", - "revealPolicy": "Reveal exact evidence sufficiency only after a qualifying qualification or trust question.", - "expectedState": "The complete prospect qualifies and the high-confidence evidence-missing prospect needs review.", - "fixtureRefs": [ - "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json" - ] - } - }, - { - "id": "PR4", - "publicConcern": "Identity and provenance", - "origin": "controller_only", - "controller": { - "wording": "Duplicate people and companies merge into stable identities while retaining every contributing source.", - "revealPolicy": "Reveal provenance retention only after a qualifying deduplication or identity question.", - "expectedState": "Two records for the same person produce one prospect with two source references.", - "fixtureRefs": [ - "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json" - ] - } - }, - { - "id": "PR5", - "publicConcern": "Suppression precedence", - "origin": "controller_only", - "controller": { - "wording": "Matching person, company, domain, or email suppression dominates qualification, approval, export, and later research runs.", - "revealPolicy": "Reveal precedence and persistence only after a qualifying exclusions or suppression question.", - "expectedState": "An otherwise qualified suppressed-domain prospect remains suppressed after the fixture is rerun and never exports.", - "fixtureRefs": [ - "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/research-batch.json" - ] - } - }, - { - "id": "PR6", - "publicConcern": "Audited operator override", - "origin": "controller_only", - "controller": { - "wording": "An override requires a reason and retains the previous automated decision in immutable audit history.", - "revealPolicy": "Reveal exact audit retention only after a qualifying override or accountability question.", - "expectedState": "A reasonless override fails; a reasoned override changes current status while preserving both decisions.", - "fixtureRefs": [] - } - }, - { - "id": "PR7", - "publicConcern": "Approved export", - "origin": "controller_only", - "controller": { - "wording": "Export contains only explicitly operator-approved, non-suppressed prospects.", - "revealPolicy": "Reveal export eligibility only after a qualifying downstream-use or approval question.", - "expectedState": "Qualified but unapproved, needs-review, rejected, and suppressed prospects are absent from JSON export.", - "fixtureRefs": [] - } - }, - { - "id": "PR8", - "publicConcern": "Provider failure honesty", - "origin": "controller_only", - "controller": { - "wording": "Provider failure and partial results remain distinct from prospect rejection and do not corrupt prior state.", - "revealPolicy": "Reveal failure classification only after a qualifying provider-error or recovery question.", - "expectedState": "The failure fixture records a failed run, creates no rejection decisions, and leaves prior prospects unchanged.", - "fixtureRefs": [ - "testing/execution-comparisons/cases/prospect-research-workspace/controller/fixtures/provider-failure.json" - ] - } - }, - { - "id": "PR9", - "publicConcern": "Restart persistence", - "origin": "controller_only", - "controller": { - "wording": "Projects, prospects, suppressions, decisions, approvals, and audit history survive application restart through SQLite.", - "revealPolicy": "Reveal exact durable scope only after a qualifying persistence or recovery question.", - "expectedState": "Restarting against the same database reproduces the complete prior state and export eligibility.", - "fixtureRefs": [] - } - } - ] -} diff --git a/testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md b/testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md deleted file mode 100644 index 4ae72d122..000000000 --- a/testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md +++ /dev/null @@ -1,34 +0,0 @@ -# Shared full-stack execution and interoperability baseline - -This baseline is visible to both elicitation targets before they begin. It controls delivery and mechanical addressability so the same black-box browser, HTTP, and SQLite journeys can exercise independently produced applications. Requirements copied from this baseline are not elicitation gains. - -## Delivery - -- Build one npm repository from a fresh empty Git repository. -- Use a React and TypeScript frontend, a Node.js and TypeScript backend, and SQLite persistence. -- `npm test` runs the implementation's own tests. -- `npm run build` produces the production frontend and backend. -- `npm start` starts the production application using `PORT`, `DATABASE_PATH`, and `RESEARCH_FIXTURE_PATH` from the environment. -- `GET /api/health` returns a successful JSON response only when the application and SQLite store are ready. -- Dependency installation may use the package registry. The running application may make no external network requests. -- Pi-compatible qualification and Clay-compatible research enter only through server-side adapters. The scored application uses the local fixture selected by `RESEARCH_FIXTURE_PATH`. - -## Accessible application surface - -- Expose one `application` named `Prospect research workspace`. -- Expose a heading named `Research projects` and a region named `Prospect queue`. -- Expose textboxes named `Project name`, `Ideal customer profile`, and `Decision reason`. -- Expose buttons named `Create project`, `Approve project`, `Run research`, `Approve prospect`, `Suppress prospect`, `Override qualification`, and `Export approved prospects`. -- Expose prospect items as buttons named `Prospect: at `. -- Operation outcomes and failures use a `status` or `alert` role. - -## Mechanical interaction vocabulary - -- Create a project by filling `Project name` and `Ideal customer profile`, then activating `Create project`. -- Select a project and activate `Approve project` before activating `Run research`. -- Select a prospect through its accessible prospect-item button. -- Prospect actions operate on the selected prospect. -- An override or suppression supplies `Decision reason` before activating its action. -- Activate `Export approved prospects` and capture the downloaded JSON document. - -This baseline does not settle qualification evidence, duplicate identity, suppression precedence, override history, export eligibility, provider-failure semantics, or restart behavior. Those remain specification and controller-oracle concerns. diff --git a/testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json b/testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json deleted file mode 100644 index 8b2033ebd..000000000 --- a/testing/end-to-end-comparisons/cases/prospect-research-workspace/study-contract.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "schemaVersion": 1, - "id": "prospect-research-workspace-e2e-v1", - "caseId": "prospect-research-workspace-v1", - "mission": { - "path": "testing/comparisons/missions/prospect-research-workspace.md", - "sha256": "sha256:ee5ec5da6aefe79287a797fa18630f8b454ae53ef1d91a9503098be5eea11090" - }, - "sharedBaseline": { - "path": "testing/end-to-end-comparisons/cases/prospect-research-workspace/shared-baseline.md", - "sha256": "sha256:8efe626c9b275b56da7577715ed2dc07046d93d9a3e445840cac510824ac9b82" - }, - "requirementRegistry": { - "path": "testing/end-to-end-comparisons/cases/prospect-research-workspace/controller/requirement-registry.json", - "sha256": "sha256:20460d405d422c12ce1faa9e954614044b1aef6a0ac9d1bdbe81ef61b54360ab" - }, - "executionContractTemplate": { - "path": "testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json", - "sha256": "sha256:6ce5985ef06809c1622d50564d0ca5a63b517c8386931303f4be435eaafacd7b" - }, - "oracle": { - "id": "prospect-research-workspace-oracles-v1", - "manifestPath": "testing/execution-comparisons/cases/prospect-research-workspace/controller/oracle-manifest.json", - "manifestSha256": "sha256:83398c66a41516ee9fc985724aa07dd88f4f3bae350670b80f1854970c92e900" - }, - "budgets": { - "elicitation": { - "qualifyingQuestions": 10, - "targetTurns": 20, - "elapsedMinutes": 60, - "mechanicalInterventions": 2 - }, - "execution": { - "elapsedMinutes": 120, - "mechanicalInterventions": 2, - "substantiveHumanInterventions": 0 - } - }, - "actorRecipes": { - "elicitation": "agent-as-user-comparison/v1", - "execution": { - "brunch": "brunch-empty-dir/v1", - "claude_code": "claude-code-empty-dir/v1" - } - }, - "specSources": ["brunch_spec", "claude_spec"], - "executors": ["brunch", "claude_code"] -} diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json b/testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json index e49e82bc5..fe8e32a75 100644 --- a/testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json +++ b/testing/execution-comparisons/cases/prospect-research-workspace/public-contract.json @@ -3,7 +3,7 @@ "case": { "id": "prospect-research-workspace-v1", "specification": "spec.md", - "specificationSha256": "c31b9857949e5f15fb6c490b6d9127356e90c5c8bb20956a9e2080da97bf4547", + "specificationSha256": "4c4f5ddafa5489ebb8bf4ef701ece12f690f3138a5c22fca7fc0746dfa31652a", "provider": "anthropic", "model": "claude-opus-4-8", "product": "prospect_research_workspace", @@ -30,6 +30,17 @@ }, "acceptance": { "healthPath": "/api/health", + "statePath": "/api/state", + "projectsPath": "/api/projects", + "projectActionPathPattern": "^/api/projects/[0-9]+/(approve|research)$", + "prospectActionPathPattern": "^/api/prospects/[0-9]+/(approve|suppress|override)$", + "exportPath": "/api/export", + "responseStatuses": { + "unapprovedResearch": 409, + "reasonlessOverride": 400, + "providerFailure": 503 + }, + "sqliteTables": ["projects", "runs", "prospects", "provenance", "suppressions", "decisions"], "executionTerminal": "promotion_prepared" }, "accessibility": { diff --git a/testing/execution-comparisons/cases/prospect-research-workspace/spec.md b/testing/execution-comparisons/cases/prospect-research-workspace/spec.md index 07ec7210c..ae755825e 100644 --- a/testing/execution-comparisons/cases/prospect-research-workspace/spec.md +++ b/testing/execution-comparisons/cases/prospect-research-workspace/spec.md @@ -10,7 +10,7 @@ The application researches and qualifies prospects. It does not generate outreac ## Core Workflow -1. The operator creates a research project with an ideal-customer profile, target roles, qualification criteria, and exclusions. +1. The operator creates a research project with a name and ideal-customer profile. 2. The operator approves the project before research can run. 3. A manual research run reads deterministic candidates from the configured Clay-compatible adapter. 4. The application normalizes people and companies, merges duplicates while retaining provenance, and applies suppressions. @@ -22,13 +22,13 @@ The application researches and qualifies prospects. It does not generate outreac - Required role, company-fit, and source-evidence fields must be present before qualification. - Provider confidence is metadata, not qualifying evidence. -- Person, company, domain, and email suppressions take precedence over qualification, approval, and later imports. +- Suppressing a prospect takes precedence over qualification, approval, export, and later imports of that prospect. - Overrides preserve the prior automated decision and record the operator's reason. -- Provider failure or partial delivery never silently becomes prospect rejection. +- Provider failure never silently becomes prospect rejection. ## Technical Boundary -Use a React and TypeScript frontend, Node.js and TypeScript backend, SQLite durable store, and server-side Pi/Clay-compatible adapter interfaces. The scored mode uses deterministic local fixtures and performs no runtime network access. Build tool, server framework, ORM, router, component library, and CSS system remain implementation choices. +Use a React and TypeScript frontend, Node.js and TypeScript backend, SQLite durable store, and server-side Pi/Clay-compatible adapter interfaces. The regression mode uses deterministic local fixtures and performs no runtime network access. The public contract fixes the HTTP and SQLite evidence surfaces used by the regression oracle; build tool, server framework, ORM, router, component library, and CSS system remain implementation choices. ## Success