From ed8e0794189b214398d835e1af828905f29fbc53 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:50:22 +0200 Subject: [PATCH 01/24] feat(lab): CL-07 task effectiveness evidence producer Add a bounded Lab-owned fabric-core synthetic-patch producer with exact-tree-diff verification, scratch sandbox, TaskSubjectV1 identity, and observation ingestion without a general Agent Fabric platform. --- .../001_pr_stack_status.md | 18 +- .../007_cl07_task_effectiveness.md | 246 ++++++++++ src/lab/fabric/constants.ts | 38 ++ src/lab/fabric/executor.ts | 343 ++++++++++++++ src/lab/fabric/index.ts | 9 + src/lab/fabric/manifest.ts | 200 ++++++++ src/lab/fabric/observe.ts | 218 +++++++++ src/lab/fabric/patch.ts | 71 +++ src/lab/fabric/scratch.ts | 195 ++++++++ src/lab/fabric/subject.ts | 95 ++++ src/lab/fabric/types.ts | 90 ++++ src/lab/fabric/verifier.ts | 114 +++++ src/lab/index.ts | 1 + src/lab/paths.ts | 2 + src/lab/query/catalog.ts | 30 ++ tests/lab-fabric-task.test.ts | 438 ++++++++++++++++++ 16 files changed, 2106 insertions(+), 2 deletions(-) create mode 100644 devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md create mode 100644 src/lab/fabric/constants.ts create mode 100644 src/lab/fabric/executor.ts create mode 100644 src/lab/fabric/index.ts create mode 100644 src/lab/fabric/manifest.ts create mode 100644 src/lab/fabric/observe.ts create mode 100644 src/lab/fabric/patch.ts create mode 100644 src/lab/fabric/scratch.ts create mode 100644 src/lab/fabric/subject.ts create mode 100644 src/lab/fabric/types.ts create mode 100644 src/lab/fabric/verifier.ts create mode 100644 tests/lab-fabric-task.test.ts diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 6601fbb29e..ef8e8d84e5 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -25,7 +25,8 @@ independent review, blockers, and whether a later phase is authorized. | CL-03 | `feat/cl-03-live-route-probes` | `4f746d13799888ea0a8c7a111aa2ad61c2126ea0` | `003f7402f49bfe8dd710a7beba52f717051bfadf` | [#1352](https://github.com/lidge-jun/opencodex/pull/1352) | MERGED TO `dev` at `68c71a4e9cdf882d812f09fd94783a28749db629`; ACCEPTED/CLOSED | | CL-04 | `feat/cl-04-lab-read-surfaces` | `68c71a4e9cdf882d812f09fd94783a28749db629` | NOT RECORDED | [#1378](https://github.com/lidge-jun/opencodex/pull/1378) | MERGED TO `dev` at `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | | CL-05 | `feat/cl-05-compatibility-matrix-ui` | `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | `2a159b8b7` (Models tab placement) | [#1384](https://github.com/lidge-jun/opencodex/pull/1384) | MERGED TO `dev` at `1072b9c39c48a4982229131613ac300560740742` | -| CL-06 | `feat/cl-06-routing-profile-compatibility` | `1072b9c39c48a4982229131613ac300560740742` | `b96eae83f2a6d1654472aeeef84799070743aeb8` | [#1394](https://github.com/lidge-jun/opencodex/pull/1394) | DRAFT / PENDING CI | +| CL-06 | `feat/cl-06-routing-profile-compatibility` | `1072b9c39c48a4982229131613ac300560740742` | `b66e33ce7207d91014644d99317e456c992a3418` (merge) | [#1394](https://github.com/lidge-jun/opencodex/pull/1394) | MERGED TO `dev` at `b66e33ce7207d91014644d99317e456c992a3418`; ACCEPTED/CLOSED | +| CL-07 | `feat/cl-07-task-effectiveness-producer` | `b66e33ce7207d91014644d99317e456c992a3418` | NOT YET | pending | AUTHORIZED / IN PROGRESS; plan `007_cl07_task_effectiveness.md` | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -162,7 +163,20 @@ Claims cannot produce `PROBED`/`VERIFIED`. - CL-03: **ACCEPTED/CLOSED** via [#1352](https://github.com/lidge-jun/opencodex/pull/1352), merged to `dev` at `68c71a4e9cdf882d812f09fd94783a28749db629`. - CL-04: **MERGED** via #1378 at `d517161aeaa3a974ad3c0360ff0c97b03b4c4520`. - CL-05: **MERGED** via #1384 at `1072b9c39c48a4982229131613ac300560740742`. -- CL-06: **AUTHORIZED / IN PROGRESS** from CL-05 merge `1072b9c39c48a4982229131613ac300560740742`; branch `feat/cl-06-routing-profile-compatibility`; plan `006_cl06_routing_compatibility.md`. +- CL-06: **ACCEPTED/CLOSED** via [#1394](https://github.com/lidge-jun/opencodex/pull/1394), merged to `dev` at `b66e33ce7207d91014644d99317e456c992a3418`. +- CL-07: **AUTHORIZED / IN PROGRESS** from CL-06 merge `b66e33ce7207d91014644d99317e456c992a3418`; branch `feat/cl-07-task-effectiveness-producer`; plan `007_cl07_task_effectiveness.md`. +- CL-08: **not started**. + +## CL-07 start log + +- **Starting/base SHA:** `b66e33ce7207d91014644d99317e456c992a3418` (exact CL-06 merge #1394) +- **Branch:** `feat/cl-07-task-effectiveness-producer` +- **Scope:** bounded Lab-owned task-effectiveness producer for `fabric-core` / + `fabric-core.task.synthetic-patch@1.0.0`; `exact-tree-diff-v1` verifier; + scratch sandbox; observation ingestion with `executionMode: fabric`; catalog + discovery for the task layer. No general Agent Fabric product API. +- **Explicitly out of scope:** CL-08 automation/background execution; CL-06 + routing semantic changes; user repositories/prompts; arbitrary shell. ## CL-03 implementation log (2026-08-09) diff --git a/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md new file mode 100644 index 0000000000..e42482dead --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md @@ -0,0 +1,246 @@ +# CL-07 implementation record — Task-effectiveness evidence producer + +## Programme position + +| Field | Value | +|---|---| +| **Phase** | CL-07 | +| **Starting SHA** | `b66e33ce7207d91014644d99317e456c992a3418` (CL-06 merge #1394) | +| **Branch** | `feat/cl-07-task-effectiveness-producer` | +| **Target** | `lidge-jun/opencodex:dev` | +| **CL-08** | **Not started** (explicit non-goal) | + +## 0. Audit summary (repository reality) + +Inspected at starting SHA `b66e33ce7207d91014644d99317e456c992a3418`: + +| Area | Reality | +|---|---| +| `TaskSubjectV1` | Defined and validated (`src/lab/events/types.ts`, `validate.ts`); query DTO mapped | +| Agent Fabric product | **Absent**. Only `devlog/_plan/800_agent-fabric/` planning notes and CL-00 reserved consumer semantics | +| Lab ledger / artifacts / projection | Present; `task_effectiveness` already a legal evidence layer | +| CL-04 catalog | Protocol + live only — no fabric suite discovery yet | +| CL-05 matrix | Already lists `task_effectiveness` column; read-only | +| CL-06 routing | `requiredSuites.evidenceLayer` is **only** `protocol_conformance` \| `live_route_compatibility` | +| Live sandbox | Counter/env limits for probes; **not** a synthetic scratch-tree executor | +| `fabric-core` / `synthetic-patch` / `exact-tree-diff-v1` | Reserved in `020_scenario_contract_and_catalogue.md`; **no runtime implementation** | + +**Decision:** CL-07 implements a **bounded Lab-owned task producer** under `src/lab/fabric/`. It does **not** invent a general Agent Fabric platform, ACP/A2A orchestration, background grind, or user-worktree execution. + +## A. Frozen producer boundary + +### A.1 Subject identity + +Evidence uses existing `TaskSubjectV1` only (no alternate task identity): + +```text +subjectSchemaVersion 1 +subjectKind task +routeSubject RouteSubjectV1 # exact nested route from CL-03/CL-06 builders +taskClassId +taskClassVersion +taskFixtureDigest +verifierManifestDigest +fabricCompatibilityVersion +sandboxProfileDigest +``` + +`subjectId = subjectIdForSubject(taskSubject)` (existing digest helper). + +Any material change to route, task class, fixture, verifier, fabric compatibility version, or sandbox profile yields a distinct subject. Evidence must not reuse across subjects. + +### A.2 Producer outcome schema (`FabricTaskOutcomeV1`) + +Closed, fail-closed on unknown fields. Schema version `1`. + +Required fields: + +| Field | Role | +|---|---| +| `schemaVersion` | `1` | +| `taskClassId` / `taskClassVersion` | Exact class | +| `routeSubject` / `taskSubject` / `subjectId` | Exact identities | +| `taskFixtureDigest` | Fixture identity | +| `verifierManifestDigest` | Verifier identity | +| `fabricCompatibilityVersion` | Executor contract version | +| `sandboxProfileDigest` | Sandbox policy identity | +| `startedAt` / `completedAt` | Execution window (ms epoch integers) | +| `limits` | Declared ceilings | +| `usage` | Actual resource counters used for limit checks | +| `outcome` | Normalized: `pass` \| `fail` \| `blocked` \| `inconclusive` | +| `verifier` | Bounded `exact-tree-diff-v1` result | +| `failure` | Optional typed failure (`class`, `code`, `retryable`, `attribution`) | +| `artifactDigests` | Content-addressed digests only | +| `sourceRefs` | Optional safe IDs (request / route-decision / attempt) — never bodies | + +**Forbidden in outcome and artifacts:** user repositories, arbitrary file trees, prompts, hidden reasoning, credentials, env secrets, host paths, unrestricted logs/stdout/stderr, raw model transcripts, arbitrary response bodies. + +### A.3 V1 executable scope (only) + +| Item | Value | +|---|---| +| Suite | `fabric-core@1.0.0` | +| Scenario | `fabric-core.task.synthetic-patch@1.0.0` | +| Evidence layer | `task_effectiveness` | +| Execution mode | `fabric` | +| Verifier | `exact-tree-diff-v1` | +| Fixture | Lab-owned scratch with `src/value.txt` = UTF-8 `before\n`; requested final `after\n` | + +No additional task classes in this phase. + +### A.4 Limits (non-weakened) + +| Limit | Value | +|---|---| +| Files touched | 1 | +| Aggregate input/output | 64 KiB | +| Patch operations | 1 | +| Total timeout | 30 s | +| Inactivity timeout | 5 s | +| Aggregate artifacts | 1 MiB | +| Network | denied in scratch | +| User MCP | unavailable | +| Arbitrary shell | unavailable | +| User repository | unreachable | + +### A.5 Patch producer seam + +Execution does **not** embed a general coding agent. The executor accepts a closed `SyntheticPatchV1` (single `replace` of `src/value.txt` under byte bounds) from a caller-supplied producer function. + +- Tests inject deterministic patches. +- A future live route adapter may call a provider **outside** the scratch sandbox and return only a validated `SyntheticPatchV1`; raw prompts/responses never enter Lab storage. +- CL-07 does **not** ship automatic background execution (CL-08). + +## B. Verifier: `exact-tree-diff-v1` + +Deterministic, no LLM. + +1. Walk only the bounded scratch root (no follow). +2. Reject symlinks, special files, path traversal (`..`, absolute, drive prefixes), unexpected paths. +3. Sort repository-relative POSIX paths by UTF-8 bytes. +4. Hash/read allowed file bytes under bounds. +5. Pass iff the sole change is `src/value.txt: before\n → after\n` with no add/delete/rename. +6. Emit bounded structured result: `{ verifierId, manifestDigest, passed, pathSummaries[], reason? }`. + +Verifier manifest bytes participate in `verifierManifestDigest` → `TaskSubjectV1`. Behavior changes require a new digest; historical observations are never reinterpreted with current bytes. + +## C. Sandbox / execution boundary + +Minimal deny-by-default scratch executor: + +- Create ephemeral Lab-owned directory under the Lab paths tree (not the user repo). +- Materialize fixture files only. +- Apply at most one validated patch operation via direct file write (no shell). +- Enforce byte/time/inactivity ceilings. +- Cleanup on success, failure, and timeout. +- Freeze `sandboxProfileDigest` from a versioned sandbox profile object. + +Reuse patterns from `src/lab/artifacts/secure-fs.ts` / live sandbox env stripping where applicable; do not reuse live provider network transport inside the scratch. + +## D. Route identity + +Nested `RouteSubjectV1` must come from existing CL-03/CL-06 builders (`buildRouteSubjectV1` / policy subject helpers). Approximate `provider/model` strings are forbidden. Optional `sourceRefs` may cite `routeDecisionId` / attempt IDs without copying request content. + +## E. Lab ingestion + +`observationFromFabricOutcome` / `persistFabricOutcome`: + +- `evidenceLayer: "task_effectiveness"` +- `executionMode: "fabric"` +- Exact scenario/suite IDs + manifest digests + fixture digests +- Exact `TaskSubjectV1` + `subjectId` +- Assertions from verifier +- Typed `failure` attribution +- Bounded environment metadata +- Sanitized artifact refs via existing artifact store + +No second ledger, mutable task DB, or separate verdict store. JSONL canonical; SQLite rebuildable; verdicts projected. + +### E.1 Idempotency + +Event identity uses existing `assignEventId` content-addressing. Replaying the same outcome (same event payload identity) must not create contradictory evidence. A legitimate second attempt uses a distinct `attempt` / timing / outcome payload and remains distinct evidence. + +## F. Verdict / failure mapping + +Layer remains independent of protocol/live. + +| Condition | Observation outcome | Failure class → attribution | Projection effect | +|---|---|---|---| +| Verifier pass (current required task) | `pass` | — | May contribute to `PROBED`/`VERIFIED` per suite rule | +| Verifier semantic fail | `fail` | `behavioral_failure` → `route` | `DEGRADED` path per suite | +| Sandbox violation / containment | `blocked` or `inconclusive` | `sandbox_violation` → `harness`/`environment` | `BLOCKED` / none | +| Timeout / inactivity / budget | `blocked` | matching blocker → `environment` | none (not route incompatibility) | +| Harness/executor defect | `inconclusive` | `harness_failure` → `harness` | none | +| Malformed producer outcome | reject (no event) | — | — | +| Artifact integrity failure | reject or invalidate | `integrity_failure` → `harness` | invalidate | + +Registry/provider claims **cannot** create task-effectiveness `CLAIMED`. + +## G. Freshness / invalidation / artifacts + +Reuse existing Lab freshness, invalidation, sensitive purge, and rebuild. Missing historical contract artifacts make evidence unusable. + +Allowed artifacts: verifier summary, normalized tree-diff summary, bounded execution metadata, sanitized failure summary. Never file bodies, prompts, or raw logs. + +## H. Read surfaces / UI / CL-06 interaction + +- Extend CL-04 `queryLabCatalog` to discover `fabric-core` scenarios. +- Projection/query already accept `task_effectiveness` subjects. +- CL-05 matrix remains read-only; show task layer if already wired. +- **CL-06 unchanged:** do **not** add `task_effectiveness` to routing `requiredSuites`. `TaskSubjectV1` cannot be resolved pre-dispatch (task class/fixture/verifier/sandbox unknown until execution). Historical task evidence may appear in Lab reads; it must not silently alter production routing without a future explicit contract (out of CL-07 scope). + +## I. Rejected alternatives + +| Alternative | Why rejected | +|---|---| +| General Agent Fabric API / worktree runner | Out of scope; unsafe surface area | +| Shell-based patch apply | Violates no-arbitrary-shell; use direct write | +| Pre-dispatch routing on task suites | Subject unknown before execution | +| Numerical effectiveness score / leaderboard | Forbidden by CL-00 | +| Second ledger / mutable verified flag | Breaks Lab persistence authority | +| Embedding prompts in observations | Privacy violation | +| Weakening byte/time limits for tests | Contract non-negotiable | +| CL-08 auto/background probing | Explicit exclusion | + +## J. Security review checklist + +Scratch-root containment; symlink/path-traversal/special-file rejection; max files/bytes; artifact publication bounds; no network/MCP/shell in scratch; minimal env; no credentials/config exposure; cleanup; concurrent isolation; hostile patch paths. Fail closed. Adversarial tests required (§18 of programme request). + +## K. Implementation layout (proposed) + +```text +src/lab/fabric/ + constants.ts + types.ts + subject.ts + fixture.ts + sandbox-profile.ts + scratch.ts + patch.ts + verifier.ts + executor.ts + manifest.ts + observe.ts + index.ts +tests/lab-fabric-task.test.ts +``` + +## L. Validation plan + +Typecheck; focused fabric/subject/sandbox/verifier/observe/ledger/projection/read-surface tests; routing regressions proving CL-06 unchanged; privacy scan; hygiene; GUI only if touched. + +## M. Status + +- **Authorization:** IN PROGRESS from CL-06 merge `b66e33ce7207d91014644d99317e456c992a3418` +- **Implementation head:** pending commit on `feat/cl-07-task-effectiveness-producer` +- **Accepted head:** NOT YET +- **PR:** pending draft + +### Local validation (pre-PR) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-fabric-task.test.ts`: 20/20 passed +- `bun test tests/lab-read-surfaces.test.ts tests/routing-compatibility.test.ts`: passed +- `bun test tests/repo-hygiene.test.ts`: 11/11 passed +- `bun run privacy:scan`: passed +- CL-08: **not started** diff --git a/src/lab/fabric/constants.ts b/src/lab/fabric/constants.ts new file mode 100644 index 0000000000..43755c3d2a --- /dev/null +++ b/src/lab/fabric/constants.ts @@ -0,0 +1,38 @@ +/** CL-07 frozen fabric-core / synthetic-patch constants. */ + +export const FABRIC_SUITE_ID = "fabric-core"; +export const FABRIC_SUITE_VERSION = "1.0.0"; +export const FABRIC_SCENARIO_ID = "fabric-core.task.synthetic-patch"; +export const FABRIC_SCENARIO_VERSION = "1.0.0"; +export const FABRIC_TASK_CLASS_ID = "fabric-core.task.synthetic-patch"; +export const FABRIC_TASK_CLASS_VERSION = "1.0.0"; +export const FABRIC_COMPATIBILITY_VERSION = "ocx-fabric-producer-v1"; +export const FABRIC_VERIFIER_ID = "exact-tree-diff-v1"; +export const FABRIC_EVIDENCE_LAYER = "task_effectiveness" as const; + +export const SYNTHETIC_VALUE_PATH = "src/value.txt"; +export const SYNTHETIC_BEFORE_UTF8 = "before\n"; +export const SYNTHETIC_AFTER_UTF8 = "after\n"; + +export const FABRIC_LIMITS = Object.freeze({ + maxFiles: 1, + maxAggregateIoBytes: 64 * 1024, + maxPatchOperations: 1, + totalTimeoutMs: 30_000, + inactivityTimeoutMs: 5_000, + aggregateArtifactBytes: 1024 * 1024, + maxArtifacts: 8, +}); + +export const SANDBOX_PROFILE_V1 = Object.freeze({ + schemaVersion: 1, + profileId: "lab-fabric-scratch-v1", + allowNetwork: false, + allowUserMcp: false, + allowShell: false, + allowUserRepository: false, + followSymlinks: false, + allowSpecialFiles: false, + scratchOnly: true, + limits: FABRIC_LIMITS, +}); diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts new file mode 100644 index 0000000000..0a49363239 --- /dev/null +++ b/src/lab/fabric/executor.ts @@ -0,0 +1,343 @@ +import type { FailureRecordV1, RouteSubjectV1 } from "../events/types"; +import { labSandboxEnvironment, rejectProxyEnvironment } from "../live/sandbox"; +import { + FABRIC_COMPATIBILITY_VERSION, + FABRIC_LIMITS, + FABRIC_TASK_CLASS_ID, + FABRIC_TASK_CLASS_VERSION, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_VALUE_PATH, +} from "./constants"; +import { applySyntheticPatch, parseSyntheticPatchV1 } from "./patch"; +import { assertNotUnderUserRepo, createSyntheticScratch } from "./scratch"; +import { + buildTaskSubjectV1, + sandboxProfileDigest, + taskFixtureDigest, + taskSubjectId, + verifierManifestDigest, +} from "./subject"; +import type { + FabricTaskOutcomeV1, + FabricUsageV1, + SyntheticPatchProducer, + SyntheticPatchV1, +} from "./types"; +import { FabricTaskError } from "./types"; +import { verifyExactTreeDiffV1 } from "./verifier"; + +export interface RunFabricTaskOptions { + routeSubject: RouteSubjectV1; + producePatch: SyntheticPatchProducer; + configDir?: string; + /** Optional absolute user repo path used only for containment proofs in tests. */ + userRepoRoot?: string; + sourceRefs?: string[]; + now?: () => number; + /** Override clocks for inactivity/timeout tests. */ + sleep?: (ms: number) => Promise; +} + +function failureFromError(error: FabricTaskError): FailureRecordV1 { + return { + class: error.code, + code: error.code, + retryable: error.code === "timeout" || error.code === "inactivity_timeout" || error.code === "budget_exhausted", + attribution: error.attribution, + }; +} + +function outcomeFromFailure(failure: FailureRecordV1): FabricTaskOutcomeV1["outcome"] { + if (failure.class === "behavioral_failure") return "fail"; + if (failure.attribution === "harness") return "inconclusive"; + return "blocked"; +} + +/** Closed successful patch for deterministic harness injection. */ +export function correctSyntheticPatch(): SyntheticPatchV1 { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} + +/** + * Bounded Lab-owned fabric task executor for fabric-core.task.synthetic-patch@1.0.0. + * Does not provide a general Agent Fabric platform. + */ +export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions): Promise { + rejectProxyEnvironment(); + labSandboxEnvironment(); + + const startedAt = options.now?.() ?? Date.now(); + const fixtureDigest = taskFixtureDigest(); + const verifierDigest = verifierManifestDigest(); + const sandboxDigest = sandboxProfileDigest(); + const taskSubject = buildTaskSubjectV1({ + routeSubject: options.routeSubject, + taskFixtureDigest: fixtureDigest, + verifierManifestDigest: verifierDigest, + sandboxProfileDigest: sandboxDigest, + fabricCompatibilityVersion: FABRIC_COMPATIBILITY_VERSION, + }); + const subjectId = taskSubjectId(taskSubject); + + const usage: FabricUsageV1 = { + inputBytes: Buffer.byteLength("before\n", "utf8"), + outputBytes: 0, + patchOperations: 0, + filesTouched: 0, + artifactBytes: 0, + elapsedMs: 0, + inactiveMs: 0, + }; + + const scratch = createSyntheticScratch(options.configDir); + try { + if (options.userRepoRoot) { + assertNotUnderUserRepo(scratch.root, options.userRepoRoot); + } + + const deadline = startedAt + FABRIC_LIMITS.totalTimeoutMs; + const produceStarted = options.now?.() ?? Date.now(); + let patchRaw: unknown; + try { + const producePromise = Promise.resolve(options.producePatch({ + taskClassId: FABRIC_TASK_CLASS_ID, + taskClassVersion: FABRIC_TASK_CLASS_VERSION, + scratchRoot: scratch.root, + })); + patchRaw = await withTimeout(producePromise, FABRIC_LIMITS.totalTimeoutMs, FABRIC_LIMITS.inactivityTimeoutMs, options); + } catch (error) { + const completedAt = options.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - produceStarted); + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: { + verifierId: "exact-tree-diff-v1", + manifestDigest: verifierDigest, + passed: false, + pathSummaries: [], + reason: error.code, + }, + failure, + sourceRefs: options.sourceRefs, + }); + } + throw error; + } + + if ((options.now?.() ?? Date.now()) > deadline) { + throw new FabricTaskError("total timeout exceeded", "timeout", "environment"); + } + + const patch = parseSyntheticPatchV1(patchRaw); + const applied = applySyntheticPatch(scratch.root, patch); + usage.outputBytes = applied.bytesWritten; + usage.patchOperations = applied.patchOperations; + usage.filesTouched = applied.filesTouched; + + const verifier = verifyExactTreeDiffV1(scratch.root); + const completedAt = options.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + + if (verifier.passed) { + return sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: "pass", + verifier, + sourceRefs: options.sourceRefs, + }); + } + + const failure: FailureRecordV1 = { + class: "behavioral_failure", + code: verifier.reason ?? "verifier_failed", + retryable: false, + attribution: "route", + }; + return sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: "fail", + verifier, + failure, + sourceRefs: options.sourceRefs, + }); + } catch (error) { + const completedAt = options.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: { + verifierId: "exact-tree-diff-v1", + manifestDigest: verifierDigest, + passed: false, + pathSummaries: [], + reason: error.code, + }, + failure, + sourceRefs: options.sourceRefs, + }); + } + const failure: FailureRecordV1 = { + class: "harness_failure", + code: "harness_failure", + retryable: false, + attribution: "harness", + }; + return sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: "inconclusive", + verifier: { + verifierId: "exact-tree-diff-v1", + manifestDigest: verifierDigest, + passed: false, + pathSummaries: [], + reason: "harness_failure", + }, + failure, + sourceRefs: options.sourceRefs, + }); + } finally { + scratch.cleanup(); + } +} + +function sealOutcome(input: { + taskSubject: FabricTaskOutcomeV1["taskSubject"]; + subjectId: string; + fixtureDigest: string; + verifierDigest: string; + sandboxDigest: string; + startedAt: number; + completedAt: number; + usage: FabricUsageV1; + outcome: FabricTaskOutcomeV1["outcome"]; + verifier: FabricTaskOutcomeV1["verifier"]; + failure?: FailureRecordV1; + sourceRefs?: string[]; +}): FabricTaskOutcomeV1 { + const sealed: FabricTaskOutcomeV1 = { + schemaVersion: 1, + taskClassId: FABRIC_TASK_CLASS_ID, + taskClassVersion: FABRIC_TASK_CLASS_VERSION, + routeSubject: input.taskSubject.routeSubject, + taskSubject: input.taskSubject, + subjectId: input.subjectId, + taskFixtureDigest: input.fixtureDigest, + verifierManifestDigest: input.verifierDigest, + fabricCompatibilityVersion: FABRIC_COMPATIBILITY_VERSION, + sandboxProfileDigest: input.sandboxDigest, + startedAt: input.startedAt, + completedAt: input.completedAt, + limits: { ...FABRIC_LIMITS }, + usage: { ...input.usage }, + outcome: input.outcome, + verifier: { + ...input.verifier, + pathSummaries: [...input.verifier.pathSummaries], + }, + ...(input.failure ? { failure: { ...input.failure } } : {}), + artifactDigests: [], + ...(input.sourceRefs ? { sourceRefs: [...input.sourceRefs] } : {}), + }; + return sealed; +} + +async function withTimeout( + promise: Promise, + totalMs: number, + inactivityMs: number, + options: RunFabricTaskOptions, +): Promise { + let settled = false; + const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + return await new Promise((resolve, reject) => { + const totalTimer = setTimeout(() => { + if (!settled) { + settled = true; + reject(new FabricTaskError("total timeout exceeded", "timeout", "environment")); + } + }, totalMs); + const inactivityTimer = setTimeout(() => { + if (!settled) { + settled = true; + reject(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); + } + }, inactivityMs); + void sleep(0); + promise.then( + (value) => { + if (settled) return; + settled = true; + clearTimeout(totalTimer); + clearTimeout(inactivityTimer); + resolve(value); + }, + (error) => { + if (settled) return; + settled = true; + clearTimeout(totalTimer); + clearTimeout(inactivityTimer); + reject(error); + }, + ); + }); +} + +/** Explicit denials for capabilities that must remain unavailable in the scratch producer. */ +export function fabricScratchCapabilityProbe(): { + network: false; + userMcp: false; + arbitraryShell: false; + userRepository: false; +} { + return { + network: false, + userMcp: false, + arbitraryShell: false, + userRepository: false, + }; +} diff --git a/src/lab/fabric/index.ts b/src/lab/fabric/index.ts new file mode 100644 index 0000000000..31e9712137 --- /dev/null +++ b/src/lab/fabric/index.ts @@ -0,0 +1,9 @@ +export * from "./constants"; +export * from "./types"; +export * from "./subject"; +export * from "./scratch"; +export * from "./patch"; +export * from "./verifier"; +export * from "./executor"; +export * from "./manifest"; +export * from "./observe"; diff --git a/src/lab/fabric/manifest.ts b/src/lab/fabric/manifest.ts new file mode 100644 index 0000000000..defe6faf85 --- /dev/null +++ b/src/lab/fabric/manifest.ts @@ -0,0 +1,200 @@ +import type { CaseRecord, VerificationRole } from "../conformance/types"; +import { fixtureDigest, scenarioManifestDigest, suiteManifestDigest } from "../digest"; +import { + FABRIC_EVIDENCE_LAYER, + FABRIC_LIMITS, + FABRIC_SCENARIO_ID, + FABRIC_SCENARIO_VERSION, + FABRIC_SUITE_ID, + FABRIC_SUITE_VERSION, + FABRIC_TASK_CLASS_ID, +} from "./constants"; +import { verifierManifestDigest } from "./subject"; + +export interface FabricCaseAuthority { + schemaVersion: number; + sourceCommit: string; + assertionDslVersion: string; + evidenceSchemaVersion: string; + manifestDefaults: { + version: string; + suiteVersion: string; + evidenceLayer: typeof FABRIC_EVIDENCE_LAYER; + verificationRole: VerificationRole; + freshness: { maxAgeMs: number | null }; + executionLimits: Record; + artifactPolicy: Record; + }; + cases: CaseRecord[]; +} + +const FABRIC_SOURCE_COMMIT = "b66e33ce7207d91014644d99317e456c992a3418"; +const FIXTURE_BYTES_UTF8 = JSON.stringify({ path: "src/value.txt", before: "before\n", after: "after\n" }); + +export function loadFabricCaseAuthority(): FabricCaseAuthority { + const fixtureBytes = new TextEncoder().encode(FIXTURE_BYTES_UTF8); + const digest = fixtureDigest(fixtureBytes); + return { + schemaVersion: 1, + sourceCommit: FABRIC_SOURCE_COMMIT, + assertionDslVersion: "1.0.0", + evidenceSchemaVersion: "1.0.0", + manifestDefaults: { + version: FABRIC_SCENARIO_VERSION, + suiteVersion: FABRIC_SUITE_VERSION, + evidenceLayer: FABRIC_EVIDENCE_LAYER, + verificationRole: "required", + freshness: { maxAgeMs: 2_592_000_000 }, + executionLimits: { + totalTimeoutMs: FABRIC_LIMITS.totalTimeoutMs, + inactivityTimeoutMs: FABRIC_LIMITS.inactivityTimeoutMs, + maxInputBytes: FABRIC_LIMITS.maxAggregateIoBytes, + maxOutputBytes: FABRIC_LIMITS.maxAggregateIoBytes, + maxArtifactBytes: FABRIC_LIMITS.aggregateArtifactBytes, + maxRequests: 0, + maxToolCalls: 0, + maxOutputTokens: null, + connectTimeoutMs: null, + firstByteTimeoutMs: null, + }, + artifactPolicy: { + retain: ["verifier_summary", "normalized_tree_diff", "execution_metadata"], + forbid: ["file_bodies", "prompts", "raw_logs"], + }, + }, + cases: [{ + id: FABRIC_SCENARIO_ID, + suite: FABRIC_SUITE_ID, + capability: FABRIC_TASK_CLASS_ID, + verificationRole: "required", + requirements: { + inboundProtocols: [], + upstreamProtocols: [], + surfaces: [], + requiredClaims: [], + requiredHarnessFeatures: ["fabric-scratch-v1"], + platforms: ["*"], + routePreconditions: ["exact-route-subject"], + }, + fixture: { + id: "fabric-core.task.synthetic-patch.fixture", + role: "adapter_vector", + mediaType: "application/json", + bytesUtf8: FIXTURE_BYTES_UTF8, + digest, + }, + assertions: [{ + id: "exact-tree-diff-pass", + operator: "equals", + selector: "/verifier/passed", + expected: true, + required: true, + }], + }], + }; +} + +export function discoverFabricScenarios( + authority: FabricCaseAuthority = loadFabricCaseAuthority(), + suites?: readonly string[], +): CaseRecord[] { + return authority.cases.filter((row) => !suites || suites.includes(row.suite)); +} + +export function expandFabricScenario( + caseRecord: CaseRecord, + authority: FabricCaseAuthority = loadFabricCaseAuthority(), +): Record { + const defaults = authority.manifestDefaults; + return { + schemaVersion: authority.schemaVersion, + id: caseRecord.id, + version: defaults.version, + suite: { + id: caseRecord.suite, + version: defaults.suiteVersion, + evidenceLayer: defaults.evidenceLayer, + }, + evidenceLayer: defaults.evidenceLayer, + capability: caseRecord.capability, + verificationRole: caseRecord.verificationRole ?? defaults.verificationRole, + requirements: caseRecord.requirements, + fixtures: [{ + id: caseRecord.fixture.id, + role: caseRecord.fixture.role, + mediaType: caseRecord.fixture.mediaType, + digest: caseRecord.fixture.digest, + byteLength: Buffer.byteLength(caseRecord.fixture.bytesUtf8, "utf8"), + syntheticMarker: "ocx-lab-synthetic-v1", + provenance: { + kind: "lab_authored", + authority: "007_cl07_task_effectiveness.md", + sourceCommit: authority.sourceCommit, + }, + }], + executionLimits: defaults.executionLimits, + assertions: caseRecord.assertions, + failureRules: [ + { + id: "verifier-fail", + match: ["behavioral_failure"], + classification: "behavioral_failure", + verdictEffect: "degraded", + retry: "never", + expected: false, + }, + { + id: "sandbox-block", + match: ["sandbox_violation", "timeout", "inactivity_timeout", "budget_exhausted"], + classification: "timeout", + verdictEffect: "none", + retry: "bounded", + expected: false, + }, + ], + artifactPolicy: defaults.artifactPolicy, + freshness: defaults.freshness, + verifierManifestDigest: verifierManifestDigest(), + }; +} + +export function expandFabricSuiteManifest( + suiteId: string = FABRIC_SUITE_ID, + authority: FabricCaseAuthority = loadFabricCaseAuthority(), +): Record { + const cases = authority.cases.filter((row) => row.suite === suiteId); + if (cases.length === 0) throw new Error(`unknown fabric suite ${suiteId}`); + const defaults = authority.manifestDefaults; + return { + schemaVersion: 1, + id: suiteId, + version: defaults.suiteVersion, + evidenceLayer: defaults.evidenceLayer, + capability: cases[0]!.capability, + assertionDslVersion: authority.assertionDslVersion, + evidenceSchemaVersion: authority.evidenceSchemaVersion, + freshness: defaults.freshness, + contradictionRule: "newest-required-observation-v1", + scenarios: cases.map((caseRecord) => ({ + id: caseRecord.id, + version: defaults.version, + role: caseRecord.verificationRole ?? defaults.verificationRole, + manifestDigest: scenarioManifestDigest(expandFabricScenario(caseRecord, authority)), + })), + verificationRule: "all-applicable-required-pass-v1", + }; +} + +export function fabricScenarioManifestDigest( + caseRecord: CaseRecord = loadFabricCaseAuthority().cases[0]!, + authority: FabricCaseAuthority = loadFabricCaseAuthority(), +): string { + return scenarioManifestDigest(expandFabricScenario(caseRecord, authority)); +} + +export function fabricSuiteManifestDigest( + suiteId: string = FABRIC_SUITE_ID, + authority: FabricCaseAuthority = loadFabricCaseAuthority(), +): string { + return suiteManifestDigest(expandFabricSuiteManifest(suiteId, authority)); +} diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts new file mode 100644 index 0000000000..4ec2230d60 --- /dev/null +++ b/src/lab/fabric/observe.ts @@ -0,0 +1,218 @@ +import { existsSync } from "node:fs"; +import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; +import { sanitizeDiagnostic, truncateUtf8 } from "../artifacts/sanitize"; +import { + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + LAB_PRODUCER_VERSION, + OBSERVATION_LIMIT_NAMES, +} from "../constants"; +import { fixtureDigest } from "../digest"; +import type { ObservationEvent } from "../events/types"; +import { assignEventId } from "../events/validate"; +import { appendLabEvent, replayLabLedger } from "../ledger/store"; +import { ensureLabDirs } from "../paths"; +import { + FABRIC_EVIDENCE_LAYER, + FABRIC_SCENARIO_ID, + FABRIC_SCENARIO_VERSION, + FABRIC_SUITE_ID, + FABRIC_SUITE_VERSION, +} from "./constants"; +import { + expandFabricScenario, + expandFabricSuiteManifest, + fabricScenarioManifestDigest, + fabricSuiteManifestDigest, + loadFabricCaseAuthority, +} from "./manifest"; +import type { FabricTaskOutcomeV1 } from "./types"; +import { FabricTaskError } from "./types"; + +export interface PersistFabricOptions { + configDir?: string; + recordedAt?: number; + producerVersion?: string; + artifactStore?: ArtifactStore; + attempt?: number; +} + +export interface PersistedFabricObservation { + event: ObservationEvent; + ledgerPath: string; +} + +const OUTCOME_KEYS = new Set([ + "schemaVersion", + "taskClassId", + "taskClassVersion", + "routeSubject", + "taskSubject", + "subjectId", + "taskFixtureDigest", + "verifierManifestDigest", + "fabricCompatibilityVersion", + "sandboxProfileDigest", + "startedAt", + "completedAt", + "limits", + "usage", + "outcome", + "verifier", + "failure", + "artifactDigests", + "sourceRefs", +]); + +export function assertFabricOutcomeV1(raw: unknown): FabricTaskOutcomeV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new FabricTaskError("malformed producer outcome", "malformed_producer_outcome", "harness"); + } + const obj = raw as Record; + for (const key of Object.keys(obj)) { + if (!OUTCOME_KEYS.has(key)) { + throw new FabricTaskError(`unknown outcome field ${key}`, "malformed_producer_outcome", "harness"); + } + } + if (obj.schemaVersion !== 1) { + throw new FabricTaskError("schemaVersion must be 1", "malformed_producer_outcome", "harness"); + } + if (obj.taskSubject && typeof obj.taskSubject === "object" && !Array.isArray(obj.taskSubject)) { + const kind = (obj.taskSubject as { subjectKind?: unknown }).subjectKind; + if (kind !== "task") { + throw new FabricTaskError("task layer rejects non-task subjects", "layer_subject_mismatch", "harness"); + } + } + return raw as FabricTaskOutcomeV1; +} + +export function observationFromFabricOutcome( + outcomeRaw: unknown, + opts: PersistFabricOptions = {}, +): { event: ObservationEvent; artifacts: ReturnType[] } { + const outcome = assertFabricOutcomeV1(outcomeRaw); + if (outcome.taskSubject.subjectKind !== "task") { + throw new FabricTaskError("task layer requires TaskSubjectV1", "layer_subject_mismatch", "harness"); + } + if (outcome.routeSubject.subjectKind !== "route") { + throw new FabricTaskError("nested route subject required", "layer_subject_mismatch", "harness"); + } + if (!Number.isInteger(outcome.startedAt) || !Number.isInteger(outcome.completedAt) || outcome.completedAt < outcome.startedAt) { + throw new FabricTaskError("invalid execution timestamps", "malformed_producer_outcome", "harness"); + } + + const paths = ensureLabDirs(opts.configDir); + const ownsStore = !opts.artifactStore; + const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { + const authority = loadFabricCaseAuthority(); + const caseRecord = authority.cases[0]!; + const scenarioDigest = fabricScenarioManifestDigest(caseRecord, authority); + const suiteDigest = fabricSuiteManifestDigest(FABRIC_SUITE_ID, authority); + const expandedScenario = expandFabricScenario(caseRecord, authority); + const expandedSuite = expandFabricSuiteManifest(FABRIC_SUITE_ID, authority); + const fixtureBytes = new TextEncoder().encode(caseRecord.fixture.bytesUtf8); + const fixtureDigests = [fixtureDigest(fixtureBytes)]; + + const artifacts: ReturnType[] = []; + artifacts.push(store.put({ + artifactClass: "fixture", + payload: fixtureBytes, + expectedDigest: fixtureDigests[0], + mediaType: caseRecord.fixture.mediaType, + })); + artifacts.push(store.put({ + artifactClass: "scenario_manifest", + payload: expandedScenario, + expectedDigest: scenarioDigest, + })); + artifacts.push(store.put({ + artifactClass: "suite_manifest", + payload: expandedSuite, + expectedDigest: suiteDigest, + })); + artifacts.push(store.put({ + artifactClass: "verifier_summary", + payload: { + scenarioId: FABRIC_SCENARIO_ID, + passed: outcome.verifier.passed, + verifier: outcome.verifier, + usage: outcome.usage, + outcome: outcome.outcome, + }, + })); + + const limits: Record = {}; + for (const key of OBSERVATION_LIMIT_NAMES) { + if (key in authority.manifestDefaults.executionLimits) { + limits[key] = authority.manifestDefaults.executionLimits[key] ?? null; + } + } + + const recordedAt = opts.recordedAt ?? outcome.completedAt; + const eventWithoutId = { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "observation" as const, + recordedAt, + producer: LAB_PRODUCER, + producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, + evidenceLayer: FABRIC_EVIDENCE_LAYER, + scenarioId: FABRIC_SCENARIO_ID, + scenarioVersion: FABRIC_SCENARIO_VERSION, + scenarioManifestDigest: scenarioDigest, + suiteId: FABRIC_SUITE_ID, + suiteVersion: FABRIC_SUITE_VERSION, + suiteManifestDigest: suiteDigest, + fixtureDigests, + subject: outcome.taskSubject, + subjectId: outcome.subjectId, + startedAt: outcome.startedAt, + completedAt: outcome.completedAt, + executionMode: "fabric" as const, + attempt: opts.attempt ?? 1, + limits, + outcome: outcome.outcome, + assertions: [{ + id: "exact-tree-diff-pass", + operator: "equals", + required: true, + passed: outcome.verifier.passed, + expectedSummary: "pass", + observedSummary: truncateUtf8(sanitizeDiagnostic(outcome.verifier.reason ?? (outcome.verifier.passed ? "pass" : "fail")), 512), + }], + environment: { + runtime: { + platform: process.platform, + arch: process.arch, + bunVersion: process.versions.bun ?? Bun.version, + }, + }, + artifactRefs: artifacts, + ...(outcome.failure ? { failure: outcome.failure } : {}), + ...(outcome.sourceRefs ? { sourceRefs: [...outcome.sourceRefs] } : {}), + }; + return { event: assignEventId(eventWithoutId) as ObservationEvent, artifacts }; + } finally { + if (ownsStore) store.close(); + } +} + +export function persistFabricOutcome( + outcome: FabricTaskOutcomeV1, + opts: PersistFabricOptions = {}, +): PersistedFabricObservation { + const paths = ensureLabDirs(opts.configDir); + const ownsStore = !opts.artifactStore; + const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { + const { event } = observationFromFabricOutcome(outcome, { ...opts, artifactStore: store }); + const replay = existsSync(paths.ledgerPath) ? replayLabLedger(paths.ledgerPath) : null; + const alreadyPresent = replay?.events.some((row) => row.eventId === event.eventId) ?? false; + if (!alreadyPresent) { + appendLabEvent(paths.ledgerPath, event); + } + return { event, ledgerPath: paths.ledgerPath }; + } finally { + if (ownsStore) store.close(); + } +} diff --git a/src/lab/fabric/patch.ts b/src/lab/fabric/patch.ts new file mode 100644 index 0000000000..7a90319705 --- /dev/null +++ b/src/lab/fabric/patch.ts @@ -0,0 +1,71 @@ +import { FABRIC_LIMITS, SYNTHETIC_VALUE_PATH } from "./constants"; +import { writeScratchFileUtf8 } from "./scratch"; +import { FabricTaskError, type SyntheticPatchV1 } from "./types"; + +const ALLOWED_KEYS = new Set(["schemaVersion", "operations"]); +const OP_KEYS = new Set(["op", "path", "contentUtf8"]); + +export function parseSyntheticPatchV1(raw: unknown): SyntheticPatchV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new FabricTaskError("malformed producer outcome: patch must be object", "malformed_producer_outcome", "harness"); + } + const obj = raw as Record; + for (const key of Object.keys(obj)) { + if (!ALLOWED_KEYS.has(key)) { + throw new FabricTaskError(`malformed producer outcome: unknown field ${key}`, "malformed_producer_outcome", "harness"); + } + } + if (obj.schemaVersion !== 1) { + throw new FabricTaskError("malformed producer outcome: schemaVersion", "malformed_producer_outcome", "harness"); + } + if (!Array.isArray(obj.operations)) { + throw new FabricTaskError("malformed producer outcome: operations", "malformed_producer_outcome", "harness"); + } + if (obj.operations.length !== 1) { + throw new FabricTaskError("exactly one patch operation is required", "budget_exhausted", "route"); + } + const opRaw = obj.operations[0]; + if (!opRaw || typeof opRaw !== "object" || Array.isArray(opRaw)) { + throw new FabricTaskError("malformed producer outcome: operation", "malformed_producer_outcome", "harness"); + } + const op = opRaw as Record; + for (const key of Object.keys(op)) { + if (!OP_KEYS.has(key)) { + throw new FabricTaskError(`malformed producer outcome: unknown op field ${key}`, "malformed_producer_outcome", "harness"); + } + } + if (op.op !== "replace" || typeof op.path !== "string" || typeof op.contentUtf8 !== "string") { + throw new FabricTaskError("malformed producer outcome: replace shape", "malformed_producer_outcome", "harness"); + } + if (op.path !== SYNTHETIC_VALUE_PATH) { + throw new FabricTaskError("patch path must be src/value.txt", "behavioral_failure", "route"); + } + const contentBytes = Buffer.byteLength(op.contentUtf8, "utf8"); + if (contentBytes > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("patch content exceeds io budget", "budget_exhausted", "environment"); + } + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: op.contentUtf8 }], + }; +} + +export function applySyntheticPatch(scratchRoot: string, patch: SyntheticPatchV1): { bytesWritten: number; filesTouched: number; patchOperations: number } { + if (patch.operations.length > FABRIC_LIMITS.maxPatchOperations) { + throw new FabricTaskError("too many patch operations", "budget_exhausted", "environment"); + } + let bytesWritten = 0; + for (const operation of patch.operations) { + bytesWritten += writeScratchFileUtf8( + scratchRoot, + operation.path, + operation.contentUtf8, + FABRIC_LIMITS.maxAggregateIoBytes, + ); + } + return { + bytesWritten, + filesTouched: patch.operations.length, + patchOperations: patch.operations.length, + }; +} diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts new file mode 100644 index 0000000000..d9b1a2f8d7 --- /dev/null +++ b/src/lab/fabric/scratch.ts @@ -0,0 +1,195 @@ +import { + chmodSync, + closeSync, + constants as fsConstants, + existsSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + rmSync, + writeSync, + type Stats, +} from "node:fs"; +import { dirname, join, posix, relative, resolve, sep } from "node:path"; +import { randomBytes } from "node:crypto"; +import { ensureLabDirs, labScratchDir } from "../paths"; +import { FABRIC_LIMITS, SYNTHETIC_BEFORE_UTF8, SYNTHETIC_VALUE_PATH } from "./constants"; +import { FabricTaskError } from "./types"; + +const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; + +function ensureRestrictedDir(dir: string): void { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + if (process.platform === "win32") return; + const mode = lstatSync(dir).mode & 0o777; + if (mode !== 0o700) chmodSync(dir, 0o700); +} + +function assertRegularFile(stats: Stats, label: string): void { + if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { + throw new FabricTaskError(`${label} must be a regular file`, "sandbox_violation", "harness"); + } +} + +/** Normalize and reject traversal / absolute / Windows drive / NUL paths. */ +export function assertSafeRelativePosixPath(raw: string): string { + if (typeof raw !== "string" || raw.length === 0) { + throw new FabricTaskError("path required", "sandbox_violation", "harness"); + } + if (raw.includes("\0") || raw.includes("\\")) { + throw new FabricTaskError("path contains forbidden characters", "sandbox_violation", "harness"); + } + if (raw.startsWith("/") || /^[A-Za-z]:/.test(raw)) { + throw new FabricTaskError("absolute paths are forbidden", "sandbox_violation", "harness"); + } + const normalized = posix.normalize(raw); + if (normalized === "." || normalized.startsWith("../") || normalized === ".." || normalized.includes("/../")) { + throw new FabricTaskError("path traversal is forbidden", "sandbox_violation", "harness"); + } + if (normalized.split("/").some((part) => part === ".." || part === "")) { + throw new FabricTaskError("invalid path segments", "sandbox_violation", "harness"); + } + return normalized; +} + +export function resolveInsideScratch(scratchRoot: string, relativePath: string): string { + const safe = assertSafeRelativePosixPath(relativePath); + const root = resolve(scratchRoot); + const target = resolve(root, ...safe.split("/")); + if (target !== root && !target.startsWith(root + sep)) { + throw new FabricTaskError("path escapes scratch root", "sandbox_violation", "harness"); + } + return target; +} + +export interface ScratchTree { + root: string; + cleanup: () => void; +} + +export function createSyntheticScratch(configDir?: string): ScratchTree { + ensureLabDirs(configDir); + const base = labScratchDir(configDir); + ensureRestrictedDir(base); + const root = join(base, `fabric-${Date.now().toString(36)}-${randomBytes(6).toString("hex")}`); + ensureRestrictedDir(root); + const valuePath = resolveInsideScratch(root, SYNTHETIC_VALUE_PATH); + ensureRestrictedDir(dirname(valuePath)); + const bytes = Buffer.from(SYNTHETIC_BEFORE_UTF8, "utf8"); + if (bytes.byteLength > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("fixture exceeds io budget", "budget_exhausted", "harness"); + } + const fd = openSync(valuePath, openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL)); + try { + writeSync(fd, bytes); + } finally { + closeSync(fd); + } + return { + root, + cleanup: () => { + try { + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + } catch { + // best-effort cleanup + } + }, + }; +} + +function openFlags(base: number): number { + if (typeof O_NOFOLLOW === "number" && O_NOFOLLOW !== 0) return base | O_NOFOLLOW; + return base; +} + +export interface WalkedFile { + relativePosix: string; + absolute: string; + byteLength: number; +} + +/** No-follow walk; rejects symlinks and special files. */ +export function walkScratchFiles(scratchRoot: string): WalkedFile[] { + const root = resolve(scratchRoot); + const out: WalkedFile[] = []; + const stack: string[] = [root]; + while (stack.length > 0) { + const current = stack.pop()!; + const stats = lstatSync(current); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + if (stats.isDirectory()) { + const entries = readdirSync(current, { withFileTypes: true }); + for (const entry of entries) { + stack.push(join(current, entry.name)); + } + continue; + } + assertRegularFile(stats, current); + const rel = relative(root, current).split(sep).join("/"); + const safe = assertSafeRelativePosixPath(rel); + out.push({ relativePosix: safe, absolute: current, byteLength: stats.size }); + } + out.sort((a, b) => { + const left = Buffer.from(a.relativePosix, "utf8"); + const right = Buffer.from(b.relativePosix, "utf8"); + const n = Math.min(left.length, right.length); + for (let i = 0; i < n; i++) { + const d = left[i]! - right[i]!; + if (d !== 0) return d; + } + return left.length - right.length; + }); + return out; +} + +export function readScratchFileUtf8(scratchRoot: string, relativePath: string, maxBytes: number): string { + const absolute = resolveInsideScratch(scratchRoot, relativePath); + if (!existsSync(absolute)) { + throw new FabricTaskError(`missing file ${relativePath}`, "sandbox_violation", "harness"); + } + const stats = lstatSync(absolute); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + assertRegularFile(stats, relativePath); + if (stats.size > maxBytes) { + throw new FabricTaskError("file exceeds io budget", "budget_exhausted", "environment"); + } + return readFileSync(absolute, "utf8"); +} + +export function writeScratchFileUtf8(scratchRoot: string, relativePath: string, contentUtf8: string, maxBytes: number): number { + const absolute = resolveInsideScratch(scratchRoot, relativePath); + const bytes = Buffer.from(contentUtf8, "utf8"); + if (bytes.byteLength > maxBytes) { + throw new FabricTaskError("write exceeds io budget", "budget_exhausted", "environment"); + } + ensureRestrictedDir(dirname(absolute)); + if (existsSync(absolute)) { + const stats = lstatSync(absolute); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + assertRegularFile(stats, relativePath); + } + const fd = openSync(absolute, openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC)); + try { + writeSync(fd, bytes); + } finally { + closeSync(fd); + } + return bytes.byteLength; +} + +/** Prove the resolved path cannot escape into a caller-provided user repository root. */ +export function assertNotUnderUserRepo(scratchRoot: string, userRepoRoot: string): void { + const scratch = resolve(scratchRoot); + const repo = resolve(userRepoRoot); + if (scratch === repo || scratch.startsWith(repo + sep)) { + throw new FabricTaskError("scratch must not live inside user repository", "sandbox_violation", "harness"); + } +} diff --git a/src/lab/fabric/subject.ts b/src/lab/fabric/subject.ts new file mode 100644 index 0000000000..2c94bc15f2 --- /dev/null +++ b/src/lab/fabric/subject.ts @@ -0,0 +1,95 @@ +import { domainHash, jcsStringify, subjectIdForSubject } from "../digest"; +import type { RouteSubjectV1, TaskSubjectV1 } from "../events/types"; +import { freezeRouteSubject } from "../subject/route-subject"; +import { + FABRIC_COMPATIBILITY_VERSION, + FABRIC_TASK_CLASS_ID, + FABRIC_TASK_CLASS_VERSION, + SANDBOX_PROFILE_V1, +} from "./constants"; + +const SANDBOX_DOMAIN = "ocx-lab:fabric-sandbox-profile:v1"; +const VERIFIER_DOMAIN = "ocx-lab:fabric-verifier-manifest:v1"; +const FIXTURE_DOMAIN = "ocx-lab:fabric-task-fixture:v1"; + +export function sandboxProfileDigest(profile: unknown = SANDBOX_PROFILE_V1): string { + return domainHash(SANDBOX_DOMAIN, jcsStringify(profile)); +} + +export function verifierManifestObject(): Record { + return { + schemaVersion: 1, + verifierId: "exact-tree-diff-v1", + sort: "utf8-bytes-posix-path", + rejectSymlinks: true, + rejectSpecialFiles: true, + rejectPathTraversal: true, + allowAdds: false, + allowDeletes: false, + allowRenames: false, + requiredChange: { + path: "src/value.txt", + beforeUtf8: "before\n", + afterUtf8: "after\n", + }, + }; +} + +export function verifierManifestDigest(manifest: unknown = verifierManifestObject()): string { + return domainHash(VERIFIER_DOMAIN, jcsStringify(manifest)); +} + +export function taskFixtureObject(): Record { + return { + schemaVersion: 1, + taskClassId: FABRIC_TASK_CLASS_ID, + taskClassVersion: FABRIC_TASK_CLASS_VERSION, + files: [ + { + path: "src/value.txt", + contentUtf8: "before\n", + }, + ], + requestedFinal: [ + { + path: "src/value.txt", + contentUtf8: "after\n", + }, + ], + }; +} + +export function taskFixtureDigest(fixture: unknown = taskFixtureObject()): string { + return domainHash(FIXTURE_DOMAIN, jcsStringify(fixture)); +} + +export function buildTaskSubjectV1(input: { + routeSubject: RouteSubjectV1; + taskClassId?: string; + taskClassVersion?: string; + taskFixtureDigest?: string; + verifierManifestDigest?: string; + fabricCompatibilityVersion?: string; + sandboxProfileDigest?: string; +}): TaskSubjectV1 { + const routeSubject = freezeRouteSubject(input.routeSubject); + const subject: TaskSubjectV1 = { + subjectSchemaVersion: 1, + subjectKind: "task", + routeSubject, + taskClassId: input.taskClassId ?? FABRIC_TASK_CLASS_ID, + taskClassVersion: input.taskClassVersion ?? FABRIC_TASK_CLASS_VERSION, + taskFixtureDigest: input.taskFixtureDigest ?? taskFixtureDigest(), + verifierManifestDigest: input.verifierManifestDigest ?? verifierManifestDigest(), + fabricCompatibilityVersion: input.fabricCompatibilityVersion ?? FABRIC_COMPATIBILITY_VERSION, + sandboxProfileDigest: input.sandboxProfileDigest ?? sandboxProfileDigest(), + }; + return Object.freeze({ + ...subject, + routeSubject: freezeRouteSubject(subject.routeSubject), + }); +} + +export function taskSubjectId(subject: TaskSubjectV1): string { + return subjectIdForSubject(subject); +} diff --git a/src/lab/fabric/types.ts b/src/lab/fabric/types.ts new file mode 100644 index 0000000000..951cae7a3b --- /dev/null +++ b/src/lab/fabric/types.ts @@ -0,0 +1,90 @@ +import type { ObservationOutcome } from "../constants"; +import type { FailureRecordV1, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; + +export type FabricOutcomeKind = ObservationOutcome; + +export interface FabricLimitsV1 { + maxFiles: number; + maxAggregateIoBytes: number; + maxPatchOperations: number; + totalTimeoutMs: number; + inactivityTimeoutMs: number; + aggregateArtifactBytes: number; + maxArtifacts: number; +} + +export interface SyntheticPatchOperationV1 { + op: "replace"; + path: string; + contentUtf8: string; +} + +export interface SyntheticPatchV1 { + schemaVersion: 1; + operations: SyntheticPatchOperationV1[]; +} + +export interface FabricUsageV1 { + inputBytes: number; + outputBytes: number; + patchOperations: number; + filesTouched: number; + artifactBytes: number; + elapsedMs: number; + inactiveMs: number; +} + +export interface ExactTreeDiffPathSummaryV1 { + path: string; + kind: "unchanged" | "modified" | "added" | "deleted" | "rejected"; + beforeDigest?: string; + afterDigest?: string; + reason?: string; +} + +export interface ExactTreeDiffResultV1 { + verifierId: string; + manifestDigest: string; + passed: boolean; + pathSummaries: ExactTreeDiffPathSummaryV1[]; + reason?: string; +} + +export interface FabricTaskOutcomeV1 { + schemaVersion: 1; + taskClassId: string; + taskClassVersion: string; + routeSubject: RouteSubjectV1; + taskSubject: TaskSubjectV1; + subjectId: string; + taskFixtureDigest: string; + verifierManifestDigest: string; + fabricCompatibilityVersion: string; + sandboxProfileDigest: string; + startedAt: number; + completedAt: number; + limits: FabricLimitsV1; + usage: FabricUsageV1; + outcome: FabricOutcomeKind; + verifier: ExactTreeDiffResultV1; + failure?: FailureRecordV1; + artifactDigests: string[]; + sourceRefs?: string[]; +} + +export type SyntheticPatchProducer = (ctx: { + taskClassId: string; + taskClassVersion: string; + scratchRoot: string; +}) => SyntheticPatchV1 | Promise; + +export class FabricTaskError extends Error { + override readonly name = "FabricTaskError"; + constructor( + message: string, + readonly code: string, + readonly attribution: FailureRecordV1["attribution"] = "harness", + ) { + super(message); + } +} diff --git a/src/lab/fabric/verifier.ts b/src/lab/fabric/verifier.ts new file mode 100644 index 0000000000..6b5bddb3b7 --- /dev/null +++ b/src/lab/fabric/verifier.ts @@ -0,0 +1,114 @@ +import { createHash } from "node:crypto"; +import { + FABRIC_LIMITS, + FABRIC_VERIFIER_ID, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, +} from "./constants"; +import { readScratchFileUtf8, walkScratchFiles } from "./scratch"; +import { verifierManifestDigest } from "./subject"; +import type { ExactTreeDiffResultV1 } from "./types"; +import { FabricTaskError } from "./types"; + +function sha256Utf8(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +/** + * Deterministic exact-tree-diff-v1 verifier. No LLM. Walks only the scratch tree. + */ +export function verifyExactTreeDiffV1(scratchRoot: string): ExactTreeDiffResultV1 { + const manifestDigest = verifierManifestDigest(); + try { + const files = walkScratchFiles(scratchRoot); + let aggregate = 0; + for (const file of files) { + aggregate += file.byteLength; + if (aggregate > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("tree exceeds io budget", "budget_exhausted", "environment"); + } + } + if (files.length !== 1 || files[0]!.relativePosix !== SYNTHETIC_VALUE_PATH) { + const unexpected = files + .filter((row) => row.relativePosix !== SYNTHETIC_VALUE_PATH) + .map((row) => row.relativePosix); + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [ + ...unexpected.map((path) => ({ + path, + kind: "added" as const, + reason: "unexpected_file", + })), + ...(files.some((row) => row.relativePosix === SYNTHETIC_VALUE_PATH) + ? [] + : [{ path: SYNTHETIC_VALUE_PATH, kind: "deleted" as const, reason: "missing_required_file" }]), + ], + reason: files.length === 0 ? "deleted_required_file" : "unexpected_tree_shape", + }; + } + const after = readScratchFileUtf8(scratchRoot, SYNTHETIC_VALUE_PATH, FABRIC_LIMITS.maxAggregateIoBytes); + const beforeDigest = sha256Utf8(SYNTHETIC_BEFORE_UTF8); + const afterDigest = sha256Utf8(after); + const expectedDigest = sha256Utf8(SYNTHETIC_AFTER_UTF8); + if (after === SYNTHETIC_BEFORE_UTF8) { + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "unchanged", + beforeDigest, + afterDigest, + reason: "unchanged", + }], + reason: "unchanged_file", + }; + } + if (after !== SYNTHETIC_AFTER_UTF8) { + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "modified", + beforeDigest, + afterDigest, + reason: "wrong_final_bytes", + }], + reason: `expected_digest_${expectedDigest}`, + }; + } + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: true, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "modified", + beforeDigest, + afterDigest, + }], + }; + } catch (error) { + if (error instanceof FabricTaskError) { + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "rejected", + reason: error.code, + }], + reason: error.code, + }; + } + throw error; + } +} diff --git a/src/lab/index.ts b/src/lab/index.ts index b410b147ff..3020e556f4 100644 --- a/src/lab/index.ts +++ b/src/lab/index.ts @@ -18,6 +18,7 @@ export * from "./ledger/artifact-refs"; export * from "./projection/verification"; export * from "./observe/from-conformance"; export * from "./observe/from-live"; +export * from "./fabric"; export * from "./live/manifest"; export * from "./live/runner"; export * from "./live/executor"; diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 068db32033..dfa51f4eee 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -54,6 +54,8 @@ export function ensureLabDirs(configDir = getConfigDir()): { const exportDir = labExportDir(configDir); ensureRestrictedDir(root); ensureRestrictedDir(artifactsDir); + ensureRestrictedDir(scratchDir); + ensureRestrictedDir(exportDir); return { root, ledgerPath: labLedgerPath(configDir), diff --git a/src/lab/query/catalog.ts b/src/lab/query/catalog.ts index 876f9c4199..45d8cd4a3a 100644 --- a/src/lab/query/catalog.ts +++ b/src/lab/query/catalog.ts @@ -14,6 +14,12 @@ import { import { suiteManifestDigestForCase, } from "../conformance/suite-manifest"; +import { + discoverFabricScenarios, + fabricScenarioManifestDigest, + fabricSuiteManifestDigest, + loadFabricCaseAuthority, +} from "../fabric/manifest"; import { scenarioManifestDigest } from "../digest"; import type { CaseRecord } from "../conformance/types"; import type { CatalogFilters, CatalogScenarioDto } from "./types"; @@ -62,6 +68,30 @@ export function queryLabCatalog(filters: CatalogFilters = {}): CatalogScenarioDt items.push(mapCaseToCatalogScenario(caseRecord, authority, liveSuiteManifestDigestForCase, expandLiveScenario)); } } + if (!filters.layer || filters.layer === "task_effectiveness") { + const authority = loadFabricCaseAuthority(); + const suites = filters.suiteId ? [filters.suiteId] : undefined; + const scenarios = discoverFabricScenarios(authority, suites ?? undefined); + for (const caseRecord of scenarios) { + items.push({ + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + evidenceLayer: authority.manifestDefaults.evidenceLayer, + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + capability: caseRecord.capability, + verificationRole: caseRecord.verificationRole ?? authority.manifestDefaults.verificationRole, + requirements: { + inboundProtocols: [...caseRecord.requirements.inboundProtocols], + upstreamProtocols: [...caseRecord.requirements.upstreamProtocols], + surfaces: [...caseRecord.requirements.surfaces], + }, + freshness: { ...authority.manifestDefaults.freshness }, + scenarioManifestDigest: fabricScenarioManifestDigest(caseRecord, authority), + suiteManifestDigest: fabricSuiteManifestDigest(caseRecord.suite, authority), + }); + } + } return items.sort((a, b) => { const layerCmp = a.evidenceLayer.localeCompare(b.evidenceLayer); if (layerCmp !== 0) return layerCmp; diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts new file mode 100644 index 0000000000..1a59a55a69 --- /dev/null +++ b/tests/lab-fabric-task.test.ts @@ -0,0 +1,438 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + existsSync, + mkdirSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + appendLabEvent, + assertNotUnderUserRepo, + assertSafeRelativePosixPath, + assignEventId, + buildTaskSubjectV1, + correctSyntheticPatch, + createSyntheticScratch, + fabricScratchCapabilityProbe, + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + observationFromFabricOutcome, + persistFabricOutcome, + queryLabCatalog, + rebuildLabProjection, + replayLabLedger, + runFabricSyntheticPatchTask, + sandboxProfileDigest, + subjectIdForSubject, + taskFixtureDigest, + taskSubjectId, + verifierManifestDigest, + writeScratchFileUtf8, + type RouteSubjectV1, +} from "../src/lab"; +import { routingProfileIssues } from "../src/routing/profile"; +import { buildInvalidationIndex } from "../src/lab/ledger/invalidation"; +import { verifyExactTreeDiffV1 } from "../src/lab/fabric/verifier"; +import { parseSyntheticPatchV1 } from "../src/lab/fabric/patch"; +import { FABRIC_LIMITS, SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "../src/lab/fabric/constants"; + +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-cl07-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +afterEach(() => { + for (const dir of HOMES.splice(0)) { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + /* ignore */ + } + } + delete process.env.OPENCODEX_HOME; +}); + +function routeSubject(overrides: Partial = {}): RouteSubjectV1 { + return { + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: "provider-a", + providerInstanceFingerprint: "a".repeat(64), + clientModelId: "model-a", + upstreamModelId: "model-a", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + opencodexCompatibilityVersion: "b".repeat(64), + behaviorFingerprint: "c".repeat(64), + endpointFingerprint: "d".repeat(64), + dependencies: [], + ...overrides, + }; +} + +describe("CL-07 task effectiveness producer", () => { + test("canonical TaskSubjectV1 identity is stable", () => { + const subject = buildTaskSubjectV1({ routeSubject: routeSubject() }); + expect(subject.subjectKind).toBe("task"); + expect(taskSubjectId(subject)).toBe(subjectIdForSubject(subject)); + expect(taskSubjectId(buildTaskSubjectV1({ routeSubject: routeSubject() }))).toBe(taskSubjectId(subject)); + }); + + test("route change changes task subject", () => { + const a = taskSubjectId(buildTaskSubjectV1({ routeSubject: routeSubject() })); + const b = taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject({ providerId: "provider-b", endpointFingerprint: "e".repeat(64) }), + })); + expect(a).not.toBe(b); + }); + + test("fixture/verifier/sandbox/fabric version changes change task subject", () => { + const base = taskSubjectId(buildTaskSubjectV1({ routeSubject: routeSubject() })); + expect(taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject(), + taskFixtureDigest: "1".repeat(64), + }))).not.toBe(base); + expect(taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject(), + verifierManifestDigest: "2".repeat(64), + }))).not.toBe(base); + expect(taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject(), + sandboxProfileDigest: "3".repeat(64), + }))).not.toBe(base); + expect(taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject(), + fabricCompatibilityVersion: "other-fabric-v1", + }))).not.toBe(base); + expect(taskFixtureDigest()).toMatch(/^[0-9a-f]{64}$/); + expect(verifierManifestDigest()).toMatch(/^[0-9a-f]{64}$/); + expect(sandboxProfileDigest()).toMatch(/^[0-9a-f]{64}$/); + }); + + test("exact synthetic patch passes", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + }); + expect(outcome.outcome).toBe("pass"); + expect(outcome.verifier.passed).toBe(true); + }); + + test("unchanged / wrong / added / deleted trees fail verifier", () => { + const home = tempHome(); + const scratch = createSyntheticScratch(home); + try { + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(verifyExactTreeDiffV1(scratch.root).reason).toBe("unchanged_file"); + + writeScratchFileUtf8(scratch.root, SYNTHETIC_VALUE_PATH, "nope\n", FABRIC_LIMITS.maxAggregateIoBytes); + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(verifyExactTreeDiffV1(scratch.root).reason?.startsWith("expected_digest_")).toBe(true); + + writeScratchFileUtf8(scratch.root, SYNTHETIC_VALUE_PATH, SYNTHETIC_AFTER_UTF8, FABRIC_LIMITS.maxAggregateIoBytes); + writeScratchFileUtf8(scratch.root, "src/extra.txt", "x\n", FABRIC_LIMITS.maxAggregateIoBytes); + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(verifyExactTreeDiffV1(scratch.root).reason).toBe("unexpected_tree_shape"); + } finally { + scratch.cleanup(); + } + + const deleted = createSyntheticScratch(home); + try { + rmSync(join(deleted.root, "src", "value.txt"), { force: true }); + expect(verifyExactTreeDiffV1(deleted.root).passed).toBe(false); + } finally { + deleted.cleanup(); + } + }); + + test("renamed file fails verifier", () => { + const home = tempHome(); + const scratch = createSyntheticScratch(home); + try { + writeScratchFileUtf8(scratch.root, "src/renamed.txt", SYNTHETIC_AFTER_UTF8, FABRIC_LIMITS.maxAggregateIoBytes); + rmSync(join(scratch.root, "src", "value.txt"), { force: true }); + const result = verifyExactTreeDiffV1(scratch.root); + expect(result.passed).toBe(false); + expect(result.pathSummaries.some((row) => row.kind === "added" || row.kind === "deleted")).toBe(true); + } finally { + scratch.cleanup(); + } + }); + + test("symlink and path traversal are rejected", () => { + expect(() => assertSafeRelativePosixPath("../etc/passwd")).toThrow(); + expect(() => assertSafeRelativePosixPath("/abs")).toThrow(); + expect(() => assertSafeRelativePosixPath("C:\\windows")).toThrow(); + + const home = tempHome(); + const scratch = createSyntheticScratch(home); + try { + const target = join(scratch.root, "src", "value.txt"); + const link = join(scratch.root, "src", "link.txt"); + try { + symlinkSync(target, link); + } catch { + return; + } + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + } finally { + scratch.cleanup(); + } + }); + + test("special file is rejected where supported", () => { + if (process.platform === "win32") return; + const home = tempHome(); + const scratch = createSyntheticScratch(home); + try { + const fifo = join(scratch.root, "src", "fifo"); + try { + symlinkSync("/dev/null", fifo); + } catch { + return; + } + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + } finally { + scratch.cleanup(); + } + }); + + test("oversized patch fails safely", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const huge = "x".repeat(FABRIC_LIMITS.maxAggregateIoBytes + 8); + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => ({ + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: huge }], + }), + }); + expect(outcome.outcome).not.toBe("pass"); + expect(outcome.failure?.code).toBe("budget_exhausted"); + }); + + test("execution and inactivity timeouts are bounded", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: async () => { + await Bun.sleep(FABRIC_LIMITS.inactivityTimeoutMs + 50); + return correctSyntheticPatch(); + }, + }); + expect(["blocked", "inconclusive"]).toContain(outcome.outcome); + expect(["timeout", "inactivity_timeout"]).toContain(outcome.failure?.code ?? ""); + }, 20_000); + + test("network / user MCP / arbitrary shell remain unavailable", () => { + expect(fabricScratchCapabilityProbe()).toEqual({ + network: false, + userMcp: false, + arbitraryShell: false, + userRepository: false, + }); + }); + + test("user repository cannot host the scratch root", () => { + const home = tempHome(); + const repo = join(home, "user-repo"); + mkdirSync(join(repo, "src"), { recursive: true }); + writeFileSync(join(repo, "src", "value.txt"), "before\n"); + expect(() => assertNotUnderUserRepo(repo, repo)).toThrow(); + const scratch = createSyntheticScratch(home); + try { + expect(existsSync(join(scratch.root, "src", "value.txt"))).toBe(true); + expect(scratch.root.startsWith(join(home, "lab", "scratch"))).toBe(true); + } finally { + scratch.cleanup(); + } + }); + + test("malformed producer result is rejected", () => { + expect(() => parseSyntheticPatchV1({ schemaVersion: 1, operations: [], extra: true })).toThrow(); + expect(() => parseSyntheticPatchV1({ + schemaVersion: 2, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: "after\n" }], + })).toThrow(); + expect(() => observationFromFabricOutcome({ + schemaVersion: 1, + taskSubject: { subjectKind: "route" }, + })).toThrow(); + }); + + test("duplicate outcome delivery is idempotent; distinct attempts remain distinct", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + now: () => 1_000, + }); + const first = persistFabricOutcome(outcome, { configDir: home, recordedAt: 1_000, attempt: 1 }); + const second = persistFabricOutcome(outcome, { configDir: home, recordedAt: 1_000, attempt: 1 }); + expect(second.event.eventId).toBe(first.event.eventId); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + const ids = replay.events.filter((row) => row.eventKind === "observation").map((row) => row.eventId); + expect(ids).toEqual([first.event.eventId]); + expect(replay.corruptions.filter((row) => row.kind === "duplicate_event")).toEqual([]); + + const secondAttempt = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + now: () => 2_000, + }); + const third = persistFabricOutcome(secondAttempt, { configDir: home, recordedAt: 2_000, attempt: 2 }); + expect(third.event.eventId).not.toBe(first.event.eventId); + }); + + test("structured result converts to fabric Lab observation", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + }); + const { event } = observationFromFabricOutcome(outcome, { configDir: home }); + expect(event.evidenceLayer).toBe("task_effectiveness"); + expect(event.executionMode).toBe("fabric"); + expect(event.subject.subjectKind).toBe("task"); + expect(event.suiteId).toBe("fabric-core"); + const serialized = JSON.stringify(event); + expect(serialized.includes("sk-")).toBe(false); + expect(serialized.includes("OPENAI_API_KEY")).toBe(false); + }); + + test("task layer rejects RouteSubjectV1 mismatches", () => { + expect(() => observationFromFabricOutcome({ + schemaVersion: 1, + taskClassId: "x", + taskClassVersion: "1", + routeSubject: routeSubject(), + taskSubject: { subjectSchemaVersion: 1, subjectKind: "route" }, + subjectId: "a".repeat(64), + taskFixtureDigest: "b".repeat(64), + verifierManifestDigest: "c".repeat(64), + fabricCompatibilityVersion: "v", + sandboxProfileDigest: "d".repeat(64), + startedAt: 1, + completedAt: 2, + limits: FABRIC_LIMITS, + usage: { + inputBytes: 0, + outputBytes: 0, + patchOperations: 0, + filesTouched: 0, + artifactBytes: 0, + elapsedMs: 1, + inactiveMs: 0, + }, + outcome: "pass", + verifier: { + verifierId: "exact-tree-diff-v1", + manifestDigest: "c".repeat(64), + passed: true, + pathSummaries: [], + }, + artifactDigests: [], + })).toThrow(); + }); + + test("projection rebuild and invalidation work for task evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + now: () => 5_000, + }); + const persisted = persistFabricOutcome(outcome, { configDir: home, recordedAt: 5_000 }); + const rebuilt = rebuildLabProjection(home); + expect(rebuilt.events).toBeGreaterThan(0); + expect(rebuilt.corruptions).toEqual([]); + + const inv = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "invalidation" as const, + recordedAt: 6_000, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + targetEventIds: [persisted.event.eventId], + reason: "manual_correction" as const, + }); + appendLabEvent(join(home, "lab", "compatibility.jsonl"), inv as never); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + const index = buildInvalidationIndex(replay.events); + expect(index.invalidatedBy.has(persisted.event.eventId)).toBe(true); + }); + + test("artifacts are bounded and catalog exposes task evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + }); + const { event, artifacts } = observationFromFabricOutcome(outcome, { configDir: home }); + expect(artifacts.length).toBeGreaterThan(0); + expect(artifacts.every((row) => row.byteCount <= 256 * 1024)).toBe(true); + expect(event.artifactRefs.every((row) => /^[0-9a-f]{64}$/.test(row.digest))).toBe(true); + + const catalog = queryLabCatalog({ layer: "task_effectiveness" }); + expect(catalog.some((row) => row.suiteId === "fabric-core" && row.scenarioId === "fabric-core.task.synthetic-patch")).toBe(true); + }); + + test("CL-06 routing still rejects task_effectiveness required suites", () => { + const issues = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m" }], + compatibility: { + requiredSuites: [{ + suiteId: "fabric-core", + evidenceLayer: "task_effectiveness" as "live_route_compatibility", + }], + }, + }, { + providers: { + a: { type: "openai", baseUrl: "http://127.0.0.1:9", models: { m: {} } }, + }, + } as never); + expect(issues.some((row) => row.message.includes("protocol_conformance or live_route_compatibility"))).toBe(true); + }); + + test("ledger lines omit prompts and credentials", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + sourceRefs: ["routeDecision:abcd1234"], + }); + persistFabricOutcome(outcome, { configDir: home }); + const text = readFileSync(join(home, "lab", "compatibility.jsonl"), "utf8"); + expect(text.includes("system prompt")).toBe(false); + expect(text.includes("sk-")).toBe(false); + }); +}); From 5e2ce66d75f8bebef0729a369f6d8c0fffccac7b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:50:56 +0200 Subject: [PATCH 02/24] docs(lab): link CL-07 draft PR #1438 Record the implementation head and draft PR on the programme status documents. --- devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md | 2 +- .../260807_compatibility_lab/007_cl07_task_effectiveness.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index ef8e8d84e5..58ac68cb34 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -26,7 +26,7 @@ independent review, blockers, and whether a later phase is authorized. | CL-04 | `feat/cl-04-lab-read-surfaces` | `68c71a4e9cdf882d812f09fd94783a28749db629` | NOT RECORDED | [#1378](https://github.com/lidge-jun/opencodex/pull/1378) | MERGED TO `dev` at `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | | CL-05 | `feat/cl-05-compatibility-matrix-ui` | `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | `2a159b8b7` (Models tab placement) | [#1384](https://github.com/lidge-jun/opencodex/pull/1384) | MERGED TO `dev` at `1072b9c39c48a4982229131613ac300560740742` | | CL-06 | `feat/cl-06-routing-profile-compatibility` | `1072b9c39c48a4982229131613ac300560740742` | `b66e33ce7207d91014644d99317e456c992a3418` (merge) | [#1394](https://github.com/lidge-jun/opencodex/pull/1394) | MERGED TO `dev` at `b66e33ce7207d91014644d99317e456c992a3418`; ACCEPTED/CLOSED | -| CL-07 | `feat/cl-07-task-effectiveness-producer` | `b66e33ce7207d91014644d99317e456c992a3418` | NOT YET | pending | AUTHORIZED / IN PROGRESS; plan `007_cl07_task_effectiveness.md` | +| CL-07 | `feat/cl-07-task-effectiveness-producer` | `b66e33ce7207d91014644d99317e456c992a3418` | `ed8e0794189b214398d835e1af828905f29fbc53` | [#1438](https://github.com/lidge-jun/opencodex/pull/1438) | DRAFT / PENDING CI; plan `007_cl07_task_effectiveness.md` | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. diff --git a/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md index e42482dead..6eb05f43b8 100644 --- a/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md +++ b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md @@ -232,9 +232,9 @@ Typecheck; focused fabric/subject/sandbox/verifier/observe/ledger/projection/rea ## M. Status - **Authorization:** IN PROGRESS from CL-06 merge `b66e33ce7207d91014644d99317e456c992a3418` -- **Implementation head:** pending commit on `feat/cl-07-task-effectiveness-producer` +- **Implementation head:** `ed8e0794189b214398d835e1af828905f29fbc53` - **Accepted head:** NOT YET -- **PR:** pending draft +- **PR:** [#1438](https://github.com/lidge-jun/opencodex/pull/1438) (draft) ### Local validation (pre-PR) From ec275ee0fd288293568eaea120f73164ac110a79 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:53:04 +0200 Subject: [PATCH 03/24] fix(lab): project VERIFIED for fabric task evidence Wire task_effectiveness + fabric through all-applicable-required-pass-v1 so successful synthetic-patch observations can leave PROBED. --- src/lab/projection/verdicts.ts | 27 +++++++++++++++++++++++++++ src/lab/projection/verification.ts | 24 ++++++++++++++++++++++-- tests/lab-fabric-task.test.ts | 8 ++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index db0bc683e3..09c960690b 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -407,6 +407,33 @@ function projectObservationGroup( notes.push("incomplete_required_coverage"); } } + } else if (key.evidenceLayer === "task_effectiveness" && newestCurrent?.executionMode === "fabric") { + if (!suiteManifest) { + verdict = "PROBED"; + notes.push("suite_manifest_unavailable"); + } else { + const evaluation = evaluateAllApplicableRequiredPassV1( + suiteManifest, + ordered, + newestCurrent.executionMode, + { + subject: newestCurrent.subject.subjectKind === "task" ? newestCurrent.subject : undefined, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }, + ); + notes.push(...evaluation.notes); + if (evaluation.applicableRequiredScenarioIds.length === 0) { + verdict = "UNKNOWN"; + } else if (evaluation.canVerify) { + verdict = "VERIFIED"; + notes.push("all-applicable-required-pass-v1"); + } else { + verdict = "PROBED"; + notes.push("incomplete_required_coverage"); + } + } } else { verdict = "PROBED"; } diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index ac48e342d1..0edc0decad 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -1,4 +1,4 @@ -import type { ObservationEvent, ProtocolSubjectV1, RouteSubjectV1 } from "../events/types"; +import type { ObservationEvent, ProtocolSubjectV1, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; import { EVIDENCE_LAYERS, type ExecutionMode } from "../constants"; import type { SuiteManifestV1 } from "../conformance/suite-manifest"; import type { VerificationRole } from "../conformance/types"; @@ -139,7 +139,7 @@ export function evaluateAllApplicableRequiredPassV1( observations: ObservationEvent[], executionMode: ExecutionMode, opts: { - subject?: ProtocolSubjectV1 | RouteSubjectV1; + subject?: ProtocolSubjectV1 | RouteSubjectV1 | TaskSubjectV1; /** For live projection this must come from validated current claim snapshots for subjectId. */ routeSupportedClaims?: readonly string[]; loadScenarioManifest?: LoadScenarioManifest; @@ -165,6 +165,26 @@ export function evaluateAllApplicableRequiredPassV1( return { applicableRequiredScenarioIds: [], passingRequiredScenarioIds: [], missingRequiredScenarioIds: [], canVerify: false, notes: ["route_claim_state_required"] }; } } + if (suiteManifest.evidenceLayer === "task_effectiveness") { + if (opts.subject?.subjectKind !== "task") { + return { + applicableRequiredScenarioIds: [], + passingRequiredScenarioIds: [], + missingRequiredScenarioIds: [], + canVerify: false, + notes: ["task_subject_required"], + }; + } + if (executionMode !== "fabric") { + return { + applicableRequiredScenarioIds: [], + passingRequiredScenarioIds: [], + missingRequiredScenarioIds: [], + canVerify: false, + notes: ["fabric_execution_mode_required"], + }; + } + } const requiredScenarios = suiteManifest.scenarios.filter( (s) => s.role === "required" || s.role === "negative_control", diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index 1a59a55a69..cd982a1f54 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -23,6 +23,7 @@ import { observationFromFabricOutcome, persistFabricOutcome, queryLabCatalog, + readVerdictSnapshot, rebuildLabProjection, replayLabLedger, runFabricSyntheticPatchTask, @@ -371,6 +372,13 @@ describe("CL-07 task effectiveness producer", () => { const rebuilt = rebuildLabProjection(home); expect(rebuilt.events).toBeGreaterThan(0); expect(rebuilt.corruptions).toEqual([]); + expect(rebuilt.verdicts).toBeGreaterThan(0); + const snap = readVerdictSnapshot(rebuilt.sqlitePath) as Array<{ + evidence_layer?: string; + verdict?: string; + }>; + const taskRow = snap.find((row) => row.evidence_layer === "task_effectiveness"); + expect(taskRow?.verdict).toBe("VERIFIED"); const inv = assignEventId({ schemaVersion: LAB_EVENT_SCHEMA_VERSION, From db8e6abdef3361bebf2e4b7453999e259be8a117 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:31:19 +0200 Subject: [PATCH 04/24] fix(lab): address CodeRabbit findings for CL-07 fabric producer Harden scratch IO, timeouts, outcome validation, ledger idempotency, and projection/catalog paths so the review findings stay closed on this branch. --- .../001_pr_stack_status.md | 2 +- src/lab/conformance/types.ts | 23 +++ src/lab/events/types.ts | 3 +- src/lab/events/validate.ts | 3 +- src/lab/fabric/constants.ts | 2 + src/lab/fabric/executor.ts | 140 +++++++++------ src/lab/fabric/index.ts | 78 +++++++- src/lab/fabric/manifest.ts | 19 +- src/lab/fabric/observe.ts | 83 +++++++-- src/lab/fabric/patch.ts | 2 +- src/lab/fabric/scratch.ts | 87 ++++++--- src/lab/fabric/subject.ts | 20 ++- src/lab/fabric/types.ts | 8 +- src/lab/ledger/store.ts | 72 ++++++++ src/lab/observe/from-conformance.ts | 4 + src/lab/observe/from-live.ts | 8 + src/lab/paths.ts | 7 +- src/lab/projection/verdicts.ts | 167 +++++++++--------- src/lab/projection/verification.ts | 56 +++++- src/lab/query/catalog.ts | 45 +++-- tests/lab-fabric-task.test.ts | 113 +++++++++++- 21 files changed, 712 insertions(+), 230 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 58ac68cb34..dcae28e01c 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -26,7 +26,7 @@ independent review, blockers, and whether a later phase is authorized. | CL-04 | `feat/cl-04-lab-read-surfaces` | `68c71a4e9cdf882d812f09fd94783a28749db629` | NOT RECORDED | [#1378](https://github.com/lidge-jun/opencodex/pull/1378) | MERGED TO `dev` at `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | | CL-05 | `feat/cl-05-compatibility-matrix-ui` | `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | `2a159b8b7` (Models tab placement) | [#1384](https://github.com/lidge-jun/opencodex/pull/1384) | MERGED TO `dev` at `1072b9c39c48a4982229131613ac300560740742` | | CL-06 | `feat/cl-06-routing-profile-compatibility` | `1072b9c39c48a4982229131613ac300560740742` | `b66e33ce7207d91014644d99317e456c992a3418` (merge) | [#1394](https://github.com/lidge-jun/opencodex/pull/1394) | MERGED TO `dev` at `b66e33ce7207d91014644d99317e456c992a3418`; ACCEPTED/CLOSED | -| CL-07 | `feat/cl-07-task-effectiveness-producer` | `b66e33ce7207d91014644d99317e456c992a3418` | `ed8e0794189b214398d835e1af828905f29fbc53` | [#1438](https://github.com/lidge-jun/opencodex/pull/1438) | DRAFT / PENDING CI; plan `007_cl07_task_effectiveness.md` | +| CL-07 | `feat/cl-07-task-effectiveness-producer` | `b66e33ce7207d91014644d99317e456c992a3418` | NOT RECORDED | [#1438](https://github.com/lidge-jun/opencodex/pull/1438) | DRAFT / PENDING CI; implementation head `ed8e0794189b214398d835e1af828905f29fbc53`; plan `007_cl07_task_effectiveness.md` | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 30cc0c9637..affc5e4a65 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -9,7 +9,11 @@ export type VerificationRole = "required" | "supplemental" | "negative_control"; export type FailureClassification = | "harness_failure" | "timeout" + | "inactivity_timeout" | "budget_exhausted" + | "sandbox_violation" + | "malformed_producer_outcome" + | "layer_subject_mismatch" | "protocol_failure" | "capability_failure" | "behavioral_failure" @@ -20,6 +24,25 @@ export type FailureClassification = | "provider_transient" | "inconclusive"; +export const FAILURE_CLASSIFICATIONS = [ + "harness_failure", + "timeout", + "inactivity_timeout", + "budget_exhausted", + "sandbox_violation", + "malformed_producer_outcome", + "layer_subject_mismatch", + "protocol_failure", + "capability_failure", + "behavioral_failure", + "authentication_blocked", + "quota_blocked", + "region_blocked", + "network_failure", + "provider_transient", + "inconclusive", +] as const satisfies readonly FailureClassification[]; + export interface FixtureRecord { id: string; role: "client_request" | "upstream_response" | "adapter_vector" | "synthetic_tool"; diff --git a/src/lab/events/types.ts b/src/lab/events/types.ts index 69ed120db0..293d9a7c5d 100644 --- a/src/lab/events/types.ts +++ b/src/lab/events/types.ts @@ -9,6 +9,7 @@ import type { ObservationOutcome, PurgeAction, } from "../constants"; +import type { FailureClassification } from "../conformance/types"; export interface LabProducer { name: string; @@ -90,7 +91,7 @@ export interface AssertionRecordV1 { } export interface FailureRecordV1 { - class: string; + class: FailureClassification; code: string; retryable: boolean; attribution: "opencodex" | "route" | "environment" | "harness"; diff --git a/src/lab/events/validate.ts b/src/lab/events/validate.ts index 1bf3c137fc..df9d15f95a 100644 --- a/src/lab/events/validate.ts +++ b/src/lab/events/validate.ts @@ -29,6 +29,7 @@ import { jcsStringify, subjectIdForSubject, } from "../digest"; +import { FAILURE_CLASSIFICATIONS } from "../conformance/types"; import type { ArtifactRefV1, ClaimCapabilityFactsV1, @@ -408,7 +409,7 @@ function validateObservation(raw: Record): ObservationEvent { if (raw.failure !== undefined) { if (!isPlainObject(raw.failure)) throw new LabValidationError("invalid_failure", "failure"); event.failure = { - class: assertString(raw.failure.class, "failure.class"), + class: assertClosed(raw.failure.class, "failure.class", FAILURE_CLASSIFICATIONS), code: assertString(raw.failure.code, "failure.code"), retryable: raw.failure.retryable === true, attribution: assertClosed(raw.failure.attribution, "failure.attribution", [ diff --git a/src/lab/fabric/constants.ts b/src/lab/fabric/constants.ts index 43755c3d2a..d15e814e87 100644 --- a/src/lab/fabric/constants.ts +++ b/src/lab/fabric/constants.ts @@ -16,6 +16,8 @@ export const SYNTHETIC_AFTER_UTF8 = "after\n"; export const FABRIC_LIMITS = Object.freeze({ maxFiles: 1, + /** Upper bound for scratch-tree walks (verifier may see unexpected extras). */ + maxScratchFiles: 8, maxAggregateIoBytes: 64 * 1024, maxPatchOperations: 1, totalTimeoutMs: 30_000, diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts index 0a49363239..30bdd9773e 100644 --- a/src/lab/fabric/executor.ts +++ b/src/lab/fabric/executor.ts @@ -5,11 +5,14 @@ import { FABRIC_LIMITS, FABRIC_TASK_CLASS_ID, FABRIC_TASK_CLASS_VERSION, + FABRIC_VERIFIER_ID, + SANDBOX_PROFILE_V1, SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, SYNTHETIC_VALUE_PATH, } from "./constants"; import { applySyntheticPatch, parseSyntheticPatchV1 } from "./patch"; -import { assertNotUnderUserRepo, createSyntheticScratch } from "./scratch"; +import { assertNotUnderUserRepo, createSyntheticScratch, type ScratchTree } from "./scratch"; import { buildTaskSubjectV1, sandboxProfileDigest, @@ -83,7 +86,7 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) const subjectId = taskSubjectId(taskSubject); const usage: FabricUsageV1 = { - inputBytes: Buffer.byteLength("before\n", "utf8"), + inputBytes: Buffer.byteLength(SYNTHETIC_BEFORE_UTF8, "utf8"), outputBytes: 0, patchOperations: 0, filesTouched: 0, @@ -92,8 +95,9 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) inactiveMs: 0, }; - const scratch = createSyntheticScratch(options.configDir); + let scratch: ScratchTree | undefined; try { + scratch = createSyntheticScratch(options.configDir); if (options.userRepoRoot) { assertNotUnderUserRepo(scratch.root, options.userRepoRoot); } @@ -102,12 +106,14 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) const produceStarted = options.now?.() ?? Date.now(); let patchRaw: unknown; try { + const controller = createTimeoutController(FABRIC_LIMITS.totalTimeoutMs, FABRIC_LIMITS.inactivityTimeoutMs, options); const producePromise = Promise.resolve(options.producePatch({ taskClassId: FABRIC_TASK_CLASS_ID, taskClassVersion: FABRIC_TASK_CLASS_VERSION, scratchRoot: scratch.root, + reportActivity: controller.reportActivity, })); - patchRaw = await withTimeout(producePromise, FABRIC_LIMITS.totalTimeoutMs, FABRIC_LIMITS.inactivityTimeoutMs, options); + patchRaw = await controller.race(producePromise); } catch (error) { const completedAt = options.now?.() ?? Date.now(); usage.elapsedMs = completedAt - startedAt; @@ -125,7 +131,7 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) usage, outcome: outcomeFromFailure(failure), verifier: { - verifierId: "exact-tree-diff-v1", + verifierId: FABRIC_VERIFIER_ID, manifestDigest: verifierDigest, passed: false, pathSummaries: [], @@ -204,7 +210,7 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) usage, outcome: outcomeFromFailure(failure), verifier: { - verifierId: "exact-tree-diff-v1", + verifierId: FABRIC_VERIFIER_ID, manifestDigest: verifierDigest, passed: false, pathSummaries: [], @@ -231,7 +237,7 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) usage, outcome: "inconclusive", verifier: { - verifierId: "exact-tree-diff-v1", + verifierId: FABRIC_VERIFIER_ID, manifestDigest: verifierDigest, passed: false, pathSummaries: [], @@ -241,7 +247,7 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) sourceRefs: options.sourceRefs, }); } finally { - scratch.cleanup(); + scratch?.cleanup(); } } @@ -286,58 +292,88 @@ function sealOutcome(input: { return sealed; } -async function withTimeout( - promise: Promise, +function createTimeoutController( totalMs: number, inactivityMs: number, options: RunFabricTaskOptions, -): Promise { - let settled = false; +): { + reportActivity: () => void; + race: (promise: Promise) => Promise; +} { const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); - return await new Promise((resolve, reject) => { - const totalTimer = setTimeout(() => { - if (!settled) { - settled = true; - reject(new FabricTaskError("total timeout exceeded", "timeout", "environment")); - } - }, totalMs); - const inactivityTimer = setTimeout(() => { - if (!settled) { - settled = true; - reject(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); - } + let settled = false; + let totalTimer: ReturnType | undefined; + let inactivityTimer: ReturnType | undefined; + let rejectRef: ((error: FabricTaskError) => void) | undefined; + + const clearTimers = () => { + if (totalTimer !== undefined) clearTimeout(totalTimer); + if (inactivityTimer !== undefined) clearTimeout(inactivityTimer); + totalTimer = undefined; + inactivityTimer = undefined; + }; + + const settleReject = (error: FabricTaskError) => { + if (settled) return; + settled = true; + clearTimers(); + rejectRef?.(error); + }; + + const armInactivity = () => { + if (inactivityTimer !== undefined) clearTimeout(inactivityTimer); + inactivityTimer = setTimeout(() => { + settleReject(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); }, inactivityMs); - void sleep(0); - promise.then( - (value) => { - if (settled) return; - settled = true; - clearTimeout(totalTimer); - clearTimeout(inactivityTimer); - resolve(value); - }, - (error) => { - if (settled) return; - settled = true; - clearTimeout(totalTimer); - clearTimeout(inactivityTimer); - reject(error); - }, - ); - }); + }; + + return { + reportActivity: () => { + if (settled) return; + armInactivity(); + }, + race: async (promise: Promise): Promise => { + return await new Promise((resolve, reject) => { + rejectRef = reject; + totalTimer = setTimeout(() => { + settleReject(new FabricTaskError("total timeout exceeded", "timeout", "environment")); + }, totalMs); + armInactivity(); + // Keep the injected sleep seam observable for tests without discarding work. + void sleep(0).catch(() => undefined); + promise.then( + (value) => { + if (settled) return; + settled = true; + clearTimers(); + resolve(value); + }, + (error) => { + if (settled) return; + settled = true; + clearTimers(); + reject(error); + }, + ); + }); + }, + }; } -/** Explicit denials for capabilities that must remain unavailable in the scratch producer. */ -export function fabricScratchCapabilityProbe(): { - network: false; - userMcp: false; - arbitraryShell: false; - userRepository: false; +/** Declared sandbox policy for the frozen scratch producer (not runtime enforcement). */ +export function fabricDeclaredSandboxPolicy(): { + network: boolean; + userMcp: boolean; + arbitraryShell: boolean; + userRepository: boolean; } { return { - network: false, - userMcp: false, - arbitraryShell: false, - userRepository: false, + network: SANDBOX_PROFILE_V1.allowNetwork, + userMcp: SANDBOX_PROFILE_V1.allowUserMcp, + arbitraryShell: SANDBOX_PROFILE_V1.allowShell, + userRepository: SANDBOX_PROFILE_V1.allowUserRepository, }; } + +/** @deprecated Use fabricDeclaredSandboxPolicy. */ +export const fabricScratchCapabilityProbe = fabricDeclaredSandboxPolicy; diff --git a/src/lab/fabric/index.ts b/src/lab/fabric/index.ts index 31e9712137..1e8f32470e 100644 --- a/src/lab/fabric/index.ts +++ b/src/lab/fabric/index.ts @@ -1,9 +1,69 @@ -export * from "./constants"; -export * from "./types"; -export * from "./subject"; -export * from "./scratch"; -export * from "./patch"; -export * from "./verifier"; -export * from "./executor"; -export * from "./manifest"; -export * from "./observe"; +export { + FABRIC_SUITE_ID, + FABRIC_SUITE_VERSION, + FABRIC_SCENARIO_ID, + FABRIC_SCENARIO_VERSION, + FABRIC_TASK_CLASS_ID, + FABRIC_TASK_CLASS_VERSION, + FABRIC_COMPATIBILITY_VERSION, + FABRIC_VERIFIER_ID, + FABRIC_EVIDENCE_LAYER, + SYNTHETIC_VALUE_PATH, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_AFTER_UTF8, + FABRIC_LIMITS, + SANDBOX_PROFILE_V1, +} from "./constants"; +export type { + FabricOutcomeKind, + FabricFailureClass, + FabricLimitsV1, + SyntheticPatchOperationV1, + SyntheticPatchV1, + FabricUsageV1, + ExactTreeDiffPathSummaryV1, + ExactTreeDiffResultV1, + FabricTaskOutcomeV1, + SyntheticPatchProducer, +} from "./types"; +export { FabricTaskError } from "./types"; +export { + sandboxProfileDigest, + verifierManifestObject, + verifierManifestDigest, + taskFixtureObject, + taskFixtureDigest, + buildTaskSubjectV1, + taskSubjectId, +} from "./subject"; +export { + assertSafeRelativePosixPath, + resolveInsideScratch, + createSyntheticScratch, + assertNotUnderUserRepo, +} from "./scratch"; +export type { ScratchTree, WalkedFile } from "./scratch"; +export { parseSyntheticPatchV1 } from "./patch"; +export { verifyExactTreeDiffV1 } from "./verifier"; +export { + correctSyntheticPatch, + runFabricSyntheticPatchTask, + fabricDeclaredSandboxPolicy, + fabricScratchCapabilityProbe, +} from "./executor"; +export type { RunFabricTaskOptions } from "./executor"; +export { + loadFabricCaseAuthority, + discoverFabricScenarios, + expandFabricScenario, + expandFabricSuiteManifest, + fabricScenarioManifestDigest, + fabricSuiteManifestDigest, +} from "./manifest"; +export type { FabricCaseAuthority } from "./manifest"; +export { + assertFabricOutcomeV1, + observationFromFabricOutcome, + persistFabricOutcome, +} from "./observe"; +export type { PersistFabricOptions, PersistedFabricObservation } from "./observe"; diff --git a/src/lab/fabric/manifest.ts b/src/lab/fabric/manifest.ts index defe6faf85..c0c8be1ee2 100644 --- a/src/lab/fabric/manifest.ts +++ b/src/lab/fabric/manifest.ts @@ -8,6 +8,9 @@ import { FABRIC_SUITE_ID, FABRIC_SUITE_VERSION, FABRIC_TASK_CLASS_ID, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, } from "./constants"; import { verifierManifestDigest } from "./subject"; @@ -29,7 +32,11 @@ export interface FabricCaseAuthority { } const FABRIC_SOURCE_COMMIT = "b66e33ce7207d91014644d99317e456c992a3418"; -const FIXTURE_BYTES_UTF8 = JSON.stringify({ path: "src/value.txt", before: "before\n", after: "after\n" }); +const FIXTURE_BYTES_UTF8 = JSON.stringify({ + path: SYNTHETIC_VALUE_PATH, + before: SYNTHETIC_BEFORE_UTF8, + after: SYNTHETIC_AFTER_UTF8, +}); export function loadFabricCaseAuthority(): FabricCaseAuthority { const fixtureBytes = new TextEncoder().encode(FIXTURE_BYTES_UTF8); @@ -145,7 +152,15 @@ export function expandFabricScenario( }, { id: "sandbox-block", - match: ["sandbox_violation", "timeout", "inactivity_timeout", "budget_exhausted"], + match: ["sandbox_violation"], + classification: "sandbox_violation", + verdictEffect: "none", + retry: "bounded", + expected: false, + }, + { + id: "timeout-block", + match: ["timeout", "inactivity_timeout", "budget_exhausted"], classification: "timeout", verdictEffect: "none", retry: "bounded", diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 4ec2230d60..2d7d5d9c50 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -1,4 +1,3 @@ -import { existsSync } from "node:fs"; import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; import { sanitizeDiagnostic, truncateUtf8 } from "../artifacts/sanitize"; import { @@ -8,9 +7,9 @@ import { OBSERVATION_LIMIT_NAMES, } from "../constants"; import { fixtureDigest } from "../digest"; -import type { ObservationEvent } from "../events/types"; +import type { ObservationEvent, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; import { assignEventId } from "../events/validate"; -import { appendLabEvent, replayLabLedger } from "../ledger/store"; +import { appendLabEventIfAbsent } from "../ledger/store"; import { ensureLabDirs } from "../paths"; import { FABRIC_EVIDENCE_LAYER, @@ -64,6 +63,29 @@ const OUTCOME_KEYS = new Set([ "sourceRefs", ]); +function assertPlainObject(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new FabricTaskError(`malformed producer outcome: ${label}`, "malformed_producer_outcome", "harness"); + } + return value as Record; +} + +function assertStringField(obj: Record, key: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.length === 0) { + throw new FabricTaskError(`malformed producer outcome: ${key}`, "malformed_producer_outcome", "harness"); + } + return value; +} + +function assertIntegerField(obj: Record, key: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value)) { + throw new FabricTaskError(`malformed producer outcome: ${key}`, "malformed_producer_outcome", "harness"); + } + return value; +} + export function assertFabricOutcomeV1(raw: unknown): FabricTaskOutcomeV1 { if (!raw || typeof raw !== "object" || Array.isArray(raw)) { throw new FabricTaskError("malformed producer outcome", "malformed_producer_outcome", "harness"); @@ -77,12 +99,40 @@ export function assertFabricOutcomeV1(raw: unknown): FabricTaskOutcomeV1 { if (obj.schemaVersion !== 1) { throw new FabricTaskError("schemaVersion must be 1", "malformed_producer_outcome", "harness"); } - if (obj.taskSubject && typeof obj.taskSubject === "object" && !Array.isArray(obj.taskSubject)) { - const kind = (obj.taskSubject as { subjectKind?: unknown }).subjectKind; - if (kind !== "task") { - throw new FabricTaskError("task layer rejects non-task subjects", "layer_subject_mismatch", "harness"); - } + + const taskSubjectObj = assertPlainObject(obj.taskSubject, "taskSubject"); + if (taskSubjectObj.subjectKind !== "task") { + throw new FabricTaskError("task layer rejects non-task subjects", "layer_subject_mismatch", "harness"); } + const routeFromTask = assertPlainObject(taskSubjectObj.routeSubject, "taskSubject.routeSubject"); + if (routeFromTask.subjectKind !== "route") { + throw new FabricTaskError("nested route subject required", "layer_subject_mismatch", "harness"); + } + + const routeSubjectObj = assertPlainObject(obj.routeSubject, "routeSubject"); + if (routeSubjectObj.subjectKind !== "route") { + throw new FabricTaskError("nested route subject required", "layer_subject_mismatch", "harness"); + } + + assertStringField(obj, "taskClassId"); + assertStringField(obj, "taskClassVersion"); + assertStringField(obj, "subjectId"); + assertStringField(obj, "taskFixtureDigest"); + assertStringField(obj, "verifierManifestDigest"); + assertStringField(obj, "fabricCompatibilityVersion"); + assertStringField(obj, "sandboxProfileDigest"); + assertIntegerField(obj, "startedAt"); + assertIntegerField(obj, "completedAt"); + assertPlainObject(obj.limits, "limits"); + assertPlainObject(obj.usage, "usage"); + assertPlainObject(obj.verifier, "verifier"); + if (typeof obj.outcome !== "string") { + throw new FabricTaskError("malformed producer outcome: outcome", "malformed_producer_outcome", "harness"); + } + if (!Array.isArray(obj.artifactDigests)) { + throw new FabricTaskError("malformed producer outcome: artifactDigests", "malformed_producer_outcome", "harness"); + } + return raw as FabricTaskOutcomeV1; } @@ -91,10 +141,12 @@ export function observationFromFabricOutcome( opts: PersistFabricOptions = {}, ): { event: ObservationEvent; artifacts: ReturnType[] } { const outcome = assertFabricOutcomeV1(outcomeRaw); - if (outcome.taskSubject.subjectKind !== "task") { + const taskSubject = outcome.taskSubject as TaskSubjectV1; + const routeSubject = outcome.routeSubject as RouteSubjectV1; + if (taskSubject.subjectKind !== "task") { throw new FabricTaskError("task layer requires TaskSubjectV1", "layer_subject_mismatch", "harness"); } - if (outcome.routeSubject.subjectKind !== "route") { + if (routeSubject.subjectKind !== "route") { throw new FabricTaskError("nested route subject required", "layer_subject_mismatch", "harness"); } if (!Number.isInteger(outcome.startedAt) || !Number.isInteger(outcome.completedAt) || outcome.completedAt < outcome.startedAt) { @@ -106,7 +158,10 @@ export function observationFromFabricOutcome( const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { const authority = loadFabricCaseAuthority(); - const caseRecord = authority.cases[0]!; + const caseRecord = authority.cases.find((row) => row.id === FABRIC_SCENARIO_ID); + if (!caseRecord) { + throw new FabricTaskError(`missing fabric scenario ${FABRIC_SCENARIO_ID}`, "harness_failure", "harness"); + } const scenarioDigest = fabricScenarioManifestDigest(caseRecord, authority); const suiteDigest = fabricSuiteManifestDigest(FABRIC_SUITE_ID, authority); const expandedScenario = expandFabricScenario(caseRecord, authority); @@ -206,11 +261,7 @@ export function persistFabricOutcome( const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { const { event } = observationFromFabricOutcome(outcome, { ...opts, artifactStore: store }); - const replay = existsSync(paths.ledgerPath) ? replayLabLedger(paths.ledgerPath) : null; - const alreadyPresent = replay?.events.some((row) => row.eventId === event.eventId) ?? false; - if (!alreadyPresent) { - appendLabEvent(paths.ledgerPath, event); - } + appendLabEventIfAbsent(paths.ledgerPath, event); return { event, ledgerPath: paths.ledgerPath }; } finally { if (ownsStore) store.close(); diff --git a/src/lab/fabric/patch.ts b/src/lab/fabric/patch.ts index 7a90319705..715acd5c02 100644 --- a/src/lab/fabric/patch.ts +++ b/src/lab/fabric/patch.ts @@ -22,7 +22,7 @@ export function parseSyntheticPatchV1(raw: unknown): SyntheticPatchV1 { throw new FabricTaskError("malformed producer outcome: operations", "malformed_producer_outcome", "harness"); } if (obj.operations.length !== 1) { - throw new FabricTaskError("exactly one patch operation is required", "budget_exhausted", "route"); + throw new FabricTaskError("exactly one patch operation is required", "malformed_producer_outcome", "harness"); } const opRaw = obj.operations[0]; if (!opRaw || typeof opRaw !== "object" || Array.isArray(opRaw)) { diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts index d9b1a2f8d7..98f176a922 100644 --- a/src/lab/fabric/scratch.ts +++ b/src/lab/fabric/scratch.ts @@ -1,10 +1,8 @@ import { - chmodSync, closeSync, constants as fsConstants, existsSync, lstatSync, - mkdirSync, openSync, readFileSync, readdirSync, @@ -14,18 +12,12 @@ import { } from "node:fs"; import { dirname, join, posix, relative, resolve, sep } from "node:path"; import { randomBytes } from "node:crypto"; -import { ensureLabDirs, labScratchDir } from "../paths"; +import { ensureLabDirs, ensureRestrictedDir, labScratchDir } from "../paths"; import { FABRIC_LIMITS, SYNTHETIC_BEFORE_UTF8, SYNTHETIC_VALUE_PATH } from "./constants"; import { FabricTaskError } from "./types"; const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; - -function ensureRestrictedDir(dir: string): void { - mkdirSync(dir, { recursive: true, mode: 0o700 }); - if (process.platform === "win32") return; - const mode = lstatSync(dir).mode & 0o777; - if (mode !== 0o700) chmodSync(dir, 0o700); -} +const FILE_MODE = 0o600; function assertRegularFile(stats: Stats, label: string): void { if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { @@ -33,6 +25,12 @@ function assertRegularFile(stats: Stats, label: string): void { } } +function assertRealDirectory(stats: Stats, label: string): void { + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new FabricTaskError(`${label} must be a real directory`, "sandbox_violation", "harness"); + } +} + /** Normalize and reject traversal / absolute / Windows drive / NUL paths. */ export function assertSafeRelativePosixPath(raw: string): string { if (typeof raw !== "string" || raw.length === 0) { @@ -54,14 +52,42 @@ export function assertSafeRelativePosixPath(raw: string): string { return normalized; } -export function resolveInsideScratch(scratchRoot: string, relativePath: string): string { - const safe = assertSafeRelativePosixPath(relativePath); - const root = resolve(scratchRoot); - const target = resolve(root, ...safe.split("/")); +function assertUnderScratchRoot(root: string, target: string): void { if (target !== root && !target.startsWith(root + sep)) { throw new FabricTaskError("path escapes scratch root", "sandbox_violation", "harness"); } - return target; +} + +/** + * Resolve a relative path under scratch without following intermediate symlinks. + * Missing final component is allowed (for create); intermediate components must exist. + */ +export function resolveInsideScratch(scratchRoot: string, relativePath: string, opts: { allowMissingFinal?: boolean } = {}): string { + const safe = assertSafeRelativePosixPath(relativePath); + const root = resolve(scratchRoot); + const rootStats = lstatSync(root); + assertRealDirectory(rootStats, "scratch root"); + + let current = root; + const parts = safe.split("/"); + for (let i = 0; i < parts.length; i++) { + const next = join(current, parts[i]!); + assertUnderScratchRoot(root, next); + const isFinal = i === parts.length - 1; + if (!existsSync(next)) { + if (isFinal && opts.allowMissingFinal) return next; + throw new FabricTaskError(isFinal ? `missing file ${relativePath}` : "missing intermediate directory", "sandbox_violation", "harness"); + } + const stats = lstatSync(next); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + if (!isFinal) { + assertRealDirectory(stats, next); + } + current = next; + } + return current; } export interface ScratchTree { @@ -75,13 +101,13 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { ensureRestrictedDir(base); const root = join(base, `fabric-${Date.now().toString(36)}-${randomBytes(6).toString("hex")}`); ensureRestrictedDir(root); - const valuePath = resolveInsideScratch(root, SYNTHETIC_VALUE_PATH); - ensureRestrictedDir(dirname(valuePath)); + ensureRestrictedDir(join(root, dirname(SYNTHETIC_VALUE_PATH))); + const valuePath = resolveInsideScratch(root, SYNTHETIC_VALUE_PATH, { allowMissingFinal: true }); const bytes = Buffer.from(SYNTHETIC_BEFORE_UTF8, "utf8"); if (bytes.byteLength > FABRIC_LIMITS.maxAggregateIoBytes) { throw new FabricTaskError("fixture exceeds io budget", "budget_exhausted", "harness"); } - const fd = openSync(valuePath, openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL)); + const fd = openSync(valuePath, openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL), FILE_MODE); try { writeSync(fd, bytes); } finally { @@ -110,11 +136,14 @@ export interface WalkedFile { byteLength: number; } -/** No-follow walk; rejects symlinks and special files. */ +/** No-follow walk; rejects symlinks and special files; enforces file/byte budgets. */ export function walkScratchFiles(scratchRoot: string): WalkedFile[] { const root = resolve(scratchRoot); + const rootStats = lstatSync(root); + assertRealDirectory(rootStats, "scratch root"); const out: WalkedFile[] = []; const stack: string[] = [root]; + let aggregateBytes = 0; while (stack.length > 0) { const current = stack.pop()!; const stats = lstatSync(current); @@ -129,6 +158,13 @@ export function walkScratchFiles(scratchRoot: string): WalkedFile[] { continue; } assertRegularFile(stats, current); + if (out.length >= FABRIC_LIMITS.maxScratchFiles) { + throw new FabricTaskError("scratch file count exceeds budget", "budget_exhausted", "harness"); + } + aggregateBytes += stats.size; + if (aggregateBytes > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("scratch aggregate bytes exceed budget", "budget_exhausted", "harness"); + } const rel = relative(root, current).split(sep).join("/"); const safe = assertSafeRelativePosixPath(rel); out.push({ relativePosix: safe, absolute: current, byteLength: stats.size }); @@ -148,9 +184,6 @@ export function walkScratchFiles(scratchRoot: string): WalkedFile[] { export function readScratchFileUtf8(scratchRoot: string, relativePath: string, maxBytes: number): string { const absolute = resolveInsideScratch(scratchRoot, relativePath); - if (!existsSync(absolute)) { - throw new FabricTaskError(`missing file ${relativePath}`, "sandbox_violation", "harness"); - } const stats = lstatSync(absolute); if (stats.isSymbolicLink()) { throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); @@ -163,20 +196,24 @@ export function readScratchFileUtf8(scratchRoot: string, relativePath: string, m } export function writeScratchFileUtf8(scratchRoot: string, relativePath: string, contentUtf8: string, maxBytes: number): number { - const absolute = resolveInsideScratch(scratchRoot, relativePath); + const absolute = resolveInsideScratch(scratchRoot, relativePath, { allowMissingFinal: true }); const bytes = Buffer.from(contentUtf8, "utf8"); if (bytes.byteLength > maxBytes) { throw new FabricTaskError("write exceeds io budget", "budget_exhausted", "environment"); } - ensureRestrictedDir(dirname(absolute)); + const parent = dirname(absolute); + const parentStats = lstatSync(parent); + assertRealDirectory(parentStats, parent); if (existsSync(absolute)) { const stats = lstatSync(absolute); if (stats.isSymbolicLink()) { throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); } assertRegularFile(stats, relativePath); + } else { + ensureRestrictedDir(parent); } - const fd = openSync(absolute, openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC)); + const fd = openSync(absolute, openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC), FILE_MODE); try { writeSync(fd, bytes); } finally { diff --git a/src/lab/fabric/subject.ts b/src/lab/fabric/subject.ts index 2c94bc15f2..1fc1fb7671 100644 --- a/src/lab/fabric/subject.ts +++ b/src/lab/fabric/subject.ts @@ -5,7 +5,11 @@ import { FABRIC_COMPATIBILITY_VERSION, FABRIC_TASK_CLASS_ID, FABRIC_TASK_CLASS_VERSION, + FABRIC_VERIFIER_ID, SANDBOX_PROFILE_V1, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, } from "./constants"; const SANDBOX_DOMAIN = "ocx-lab:fabric-sandbox-profile:v1"; @@ -19,7 +23,7 @@ export function sandboxProfileDigest(profile: unknown = SANDBOX_PROFILE_V1): str export function verifierManifestObject(): Record { return { schemaVersion: 1, - verifierId: "exact-tree-diff-v1", + verifierId: FABRIC_VERIFIER_ID, sort: "utf8-bytes-posix-path", rejectSymlinks: true, rejectSpecialFiles: true, @@ -28,9 +32,9 @@ export function verifierManifestObject(): Record { allowDeletes: false, allowRenames: false, requiredChange: { - path: "src/value.txt", - beforeUtf8: "before\n", - afterUtf8: "after\n", + path: SYNTHETIC_VALUE_PATH, + beforeUtf8: SYNTHETIC_BEFORE_UTF8, + afterUtf8: SYNTHETIC_AFTER_UTF8, }, }; } @@ -46,14 +50,14 @@ export function taskFixtureObject(): Record { taskClassVersion: FABRIC_TASK_CLASS_VERSION, files: [ { - path: "src/value.txt", - contentUtf8: "before\n", + path: SYNTHETIC_VALUE_PATH, + contentUtf8: SYNTHETIC_BEFORE_UTF8, }, ], requestedFinal: [ { - path: "src/value.txt", - contentUtf8: "after\n", + path: SYNTHETIC_VALUE_PATH, + contentUtf8: SYNTHETIC_AFTER_UTF8, }, ], }; diff --git a/src/lab/fabric/types.ts b/src/lab/fabric/types.ts index 951cae7a3b..004f227760 100644 --- a/src/lab/fabric/types.ts +++ b/src/lab/fabric/types.ts @@ -1,10 +1,14 @@ +import type { FailureClassification } from "../conformance/types"; import type { ObservationOutcome } from "../constants"; import type { FailureRecordV1, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; export type FabricOutcomeKind = ObservationOutcome; +export type FabricFailureClass = FailureClassification; + export interface FabricLimitsV1 { maxFiles: number; + maxScratchFiles: number; maxAggregateIoBytes: number; maxPatchOperations: number; totalTimeoutMs: number; @@ -76,13 +80,15 @@ export type SyntheticPatchProducer = (ctx: { taskClassId: string; taskClassVersion: string; scratchRoot: string; + /** Reset the inactivity deadline while the producer is making progress. */ + reportActivity: () => void; }) => SyntheticPatchV1 | Promise; export class FabricTaskError extends Error { override readonly name = "FabricTaskError"; constructor( message: string, - readonly code: string, + readonly code: FabricFailureClass, readonly attribution: FailureRecordV1["attribution"] = "harness", ) { super(message); diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 204d96dc58..3ff5af4a80 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -1,11 +1,13 @@ import { closeSync, + constants as fsConstants, existsSync, fsyncSync, mkdirSync, openSync, readSync, statSync, + unlinkSync, writeSync, } from "node:fs"; import { dirname } from "node:path"; @@ -21,6 +23,54 @@ export interface LedgerStore { replay(): ReplayResult; } +const eventIdIndexByLedger = new Map>(); + +function sleepSyncMs(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) { + /* spin */ + } +} + +function withLedgerLock(ledgerPath: string, fn: () => T): T { + const lockPath = `${ledgerPath}.lock`; + mkdirSync(dirname(ledgerPath), { recursive: true, mode: 0o700 }); + const deadline = Date.now() + 5_000; + let lockFd: number | undefined; + while (lockFd === undefined) { + try { + lockFd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); + } catch (error) { + if (Date.now() >= deadline) throw error; + sleepSyncMs(10); + } + } + try { + return fn(); + } finally { + closeSync(lockFd); + try { + unlinkSync(lockPath); + } catch { + /* best-effort */ + } + } +} + +function loadEventIdIndex(ledgerPath: string): Set { + let index = eventIdIndexByLedger.get(ledgerPath); + if (!index) { + index = new Set(); + if (existsSync(ledgerPath)) { + for (const event of replayLabLedger(ledgerPath).events) { + index.add(event.eventId); + } + } + eventIdIndexByLedger.set(ledgerPath, index); + } + return index; +} + /** Durable append of one validated event as a single JSONL line + fsync. */ export function appendLabEvent(ledgerPath: string, event: LabEvent): void { const validated = validateLabEvent(event); @@ -41,6 +91,28 @@ export function appendLabEvent(ledgerPath: string, event: LabEvent): void { } finally { closeSync(fd); } + loadEventIdIndex(ledgerPath).add(validated.eventId); +} + +/** + * Append only when eventId is absent. Uses an exclusive lock file plus a + * process-local event-id index refreshed under the lock. + */ +export function appendLabEventIfAbsent(ledgerPath: string, event: LabEvent): boolean { + const validated = validateLabEvent(event); + return withLedgerLock(ledgerPath, () => { + // Refresh from disk under the lock so concurrent writers are visible. + const fresh = new Set(); + if (existsSync(ledgerPath)) { + for (const row of replayLabLedger(ledgerPath).events) { + fresh.add(row.eventId); + } + } + eventIdIndexByLedger.set(ledgerPath, fresh); + if (fresh.has(validated.eventId)) return false; + appendLabEvent(ledgerPath, validated); + return true; + }); } function processLine( diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 35dbd06001..ba353f3050 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -90,7 +90,9 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { if (result.passed) return "pass"; switch (result.classification) { case "timeout": + case "inactivity_timeout": case "budget_exhausted": + case "sandbox_violation": case "authentication_blocked": case "quota_blocked": case "region_blocked": @@ -99,6 +101,8 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { return "blocked"; case "inconclusive": case "harness_failure": + case "malformed_producer_outcome": + case "layer_subject_mismatch": return "inconclusive"; case "protocol_failure": case "capability_failure": diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts index 2169daae1e..a8c3e8bb41 100644 --- a/src/lab/observe/from-live.ts +++ b/src/lab/observe/from-live.ts @@ -20,7 +20,9 @@ function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcom if (result.passed) return "pass"; switch (result.classification) { case "timeout": + case "inactivity_timeout": case "budget_exhausted": + case "sandbox_violation": case "authentication_blocked": case "quota_blocked": case "region_blocked": @@ -33,7 +35,13 @@ function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcom return "fail"; case "inconclusive": case "harness_failure": + case "malformed_producer_outcome": + case "layer_subject_mismatch": return "inconclusive"; + default: { + const _never: never = result.classification; + throw new Error(`unmapped failure classification: ${String(_never)}`); + } } } diff --git a/src/lab/paths.ts b/src/lab/paths.ts index dfa51f4eee..390308658b 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -1,11 +1,12 @@ -import { chmodSync, mkdirSync, statSync } from "node:fs"; +import { chmodSync, mkdirSync, lstatSync } from "node:fs"; import { join } from "node:path"; import { getConfigDir } from "../config"; -function ensureRestrictedDir(dir: string): void { +/** Create (or harden) a directory to mode 0o700 without following symlinks. */ +export function ensureRestrictedDir(dir: string): void { mkdirSync(dir, { recursive: true, mode: 0o700 }); if (process.platform === "win32") return; - const mode = statSync(dir).mode & 0o777; + const mode = lstatSync(dir).mode & 0o777; if (mode !== 0o700) chmodSync(dir, 0o700); } diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index 09c960690b..cbc0a2acc9 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -301,6 +301,50 @@ function isMatchedCapabilityAbsenceControl(obs: ObservationEvent): boolean { ); } +function evaluateRequiredPassVerdict(input: { + suiteManifest: SuiteManifestV1 | null; + ordered: ObservationEvent[]; + executionMode: ObservationEvent["executionMode"]; + subject?: ObservationEvent["subject"]; + routeSupportedClaims?: readonly string[]; + fabricCapability?: { + harnessFeatures: readonly string[]; + platforms: readonly string[]; + routePreconditions: readonly string[]; + }; + loadScenarioManifest?: (digest: string) => Record | null; + loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"]; + asOf: number; +}): { verdict: CompatibilityVerdict; notes: string[] } { + const notes: string[] = []; + if (!input.suiteManifest) { + return { verdict: "PROBED", notes: ["suite_manifest_unavailable"] }; + } + const evaluation = evaluateAllApplicableRequiredPassV1( + input.suiteManifest, + input.ordered, + input.executionMode, + { + subject: input.subject, + routeSupportedClaims: input.routeSupportedClaims, + fabricCapability: input.fabricCapability, + loadScenarioManifest: input.loadScenarioManifest, + loadScenarioRequirements: input.loadScenarioRequirements, + asOf: input.asOf, + }, + ); + notes.push(...evaluation.notes); + if (evaluation.applicableRequiredScenarioIds.length === 0) { + return { verdict: "UNKNOWN", notes }; + } + if (evaluation.canVerify) { + notes.push("all-applicable-required-pass-v1"); + return { verdict: "VERIFIED", notes }; + } + notes.push("incomplete_required_coverage"); + return { verdict: "PROBED", notes }; +} + function projectObservationGroup( key: ProjectionKey, ordered: ObservationEvent[], @@ -310,6 +354,11 @@ function projectObservationGroup( loadScenarioManifest?: (digest: string) => Record | null; loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"]; routeSupportedClaims?: readonly string[]; + fabricCapability?: { + harnessFeatures: readonly string[]; + platforms: readonly string[]; + routePreconditions: readonly string[]; + }; } = {}, ): DerivedVerdict { const contributing: string[] = []; @@ -353,87 +402,47 @@ function projectObservationGroup( verdict = "PROBED"; notes.push("mixed_execution_modes"); } else if (key.evidenceLayer === "protocol_conformance" && newestCurrent?.executionMode === "fixture") { - if (!suiteManifest) { - verdict = "PROBED"; - notes.push("suite_manifest_unavailable"); - } else { - const evaluation = evaluateAllApplicableRequiredPassV1( - suiteManifest, - ordered, - newestCurrent.executionMode, - { - subject: newestCurrent.subject.subjectKind === "protocol" ? newestCurrent.subject : undefined, - loadScenarioManifest: opts.loadScenarioManifest, - loadScenarioRequirements: opts.loadScenarioRequirements, - asOf, - }, - ); - notes.push(...evaluation.notes); - if (evaluation.applicableRequiredScenarioIds.length === 0) { - verdict = "UNKNOWN"; - } else if (evaluation.canVerify) { - verdict = "VERIFIED"; - notes.push("all-applicable-required-pass-v1"); - } else { - verdict = "PROBED"; - notes.push("incomplete_required_coverage"); - } - } + const evaluated = evaluateRequiredPassVerdict({ + suiteManifest, + ordered, + executionMode: newestCurrent.executionMode, + subject: newestCurrent.subject.subjectKind === "protocol" ? newestCurrent.subject : undefined, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }); + verdict = evaluated.verdict; + notes.push(...evaluated.notes); } else if (key.evidenceLayer === "live_route_compatibility" && newestCurrent?.executionMode === "live") { - if (!suiteManifest) { - verdict = "PROBED"; - notes.push("suite_manifest_unavailable"); - } else { - const evaluation = evaluateAllApplicableRequiredPassV1( - suiteManifest, - ordered, - newestCurrent.executionMode, - { - subject: newestCurrent.subject.subjectKind === "route" ? newestCurrent.subject : undefined, - routeSupportedClaims: opts.routeSupportedClaims, - loadScenarioManifest: opts.loadScenarioManifest, - loadScenarioRequirements: opts.loadScenarioRequirements, - asOf, - }, - ); - notes.push(...evaluation.notes); - if (evaluation.applicableRequiredScenarioIds.length === 0) { - verdict = "UNKNOWN"; - } else if (evaluation.canVerify) { - verdict = "VERIFIED"; - notes.push("all-applicable-required-pass-v1"); - } else { - verdict = "PROBED"; - notes.push("incomplete_required_coverage"); - } - } + const evaluated = evaluateRequiredPassVerdict({ + suiteManifest, + ordered, + executionMode: newestCurrent.executionMode, + subject: newestCurrent.subject.subjectKind === "route" ? newestCurrent.subject : undefined, + routeSupportedClaims: opts.routeSupportedClaims, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }); + verdict = evaluated.verdict; + notes.push(...evaluated.notes); } else if (key.evidenceLayer === "task_effectiveness" && newestCurrent?.executionMode === "fabric") { - if (!suiteManifest) { - verdict = "PROBED"; - notes.push("suite_manifest_unavailable"); - } else { - const evaluation = evaluateAllApplicableRequiredPassV1( - suiteManifest, - ordered, - newestCurrent.executionMode, - { - subject: newestCurrent.subject.subjectKind === "task" ? newestCurrent.subject : undefined, - loadScenarioManifest: opts.loadScenarioManifest, - loadScenarioRequirements: opts.loadScenarioRequirements, - asOf, - }, - ); - notes.push(...evaluation.notes); - if (evaluation.applicableRequiredScenarioIds.length === 0) { - verdict = "UNKNOWN"; - } else if (evaluation.canVerify) { - verdict = "VERIFIED"; - notes.push("all-applicable-required-pass-v1"); - } else { - verdict = "PROBED"; - notes.push("incomplete_required_coverage"); - } - } + const evaluated = evaluateRequiredPassVerdict({ + suiteManifest, + ordered, + executionMode: newestCurrent.executionMode, + subject: newestCurrent.subject.subjectKind === "task" ? newestCurrent.subject : undefined, + fabricCapability: opts.fabricCapability ?? { + harnessFeatures: ["fabric-scratch-v1"], + platforms: [process.platform, "*"], + routePreconditions: ["exact-route-subject"], + }, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }); + verdict = evaluated.verdict; + notes.push(...evaluated.notes); } else { verdict = "PROBED"; } diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index 0edc0decad..8efb68dd8d 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -19,6 +19,9 @@ export interface ScenarioRequirements { upstreamProtocols?: string[]; surfaces?: string[]; requiredClaims?: string[]; + requiredHarnessFeatures?: string[]; + platforms?: string[]; + routePreconditions?: string[]; freshness?: { maxAgeMs: number | null }; } @@ -71,6 +74,24 @@ export function routeSubjectApplicableToRequirements( ); } +/** Task-layer applicability against harness/platform/precondition state. */ +export function taskSubjectApplicableToRequirements( + requirements: ScenarioRequirements, + capability: { + harnessFeatures: readonly string[]; + platforms: readonly string[]; + routePreconditions: readonly string[]; + }, +): boolean { + const platforms = requirements.platforms ?? []; + const features = requirements.requiredHarnessFeatures ?? []; + const preconditions = requirements.routePreconditions ?? []; + const platformOk = platforms.includes("*") || platforms.some((platform) => capability.platforms.includes(platform)); + const featuresOk = features.every((feature) => capability.harnessFeatures.includes(feature)); + const preconditionsOk = preconditions.every((item) => capability.routePreconditions.includes(item)); + return platformOk && featuresOk && preconditionsOk; +} + function isNonNegativeInteger(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0; } @@ -101,9 +122,28 @@ function scenarioContractFromManifest( if (!inboundProtocols || !upstreamProtocols || !surfaces) return null; const requiredClaims = row.requiredClaims === undefined ? [] : parseStringArray(row.requiredClaims); if (!requiredClaims) return null; + const requiredHarnessFeatures = row.requiredHarnessFeatures === undefined + ? [] + : parseStringArray(row.requiredHarnessFeatures); + if (!requiredHarnessFeatures) return null; + const platforms = row.platforms === undefined ? ["*"] : parseStringArray(row.platforms); + if (!platforms) return null; + const routePreconditions = row.routePreconditions === undefined + ? [] + : parseStringArray(row.routePreconditions); + if (!routePreconditions) return null; const freshness = parseFreshness(scenarioManifest.freshness); if (!freshness) return null; - return { inboundProtocols, upstreamProtocols, surfaces, requiredClaims, freshness }; + return { + inboundProtocols, + upstreamProtocols, + surfaces, + requiredClaims, + requiredHarnessFeatures, + platforms, + routePreconditions, + freshness, + }; } function effectiveMaxAgeMs( @@ -142,6 +182,12 @@ export function evaluateAllApplicableRequiredPassV1( subject?: ProtocolSubjectV1 | RouteSubjectV1 | TaskSubjectV1; /** For live projection this must come from validated current claim snapshots for subjectId. */ routeSupportedClaims?: readonly string[]; + /** Fabric harness/platform/precondition state for task_effectiveness applicability. */ + fabricCapability?: { + harnessFeatures: readonly string[]; + platforms: readonly string[]; + routePreconditions: readonly string[]; + }; loadScenarioManifest?: LoadScenarioManifest; loadScenarioRequirements?: LoadScenarioRequirements; asOf?: number; @@ -218,6 +264,14 @@ export function evaluateAllApplicableRequiredPassV1( if (suiteManifest.evidenceLayer === "live_route_compatibility" && opts.subject?.subjectKind === "route") { if (!routeSubjectApplicableToRequirements(requirements, opts.subject, opts.routeSupportedClaims!)) continue; } + if (suiteManifest.evidenceLayer === "task_effectiveness" && opts.subject?.subjectKind === "task") { + const capability = opts.fabricCapability ?? { + harnessFeatures: ["fabric-scratch-v1"], + platforms: [process.platform, "*"], + routePreconditions: ["exact-route-subject"], + }; + if (!taskSubjectApplicableToRequirements(requirements, capability)) continue; + } applicableRequired.push(s.id); scenarioMaxAgeById.set(s.id, requirements.freshness?.maxAgeMs ?? null); } diff --git a/src/lab/query/catalog.ts b/src/lab/query/catalog.ts index 45d8cd4a3a..d3f8159333 100644 --- a/src/lab/query/catalog.ts +++ b/src/lab/query/catalog.ts @@ -16,19 +16,29 @@ import { } from "../conformance/suite-manifest"; import { discoverFabricScenarios, - fabricScenarioManifestDigest, + expandFabricScenario, fabricSuiteManifestDigest, loadFabricCaseAuthority, } from "../fabric/manifest"; import { scenarioManifestDigest } from "../digest"; -import type { CaseRecord } from "../conformance/types"; +import type { CaseRecord, EvidenceLayer, VerificationRole } from "../conformance/types"; import type { CatalogFilters, CatalogScenarioDto } from "./types"; -function mapCaseToCatalogScenario( +type CatalogAuthority = { + manifestDefaults: { + version: string; + suiteVersion: string; + evidenceLayer: EvidenceLayer; + verificationRole: VerificationRole; + freshness: { maxAgeMs: number | null }; + }; +}; + +function mapCaseToCatalogScenario( caseRecord: CaseRecord, - authority: ReturnType, - suiteDigestFn: (caseRecord: CaseRecord, authority: ReturnType) => string, - expandFn: (caseRecord: CaseRecord, authority: ReturnType) => Record, + authority: TAuthority, + suiteDigestFn: (caseRecord: CaseRecord, authority: TAuthority) => string, + expandFn: (caseRecord: CaseRecord, authority: TAuthority) => Record, ): CatalogScenarioDto { const expanded = expandFn(caseRecord, authority); return { @@ -73,23 +83,12 @@ export function queryLabCatalog(filters: CatalogFilters = {}): CatalogScenarioDt const suites = filters.suiteId ? [filters.suiteId] : undefined; const scenarios = discoverFabricScenarios(authority, suites ?? undefined); for (const caseRecord of scenarios) { - items.push({ - scenarioId: caseRecord.id, - scenarioVersion: authority.manifestDefaults.version, - evidenceLayer: authority.manifestDefaults.evidenceLayer, - suiteId: caseRecord.suite, - suiteVersion: authority.manifestDefaults.suiteVersion, - capability: caseRecord.capability, - verificationRole: caseRecord.verificationRole ?? authority.manifestDefaults.verificationRole, - requirements: { - inboundProtocols: [...caseRecord.requirements.inboundProtocols], - upstreamProtocols: [...caseRecord.requirements.upstreamProtocols], - surfaces: [...caseRecord.requirements.surfaces], - }, - freshness: { ...authority.manifestDefaults.freshness }, - scenarioManifestDigest: fabricScenarioManifestDigest(caseRecord, authority), - suiteManifestDigest: fabricSuiteManifestDigest(caseRecord.suite, authority), - }); + items.push(mapCaseToCatalogScenario( + caseRecord, + authority, + (row, auth) => fabricSuiteManifestDigest(row.suite, auth), + expandFabricScenario, + )); } } return items.sort((a, b) => { diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index cd982a1f54..e406b4e4ae 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -17,7 +17,8 @@ import { buildTaskSubjectV1, correctSyntheticPatch, createSyntheticScratch, - fabricScratchCapabilityProbe, + fabricDeclaredSandboxPolicy, + FABRIC_VERIFIER_ID, LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, observationFromFabricOutcome, @@ -29,17 +30,24 @@ import { runFabricSyntheticPatchTask, sandboxProfileDigest, subjectIdForSubject, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, taskFixtureDigest, + taskFixtureObject, taskSubjectId, verifierManifestDigest, - writeScratchFileUtf8, + verifierManifestObject, type RouteSubjectV1, } from "../src/lab"; +import { FabricTaskError } from "../src/lab/fabric/types"; +import { writeScratchFileUtf8, readScratchFileUtf8 } from "../src/lab/fabric/scratch"; import { routingProfileIssues } from "../src/routing/profile"; import { buildInvalidationIndex } from "../src/lab/ledger/invalidation"; import { verifyExactTreeDiffV1 } from "../src/lab/fabric/verifier"; import { parseSyntheticPatchV1 } from "../src/lab/fabric/patch"; -import { FABRIC_LIMITS, SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "../src/lab/fabric/constants"; +import { FABRIC_LIMITS } from "../src/lab/fabric/constants"; +import { taskSubjectApplicableToRequirements } from "../src/lab/projection/verification"; const HOMES: string[] = []; @@ -201,12 +209,30 @@ describe("CL-07 task effectiveness producer", () => { const scratch = createSyntheticScratch(home); try { const fifo = join(scratch.root, "src", "fifo"); + const created = Bun.spawnSync(["mkfifo", fifo], { stdout: "pipe", stderr: "pipe" }); + if (created.exitCode !== 0) return; + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + } finally { + scratch.cleanup(); + } + }); + + test("intermediate symlink cannot redirect scratch IO", () => { + if (process.platform === "win32") return; + const home = tempHome(); + const scratch = createSyntheticScratch(home); + const outside = join(home, "outside.txt"); + writeFileSync(outside, "secret\n"); + try { + const srcDir = join(scratch.root, "src"); + rmSync(srcDir, { recursive: true, force: true }); try { - symlinkSync("/dev/null", fifo); + symlinkSync(home, srcDir); } catch { return; } - expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(() => readScratchFileUtf8(scratch.root, SYNTHETIC_VALUE_PATH, FABRIC_LIMITS.maxAggregateIoBytes)).toThrow(); + expect(() => writeScratchFileUtf8(scratch.root, SYNTHETIC_VALUE_PATH, "x\n", FABRIC_LIMITS.maxAggregateIoBytes)).toThrow(); } finally { scratch.cleanup(); } @@ -243,8 +269,25 @@ describe("CL-07 task effectiveness producer", () => { expect(["timeout", "inactivity_timeout"]).toContain(outcome.failure?.code ?? ""); }, 20_000); - test("network / user MCP / arbitrary shell remain unavailable", () => { - expect(fabricScratchCapabilityProbe()).toEqual({ + test("activity resets inactivity deadline within total budget", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: async ({ reportActivity }) => { + for (let i = 0; i < 3; i++) { + reportActivity(); + await Bun.sleep(Math.floor(FABRIC_LIMITS.inactivityTimeoutMs * 0.6)); + } + return correctSyntheticPatch(); + }, + }); + expect(outcome.outcome).toBe("pass"); + }, 30_000); + + test("declared sandbox policy denies network MCP shell and user repo", () => { + expect(fabricDeclaredSandboxPolicy()).toEqual({ network: false, userMcp: false, arbitraryShell: false, @@ -277,6 +320,62 @@ describe("CL-07 task effectiveness producer", () => { schemaVersion: 1, taskSubject: { subjectKind: "route" }, })).toThrow(); + let incompleteError: unknown; + try { + observationFromFabricOutcome({ schemaVersion: 1 }); + } catch (error) { + incompleteError = error; + } + expect((incompleteError as FabricTaskError | undefined)?.name).toBe("FabricTaskError"); + }); + + test("fixture and verifier manifests derive from frozen constants", () => { + const verifier = verifierManifestObject() as { + verifierId: string; + requiredChange: { path: string; beforeUtf8: string; afterUtf8: string }; + }; + expect(verifier.verifierId).toBe(FABRIC_VERIFIER_ID); + expect(verifier.requiredChange).toEqual({ + path: SYNTHETIC_VALUE_PATH, + beforeUtf8: SYNTHETIC_BEFORE_UTF8, + afterUtf8: SYNTHETIC_AFTER_UTF8, + }); + const fixture = taskFixtureObject() as { + files: Array<{ path: string; contentUtf8: string }>; + requestedFinal: Array<{ path: string; contentUtf8: string }>; + }; + expect(fixture.files[0]).toEqual({ path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_BEFORE_UTF8 }); + expect(fixture.requestedFinal[0]).toEqual({ path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }); + }); + + test("task applicability excludes mismatched platform and harness features", () => { + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: ["fabric-scratch-v1"], + platforms: ["linux"], + routePreconditions: ["exact-route-subject"], + }, { + harnessFeatures: ["fabric-scratch-v1"], + platforms: ["win32"], + routePreconditions: ["exact-route-subject"], + })).toBe(false); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: ["fabric-scratch-v1", "missing-feature"], + platforms: ["*"], + routePreconditions: ["exact-route-subject"], + }, { + harnessFeatures: ["fabric-scratch-v1"], + platforms: ["win32"], + routePreconditions: ["exact-route-subject"], + })).toBe(false); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: ["fabric-scratch-v1"], + platforms: ["*"], + routePreconditions: ["exact-route-subject"], + }, { + harnessFeatures: ["fabric-scratch-v1"], + platforms: ["win32"], + routePreconditions: ["exact-route-subject"], + })).toBe(true); }); test("duplicate outcome delivery is idempotent; distinct attempts remain distinct", async () => { From 5dd527c80324a10c9ffb5643aab8d0a71eb6bb4d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:59:21 +0200 Subject: [PATCH 05/24] fix(lab): address second CodeRabbit round for CL-07 fabric producer Harden fabric observation validation and scratch IO, recover stale ledger locks, attribute inactivity_timeout as environment, and add regression tests. --- src/lab/fabric/executor.ts | 16 +- src/lab/fabric/observe.ts | 208 +++++++++++++++++++---- src/lab/fabric/scratch.ts | 258 ++++++++++++++++++++++++++--- src/lab/ledger/store.ts | 46 ++++- src/lab/live/transport.ts | 4 +- src/lab/observe/from-live.ts | 2 +- src/lab/paths.ts | 12 +- src/lab/projection/verification.ts | 2 +- tests/lab-fabric-task.test.ts | 96 +++++++++++ tests/lab-live-probe.test.ts | 19 +++ 10 files changed, 594 insertions(+), 69 deletions(-) diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts index 30bdd9773e..1b46c4dccb 100644 --- a/src/lab/fabric/executor.ts +++ b/src/lab/fabric/executor.ts @@ -107,13 +107,15 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) let patchRaw: unknown; try { const controller = createTimeoutController(FABRIC_LIMITS.totalTimeoutMs, FABRIC_LIMITS.inactivityTimeoutMs, options); - const producePromise = Promise.resolve(options.producePatch({ - taskClassId: FABRIC_TASK_CLASS_ID, - taskClassVersion: FABRIC_TASK_CLASS_VERSION, - scratchRoot: scratch.root, - reportActivity: controller.reportActivity, - })); - patchRaw = await controller.race(producePromise); + const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + patchRaw = await controller.race( + sleep(0).then(() => options.producePatch({ + taskClassId: FABRIC_TASK_CLASS_ID, + taskClassVersion: FABRIC_TASK_CLASS_VERSION, + scratchRoot: scratch!.root, + reportActivity: controller.reportActivity, + })), + ); } catch (error) { const completedAt = options.now?.() ?? Date.now(); usage.elapsedMs = completedAt - startedAt; diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 2d7d5d9c50..61aeb69653 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -5,10 +5,13 @@ import { LAB_PRODUCER, LAB_PRODUCER_VERSION, OBSERVATION_LIMIT_NAMES, + OUTCOMES, } from "../constants"; -import { fixtureDigest } from "../digest"; +import { FAILURE_CLASSIFICATIONS } from "../conformance/types"; +import { fixtureDigest, isSha256Hex } from "../digest"; import type { ObservationEvent, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; -import { assignEventId } from "../events/validate"; +import { LabValidationError } from "../events/errors"; +import { assignEventId, validateSubject } from "../events/validate"; import { appendLabEventIfAbsent } from "../ledger/store"; import { ensureLabDirs } from "../paths"; import { @@ -17,6 +20,8 @@ import { FABRIC_SCENARIO_VERSION, FABRIC_SUITE_ID, FABRIC_SUITE_VERSION, + FABRIC_LIMITS, + FABRIC_VERIFIER_ID, } from "./constants"; import { expandFabricScenario, @@ -25,7 +30,7 @@ import { fabricSuiteManifestDigest, loadFabricCaseAuthority, } from "./manifest"; -import type { FabricTaskOutcomeV1 } from "./types"; +import type { FabricLimitsV1, FabricTaskOutcomeV1 } from "./types"; import { FabricTaskError } from "./types"; export interface PersistFabricOptions { @@ -63,6 +68,14 @@ const OUTCOME_KEYS = new Set([ "sourceRefs", ]); +const VERIFIER_KEYS = new Set(["verifierId", "manifestDigest", "passed", "pathSummaries", "reason"]); +const PATH_SUMMARY_KEYS = new Set(["path", "kind", "beforeDigest", "afterDigest", "reason"]); +const PATH_SUMMARY_KINDS = new Set(["unchanged", "modified", "added", "deleted", "rejected"]); +const USAGE_KEYS = ["inputBytes", "outputBytes", "patchOperations", "filesTouched", "artifactBytes", "elapsedMs", "inactiveMs"] as const; +const LIMIT_KEYS = Object.keys(FABRIC_LIMITS) as Array; +const FAILURE_KEYS = new Set(["class", "code", "retryable", "attribution"]); +const FAILURE_ATTRIBUTIONS = new Set(["opencodex", "route", "environment", "harness"]); + function assertPlainObject(value: unknown, label: string): Record { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new FabricTaskError(`malformed producer outcome: ${label}`, "malformed_producer_outcome", "harness"); @@ -86,6 +99,128 @@ function assertIntegerField(obj: Record, key: string): number { return value; } +function assertNonNegativeIntegerField(obj: Record, key: string): number { + const value = assertIntegerField(obj, key); + if (value < 0) { + throw new FabricTaskError(`malformed producer outcome: ${key}`, "malformed_producer_outcome", "harness"); + } + return value; +} + +function wrapValidationError(error: unknown): void { + if (error instanceof LabValidationError) { + throw new FabricTaskError(error.message, "malformed_producer_outcome", "harness"); + } + throw error; +} + +function validateFabricVerifier(raw: Record): void { + for (const key of Object.keys(raw)) { + if (!VERIFIER_KEYS.has(key)) { + throw new FabricTaskError(`unknown verifier field ${key}`, "malformed_producer_outcome", "harness"); + } + } + if (raw.verifierId !== FABRIC_VERIFIER_ID) { + throw new FabricTaskError("malformed producer outcome: verifierId", "malformed_producer_outcome", "harness"); + } + const manifestDigest = assertStringField(raw, "manifestDigest"); + if (!isSha256Hex(manifestDigest)) { + throw new FabricTaskError("malformed producer outcome: manifestDigest", "malformed_producer_outcome", "harness"); + } + if (typeof raw.passed !== "boolean") { + throw new FabricTaskError("malformed producer outcome: passed", "malformed_producer_outcome", "harness"); + } + if (!Array.isArray(raw.pathSummaries)) { + throw new FabricTaskError("malformed producer outcome: pathSummaries", "malformed_producer_outcome", "harness"); + } + for (const row of raw.pathSummaries) { + const summary = assertPlainObject(row, "pathSummaries[]"); + for (const key of Object.keys(summary)) { + if (!PATH_SUMMARY_KEYS.has(key)) { + throw new FabricTaskError(`unknown path summary field ${key}`, "malformed_producer_outcome", "harness"); + } + } + assertStringField(summary, "path"); + const kind = assertStringField(summary, "kind"); + if (!PATH_SUMMARY_KINDS.has(kind)) { + throw new FabricTaskError("malformed producer outcome: path summary kind", "malformed_producer_outcome", "harness"); + } + if (summary.beforeDigest !== undefined && !isSha256Hex(assertStringField(summary, "beforeDigest"))) { + throw new FabricTaskError("malformed producer outcome: beforeDigest", "malformed_producer_outcome", "harness"); + } + if (summary.afterDigest !== undefined && !isSha256Hex(assertStringField(summary, "afterDigest"))) { + throw new FabricTaskError("malformed producer outcome: afterDigest", "malformed_producer_outcome", "harness"); + } + if (summary.reason !== undefined && typeof summary.reason !== "string") { + throw new FabricTaskError("malformed producer outcome: path summary reason", "malformed_producer_outcome", "harness"); + } + } + if (raw.reason !== undefined && typeof raw.reason !== "string") { + throw new FabricTaskError("malformed producer outcome: reason", "malformed_producer_outcome", "harness"); + } +} + +function validateFabricUsage(raw: Record): void { + for (const key of USAGE_KEYS) { + assertNonNegativeIntegerField(raw, key); + } +} + +function validateFabricLimits(raw: Record): void { + for (const key of LIMIT_KEYS) { + assertNonNegativeIntegerField(raw, key); + } +} + +function validateFailureRecord(raw: Record): void { + for (const key of Object.keys(raw)) { + if (!FAILURE_KEYS.has(key)) { + throw new FabricTaskError(`unknown failure field ${key}`, "malformed_producer_outcome", "harness"); + } + } + const failureClass = assertStringField(raw, "class"); + if (!FAILURE_CLASSIFICATIONS.includes(failureClass as typeof FAILURE_CLASSIFICATIONS[number])) { + throw new FabricTaskError("malformed producer outcome: failure.class", "malformed_producer_outcome", "harness"); + } + assertStringField(raw, "code"); + if (typeof raw.retryable !== "boolean") { + throw new FabricTaskError("malformed producer outcome: failure.retryable", "malformed_producer_outcome", "harness"); + } + const attribution = assertStringField(raw, "attribution"); + if (!FAILURE_ATTRIBUTIONS.has(attribution)) { + throw new FabricTaskError("malformed producer outcome: failure.attribution", "malformed_producer_outcome", "harness"); + } +} + +function routeSubjectsMatch(top: RouteSubjectV1, nested: RouteSubjectV1): boolean { + return JSON.stringify(top) === JSON.stringify(nested); +} + +function sanitizedVerifierSummary(outcome: FabricTaskOutcomeV1): Record { + const verifier = { + verifierId: outcome.verifier.verifierId, + manifestDigest: outcome.verifier.manifestDigest, + passed: outcome.verifier.passed, + pathSummaries: outcome.verifier.pathSummaries.map((row) => ({ + path: row.path, + kind: row.kind, + ...(row.beforeDigest ? { beforeDigest: row.beforeDigest } : {}), + ...(row.afterDigest ? { afterDigest: row.afterDigest } : {}), + ...(row.reason ? { reason: truncateUtf8(sanitizeDiagnostic(row.reason), 512) } : {}), + })), + ...(outcome.verifier.reason + ? { reason: truncateUtf8(sanitizeDiagnostic(outcome.verifier.reason), 512) } + : {}), + }; + return { + scenarioId: FABRIC_SCENARIO_ID, + passed: outcome.verifier.passed, + verifier, + usage: outcome.usage, + outcome: outcome.outcome, + }; +} + export function assertFabricOutcomeV1(raw: unknown): FabricTaskOutcomeV1 { if (!raw || typeof raw !== "object" || Array.isArray(raw)) { throw new FabricTaskError("malformed producer outcome", "malformed_producer_outcome", "harness"); @@ -101,37 +236,60 @@ export function assertFabricOutcomeV1(raw: unknown): FabricTaskOutcomeV1 { } const taskSubjectObj = assertPlainObject(obj.taskSubject, "taskSubject"); - if (taskSubjectObj.subjectKind !== "task") { - throw new FabricTaskError("task layer rejects non-task subjects", "layer_subject_mismatch", "harness"); - } - const routeFromTask = assertPlainObject(taskSubjectObj.routeSubject, "taskSubject.routeSubject"); - if (routeFromTask.subjectKind !== "route") { - throw new FabricTaskError("nested route subject required", "layer_subject_mismatch", "harness"); - } - const routeSubjectObj = assertPlainObject(obj.routeSubject, "routeSubject"); - if (routeSubjectObj.subjectKind !== "route") { - throw new FabricTaskError("nested route subject required", "layer_subject_mismatch", "harness"); + + let taskSubject!: TaskSubjectV1; + let routeSubject!: RouteSubjectV1; + try { + taskSubject = validateSubject(taskSubjectObj, "task_effectiveness") as TaskSubjectV1; + routeSubject = validateSubject(routeSubjectObj, "live_route_compatibility") as RouteSubjectV1; + } catch (error) { + wrapValidationError(error); + } + if (!routeSubjectsMatch(routeSubject, taskSubject.routeSubject)) { + throw new FabricTaskError("contradictory route subjects", "layer_subject_mismatch", "harness"); } assertStringField(obj, "taskClassId"); assertStringField(obj, "taskClassVersion"); - assertStringField(obj, "subjectId"); - assertStringField(obj, "taskFixtureDigest"); - assertStringField(obj, "verifierManifestDigest"); + const subjectId = assertStringField(obj, "subjectId"); + if (!isSha256Hex(subjectId)) { + throw new FabricTaskError("malformed producer outcome: subjectId", "malformed_producer_outcome", "harness"); + } + const taskFixtureDigest = assertStringField(obj, "taskFixtureDigest"); + const verifierManifestDigest = assertStringField(obj, "verifierManifestDigest"); + if (!isSha256Hex(taskFixtureDigest) || !isSha256Hex(verifierManifestDigest)) { + throw new FabricTaskError("malformed producer outcome: digest field", "malformed_producer_outcome", "harness"); + } assertStringField(obj, "fabricCompatibilityVersion"); - assertStringField(obj, "sandboxProfileDigest"); + const sandboxProfileDigest = assertStringField(obj, "sandboxProfileDigest"); + if (!isSha256Hex(sandboxProfileDigest)) { + throw new FabricTaskError("malformed producer outcome: sandboxProfileDigest", "malformed_producer_outcome", "harness"); + } assertIntegerField(obj, "startedAt"); assertIntegerField(obj, "completedAt"); - assertPlainObject(obj.limits, "limits"); - assertPlainObject(obj.usage, "usage"); - assertPlainObject(obj.verifier, "verifier"); - if (typeof obj.outcome !== "string") { + validateFabricLimits(assertPlainObject(obj.limits, "limits")); + validateFabricUsage(assertPlainObject(obj.usage, "usage")); + validateFabricVerifier(assertPlainObject(obj.verifier, "verifier")); + if (!OUTCOMES.includes(obj.outcome as typeof OUTCOMES[number])) { throw new FabricTaskError("malformed producer outcome: outcome", "malformed_producer_outcome", "harness"); } if (!Array.isArray(obj.artifactDigests)) { throw new FabricTaskError("malformed producer outcome: artifactDigests", "malformed_producer_outcome", "harness"); } + for (const digest of obj.artifactDigests) { + if (typeof digest !== "string" || !isSha256Hex(digest)) { + throw new FabricTaskError("malformed producer outcome: artifactDigests", "malformed_producer_outcome", "harness"); + } + } + if (obj.failure !== undefined) { + validateFailureRecord(assertPlainObject(obj.failure, "failure")); + } + if (obj.sourceRefs !== undefined) { + if (!Array.isArray(obj.sourceRefs) || !obj.sourceRefs.every((row) => typeof row === "string" && row.length > 0)) { + throw new FabricTaskError("malformed producer outcome: sourceRefs", "malformed_producer_outcome", "harness"); + } + } return raw as FabricTaskOutcomeV1; } @@ -188,13 +346,7 @@ export function observationFromFabricOutcome( })); artifacts.push(store.put({ artifactClass: "verifier_summary", - payload: { - scenarioId: FABRIC_SCENARIO_ID, - passed: outcome.verifier.passed, - verifier: outcome.verifier, - usage: outcome.usage, - outcome: outcome.outcome, - }, + payload: sanitizedVerifierSummary(outcome), })); const limits: Record = {}; diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts index 98f176a922..ff456c7063 100644 --- a/src/lab/fabric/scratch.ts +++ b/src/lab/fabric/scratch.ts @@ -2,11 +2,16 @@ import { closeSync, constants as fsConstants, existsSync, + fstatSync, lstatSync, + mkdirSync, openSync, readFileSync, readdirSync, + readSync, + renameSync, rmSync, + unlinkSync, writeSync, type Stats, } from "node:fs"; @@ -17,8 +22,27 @@ import { FABRIC_LIMITS, SYNTHETIC_BEFORE_UTF8, SYNTHETIC_VALUE_PATH } from "./co import { FabricTaskError } from "./types"; const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; +const O_DIRECTORY = (fsConstants as { O_DIRECTORY?: number }).O_DIRECTORY; const FILE_MODE = 0o600; +type OpenSyncWithDir = ( + path: string, + flags: number, + mode: number, + options: { dir: number }, +) => number; + +type ScratchIoMode = "dirfd" | "win32_pinned"; + +interface TrustedScratchDir { + path: string; + fd: number; + identity: string; +} + +let scratchIoMode: ScratchIoMode | null = null; +const trustedScratchRoots = new Map(); + function assertRegularFile(stats: Stats, label: string): void { if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { throw new FabricTaskError(`${label} must be a regular file`, "sandbox_violation", "harness"); @@ -31,6 +55,174 @@ function assertRealDirectory(stats: Stats, label: string): void { } } +function identityOf(stats: Stats): string { + return `${stats.dev}:${stats.ino}`; +} + +function platformSupportsNoFollow(): boolean { + return typeof O_NOFOLLOW === "number" && O_NOFOLLOW !== 0; +} + +function openFlags(base: number, noFollow = true): number { + if (noFollow && platformSupportsNoFollow()) return base | O_NOFOLLOW!; + return base; +} + +function assertScratchName(name: string): void { + if (name.includes("/") || name.includes("\\") || name.includes("..") || name.includes("\0")) { + throw new FabricTaskError("invalid scratch relative name", "sandbox_violation", "harness"); + } +} + +function revalidateScratchDir(dir: TrustedScratchDir): void { + const stats = fstatSync(dir.fd); + assertRealDirectory(stats, "scratch dir"); + if (identityOf(stats) !== dir.identity) { + throw new FabricTaskError("scratch directory identity changed", "sandbox_violation", "harness"); + } +} + +function childScratchPath(dir: TrustedScratchDir, name: string): string { + revalidateScratchDir(dir); + assertScratchName(name); + return join(dir.path, name); +} + +function detectScratchIoMode(dir: TrustedScratchDir): ScratchIoMode { + if (scratchIoMode !== null) return scratchIoMode; + if (process.platform === "win32") { + scratchIoMode = "win32_pinned"; + return scratchIoMode; + } + const probe = `.dirfd-probe-${process.pid}`; + const finalName = `${probe}.ok`; + try { + const openWithDir = openSync as unknown as OpenSyncWithDir; + const fd = openWithDir(probe, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_RDWR, FILE_MODE, { dir: dir.fd }); + closeSync(fd); + renameSync(join(dir.path, probe), join(dir.path, finalName)); + if (!existsSync(join(dir.path, finalName))) { + scratchIoMode = "win32_pinned"; + return scratchIoMode; + } + unlinkSync(join(dir.path, finalName)); + scratchIoMode = "dirfd"; + } catch { + scratchIoMode = "win32_pinned"; + } + return scratchIoMode; +} + +function openAtScratch(dir: TrustedScratchDir, name: string, flags: number, mode = 0, expectDirectory = false): number { + revalidateScratchDir(dir); + assertScratchName(name); + const mode_ = detectScratchIoMode(dir); + if (mode_ === "dirfd") { + return (openSync as unknown as OpenSyncWithDir)(name, flags, mode, { dir: dir.fd }); + } + const fd = openSync(childScratchPath(dir, name), flags, mode); + const opened = fstatSync(fd); + if (expectDirectory) { + assertRealDirectory(opened, name); + } else { + assertRegularFile(opened, name); + const pathEntry = lstatSync(childScratchPath(dir, name)); + if (identityOf(opened) !== identityOf(pathEntry)) { + throw new FabricTaskError("scratch path redirection detected", "sandbox_violation", "harness"); + } + } + return fd; +} + +function openTrustedScratchRoot(scratchRoot: string): TrustedScratchDir { + const abs = resolve(scratchRoot); + let fd: number; + if (typeof O_DIRECTORY === "number") { + fd = openSync(abs, openFlags(fsConstants.O_RDONLY | O_DIRECTORY, true)); + } else { + fd = openSync(abs, fsConstants.O_RDONLY); + } + const stats = fstatSync(fd); + assertRealDirectory(stats, "scratch root"); + return { path: abs, fd, identity: identityOf(stats) }; +} + +function getTrustedScratch(scratchRoot: string): TrustedScratchDir { + const abs = resolve(scratchRoot); + let trusted = trustedScratchRoots.get(abs); + if (!trusted) { + trusted = openTrustedScratchRoot(abs); + trustedScratchRoots.set(abs, trusted); + } + revalidateScratchDir(trusted); + return trusted; +} + +function releaseTrustedScratch(scratchRoot: string): void { + const abs = resolve(scratchRoot); + const trusted = trustedScratchRoots.get(abs); + if (!trusted) return; + trustedScratchRoots.delete(abs); + try { + closeSync(trusted.fd); + } catch { + /* ignore */ + } +} + +function openScratchRelativePath( + root: TrustedScratchDir, + relativePath: string, + flags: number, + mode: number, +): number { + const parts = assertSafeRelativePosixPath(relativePath).split("/"); + let current = root; + const intermediateFds: number[] = []; + try { + for (let i = 0; i < parts.length; i++) { + const isFinal = i === parts.length - 1; + const part = parts[i]!; + if (!isFinal) { + const dirFlags = typeof O_DIRECTORY === "number" + ? openFlags(fsConstants.O_RDONLY | O_DIRECTORY, true) + : openFlags(fsConstants.O_RDONLY, true); + const subFd = openAtScratch(current, part, dirFlags, 0, true); + intermediateFds.push(subFd); + const subStats = fstatSync(subFd); + assertRealDirectory(subStats, part); + current = { path: join(current.path, part), fd: subFd, identity: identityOf(subStats) }; + continue; + } + return openAtScratch(current, part, flags, mode); + } + throw new FabricTaskError("empty scratch path", "sandbox_violation", "harness"); + } catch (error) { + for (const fd of intermediateFds) { + try { + closeSync(fd); + } catch { + /* ignore */ + } + } + throw error; + } +} + +function readAllFromFd(fd: number, size: number): string { + const buf = Buffer.alloc(size); + let offset = 0; + while (offset < buf.length) { + const n = readSync(fd, buf, offset, buf.length - offset, offset); + if (n <= 0) break; + offset += n; + } + if (offset !== size) { + throw new FabricTaskError("short read from scratch file", "sandbox_violation", "harness"); + } + return buf.toString("utf8"); +} + /** Normalize and reject traversal / absolute / Windows drive / NUL paths. */ export function assertSafeRelativePosixPath(raw: string): string { if (typeof raw !== "string" || raw.length === 0) { @@ -102,12 +294,18 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { const root = join(base, `fabric-${Date.now().toString(36)}-${randomBytes(6).toString("hex")}`); ensureRestrictedDir(root); ensureRestrictedDir(join(root, dirname(SYNTHETIC_VALUE_PATH))); - const valuePath = resolveInsideScratch(root, SYNTHETIC_VALUE_PATH, { allowMissingFinal: true }); + const trusted = openTrustedScratchRoot(root); + trustedScratchRoots.set(trusted.path, trusted); const bytes = Buffer.from(SYNTHETIC_BEFORE_UTF8, "utf8"); if (bytes.byteLength > FABRIC_LIMITS.maxAggregateIoBytes) { throw new FabricTaskError("fixture exceeds io budget", "budget_exhausted", "harness"); } - const fd = openSync(valuePath, openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL), FILE_MODE); + const fd = openScratchRelativePath( + trusted, + SYNTHETIC_VALUE_PATH, + openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, true), + FILE_MODE, + ); try { writeSync(fd, bytes); } finally { @@ -116,6 +314,7 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { return { root, cleanup: () => { + releaseTrustedScratch(root); try { rmSync(root, { recursive: true, force: true, maxRetries: 3 }); } catch { @@ -125,11 +324,6 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { }; } -function openFlags(base: number): number { - if (typeof O_NOFOLLOW === "number" && O_NOFOLLOW !== 0) return base | O_NOFOLLOW; - return base; -} - export interface WalkedFile { relativePosix: string; absolute: string; @@ -183,38 +377,48 @@ export function walkScratchFiles(scratchRoot: string): WalkedFile[] { } export function readScratchFileUtf8(scratchRoot: string, relativePath: string, maxBytes: number): string { - const absolute = resolveInsideScratch(scratchRoot, relativePath); - const stats = lstatSync(absolute); - if (stats.isSymbolicLink()) { - throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); - } - assertRegularFile(stats, relativePath); - if (stats.size > maxBytes) { - throw new FabricTaskError("file exceeds io budget", "budget_exhausted", "environment"); + resolveInsideScratch(scratchRoot, relativePath); + const trusted = getTrustedScratch(scratchRoot); + const fd = openScratchRelativePath(trusted, relativePath, openFlags(fsConstants.O_RDONLY, true), 0); + try { + const stats = fstatSync(fd); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + assertRegularFile(stats, relativePath); + if (stats.size > maxBytes) { + throw new FabricTaskError("file exceeds io budget", "budget_exhausted", "environment"); + } + return readAllFromFd(fd, stats.size); + } finally { + closeSync(fd); } - return readFileSync(absolute, "utf8"); } export function writeScratchFileUtf8(scratchRoot: string, relativePath: string, contentUtf8: string, maxBytes: number): number { - const absolute = resolveInsideScratch(scratchRoot, relativePath, { allowMissingFinal: true }); const bytes = Buffer.from(contentUtf8, "utf8"); if (bytes.byteLength > maxBytes) { throw new FabricTaskError("write exceeds io budget", "budget_exhausted", "environment"); } - const parent = dirname(absolute); - const parentStats = lstatSync(parent); - assertRealDirectory(parentStats, parent); - if (existsSync(absolute)) { - const stats = lstatSync(absolute); + const safe = assertSafeRelativePosixPath(relativePath); + const parentPath = safe.includes("/") ? safe.slice(0, safe.lastIndexOf("/")) : ""; + if (parentPath) { + ensureRestrictedDir(join(scratchRoot, parentPath.split("/").join(sep))); + } + resolveInsideScratch(scratchRoot, relativePath, { allowMissingFinal: true }); + const trusted = getTrustedScratch(scratchRoot); + const fd = openScratchRelativePath( + trusted, + relativePath, + openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC, true), + FILE_MODE, + ); + try { + const stats = fstatSync(fd); if (stats.isSymbolicLink()) { throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); } assertRegularFile(stats, relativePath); - } else { - ensureRestrictedDir(parent); - } - const fd = openSync(absolute, openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC), FILE_MODE); - try { writeSync(fd, bytes); } finally { closeSync(fd); diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 3ff5af4a80..a6af886d97 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -5,6 +5,7 @@ import { fsyncSync, mkdirSync, openSync, + readFileSync, readSync, statSync, unlinkSync, @@ -24,6 +25,13 @@ export interface LedgerStore { } const eventIdIndexByLedger = new Map>(); +const LEDGER_LOCK_STALE_MS = 60_000; +const LEDGER_LOCK_WAIT_MS = 5_000; + +interface LedgerLockMeta { + pid: number; + createdAt: number; +} function sleepSyncMs(ms: number): void { const end = Date.now() + ms; @@ -32,14 +40,48 @@ function sleepSyncMs(ms: number): void { } } +function readLedgerLockMeta(lockPath: string): LedgerLockMeta | null { + try { + const parsed = JSON.parse(readFileSync(lockPath, "utf8")) as LedgerLockMeta; + if (typeof parsed.pid === "number" && typeof parsed.createdAt === "number") return parsed; + } catch { + /* ignore */ + } + return null; +} + +function isLedgerLockStale(lockPath: string): boolean { + const meta = readLedgerLockMeta(lockPath); + if (!meta) return true; + return Date.now() - meta.createdAt > LEDGER_LOCK_STALE_MS; +} + +function tryAcquireLedgerLock(lockPath: string): number { + try { + const fd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); + writeSync(fd, Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now() }), "utf8")); + return fd; + } catch (error) { + if (existsSync(lockPath) && isLedgerLockStale(lockPath)) { + try { + unlinkSync(lockPath); + } catch { + /* ignore */ + } + return tryAcquireLedgerLock(lockPath); + } + throw error; + } +} + function withLedgerLock(ledgerPath: string, fn: () => T): T { const lockPath = `${ledgerPath}.lock`; mkdirSync(dirname(ledgerPath), { recursive: true, mode: 0o700 }); - const deadline = Date.now() + 5_000; + const deadline = Date.now() + LEDGER_LOCK_WAIT_MS; let lockFd: number | undefined; while (lockFd === undefined) { try { - lockFd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); + lockFd = tryAcquireLedgerLock(lockPath); } catch (error) { if (Date.now() >= deadline) throw error; sleepSyncMs(10); diff --git a/src/lab/live/transport.ts b/src/lab/live/transport.ts index 90b0b8bea2..f1dac8e211 100644 --- a/src/lab/live/transport.ts +++ b/src/lab/live/transport.ts @@ -104,8 +104,10 @@ export function classifyTransportError(error: unknown): { classification: Failur case "network_blocked": return { classification: "network_failure", secondaryCode: code }; case "region_blocked": return { classification: "region_blocked", secondaryCode: code }; case "provider_transient": return { classification: "provider_transient", secondaryCode: code }; - case "connect_timeout": case "first_byte_timeout": case "inactivity_timeout": case "total_timeout": + case "connect_timeout": case "first_byte_timeout": case "total_timeout": return { classification: "timeout", secondaryCode: code }; + case "inactivity_timeout": + return { classification: "inactivity_timeout", secondaryCode: code }; case "request_limit": case "input_byte_limit": case "output_byte_limit": case "output_token_limit": case "tool_call_limit": case "artifact_byte_limit": case "memory_limit": case "child_process_limit": return { classification: "budget_exhausted", secondaryCode: code }; diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts index a8c3e8bb41..857874ed49 100644 --- a/src/lab/observe/from-live.ts +++ b/src/lab/observe/from-live.ts @@ -49,7 +49,7 @@ function failureFromLiveResult(result: LiveScenarioRunResult, retryable: boolean if (result.passed || result.classification === "inconclusive") return undefined; const attribution: FailureRecordV1["attribution"] = result.classification === "harness_failure" ? "harness" - : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) + : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "inactivity_timeout", "budget_exhausted"].includes(result.classification) ? "environment" : "route"; return { diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 390308658b..565a6a3b11 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -1,4 +1,4 @@ -import { chmodSync, mkdirSync, lstatSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, lstatSync } from "node:fs"; import { join } from "node:path"; import { getConfigDir } from "../config"; @@ -6,7 +6,15 @@ import { getConfigDir } from "../config"; export function ensureRestrictedDir(dir: string): void { mkdirSync(dir, { recursive: true, mode: 0o700 }); if (process.platform === "win32") return; - const mode = lstatSync(dir).mode & 0o777; + if (!existsSync(dir)) return; + const stats = lstatSync(dir); + if (stats.isSymbolicLink()) { + throw new Error(`restricted directory is a symbolic link: ${dir}`); + } + if (!stats.isDirectory()) { + throw new Error(`restricted path is not a directory: ${dir}`); + } + const mode = stats.mode & 0o777; if (mode !== 0o700) chmodSync(dir, 0o700); } diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index 8efb68dd8d..1872a93797 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -83,7 +83,7 @@ export function taskSubjectApplicableToRequirements( routePreconditions: readonly string[]; }, ): boolean { - const platforms = requirements.platforms ?? []; + const platforms = requirements.platforms ?? ["*"]; const features = requirements.requiredHarnessFeatures ?? []; const preconditions = requirements.routePreconditions ?? []; const platformOk = platforms.includes("*") || platforms.some((platform) => capability.platforms.includes(platform)); diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index e406b4e4ae..6d0b80d4a6 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -3,6 +3,7 @@ import { existsSync, mkdirSync, readFileSync, + readdirSync, rmSync, symlinkSync, writeFileSync, @@ -44,6 +45,8 @@ import { FabricTaskError } from "../src/lab/fabric/types"; import { writeScratchFileUtf8, readScratchFileUtf8 } from "../src/lab/fabric/scratch"; import { routingProfileIssues } from "../src/routing/profile"; import { buildInvalidationIndex } from "../src/lab/ledger/invalidation"; +import { createArtifactStore } from "../src/lab/artifacts/store"; +import { ensureLabDirs } from "../src/lab/paths"; import { verifyExactTreeDiffV1 } from "../src/lab/fabric/verifier"; import { parseSyntheticPatchV1 } from "../src/lab/fabric/patch"; import { FABRIC_LIMITS } from "../src/lab/fabric/constants"; @@ -376,6 +379,99 @@ describe("CL-07 task effectiveness producer", () => { platforms: ["win32"], routePreconditions: ["exact-route-subject"], })).toBe(true); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: ["fabric-scratch-v1"], + routePreconditions: ["exact-route-subject"], + }, { + harnessFeatures: ["fabric-scratch-v1"], + platforms: ["win32"], + routePreconditions: ["exact-route-subject"], + })).toBe(true); + }); + + test("malformed nested outcome fields throw FabricTaskError", async () => { + const home = tempHome(); + const base = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + }); + expect(() => observationFromFabricOutcome({ ...base, verifier: {} })).toThrow(); + expect(() => observationFromFabricOutcome({ + ...base, + usage: { ...base.usage, inputBytes: -1 }, + })).toThrow(); + expect(() => observationFromFabricOutcome({ + ...base, + taskSubject: { ...base.taskSubject, subjectKind: "route" }, + })).toThrow(); + }); + + test("verifier summary artifact omits raw credential diagnostics", async () => { + const home = tempHome(); + const paths = ensureLabDirs(home); + const store = createArtifactStore(paths.artifactsDir); + const base = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + }); + const secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890"; + try { + const { artifacts } = observationFromFabricOutcome({ + ...base, + verifier: { + ...base.verifier, + reason: `failed ${secret}`, + }, + }, { configDir: home, artifactStore: store }); + const summaryRef = artifacts.find((row) => row.artifactClass === "verifier_summary"); + expect(summaryRef).toBeDefined(); + const text = new TextDecoder().decode(store.get(summaryRef!.digest)); + expect(text.includes(secret)).toBe(false); + persistFabricOutcome({ + ...base, + verifier: { + ...base.verifier, + reason: `failed ${secret}`, + }, + }, { configDir: home }); + const ledger = readFileSync(join(home, "lab", "compatibility.jsonl"), "utf8"); + expect(ledger.includes(secret)).toBe(false); + } finally { + store.close(); + } + }); + + test("expired ledger lock is recovered before append", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); + const lockPath = join(home, "lab", "compatibility.jsonl.lock"); + writeFileSync(lockPath, JSON.stringify({ pid: 99999, createdAt: Date.now() - 120_000 }), { mode: 0o600 }); + const outcome = await runFabricSyntheticPatchTask({ + routeSubject: routeSubject(), + configDir: home, + producePatch: () => correctSyntheticPatch(), + }); + persistFabricOutcome(outcome, { configDir: home }); + expect(existsSync(join(home, "lab", "compatibility.jsonl"))).toBe(true); + }); + + test("symlinked lab scratch dir is rejected", () => { + if (process.platform === "win32") return; + const home = tempHome(); + const outside = join(home, "outside-scratch"); + mkdirSync(outside, { recursive: true, mode: 0o700 }); + mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); + const scratchLink = join(home, "lab", "scratch"); + try { + symlinkSync(outside, scratchLink); + } catch { + return; + } + expect(() => createSyntheticScratch(home)).toThrow(); + expect(readdirSync(outside).some((name) => name.startsWith("fabric-"))).toBe(false); }); test("duplicate outcome delivery is idempotent; distinct attempts remain distinct", async () => { diff --git a/tests/lab-live-probe.test.ts b/tests/lab-live-probe.test.ts index 95d40e7f0c..d869f227ea 100644 --- a/tests/lab-live-probe.test.ts +++ b/tests/lab-live-probe.test.ts @@ -12,6 +12,7 @@ import { import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; import { replayLabLedger } from "../src/lab/ledger/store"; import { clearMcpStub } from "../src/lab/live/mcp-loopback"; +import { TransportError } from "../src/lab/live/transport"; import { expandLiveSuiteManifest } from "../src/lab/live/suite-manifest"; import type { LabBehaviorValues, LabRouteContext, LiveScenarioRunResult } from "../src/lab/live/types"; import type { ProtocolSubjectV1, RouteSubjectV1 } from "../src/lab/events/types"; @@ -182,4 +183,22 @@ describe("CL-03 live probe harness", () => { const result = await runLiveScenario(scenario, mockRoute({ requiredClaims: ["reasoning"] }), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(reasoningObservation()) }); expect(result.passed).toBe(true); const { event } = observationFromLiveResult(result, scenario, authority, { configDir: home }); expect(JSON.stringify(event)).not.toContain("PLAN"); }); + + test("inactivity_timeout failures attribute environment on observation", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases[0]!; + const result = await runLiveScenario(caseRecord, mockRoute(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + routeExecutor: createHostIssuedLabRouteExecutor(async () => { + throw new TransportError("inactivity_timeout", "inactivity timeout exceeded"); + }), + }); + expect(result.classification).toBe("inactivity_timeout"); + const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); + expect(event.failure?.class).toBe("inactivity_timeout"); + expect(event.failure?.attribution).toBe("environment"); + }); }); From bc00433e565bf94956d9050122b2fd44a23bc5e9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:09:36 +0200 Subject: [PATCH 06/24] docs(lab): add JSDoc to CL-07 fabric module for CodeRabbit coverage Document exported and internal fabric helpers, scratch IO, observation validation, and ledger lock recovery. --- src/lab/fabric/executor.ts | 5 +++++ src/lab/fabric/index.ts | 1 + src/lab/fabric/manifest.ts | 7 +++++++ src/lab/fabric/observe.ts | 18 ++++++++++++++++++ src/lab/fabric/patch.ts | 2 ++ src/lab/fabric/scratch.ts | 23 +++++++++++++++++++++++ src/lab/fabric/subject.ts | 7 +++++++ src/lab/fabric/types.ts | 1 + src/lab/fabric/verifier.ts | 1 + src/lab/ledger/store.ts | 6 ++++++ src/lab/projection/verification.ts | 2 ++ 11 files changed, 73 insertions(+) diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts index 1b46c4dccb..5e10fb436c 100644 --- a/src/lab/fabric/executor.ts +++ b/src/lab/fabric/executor.ts @@ -29,6 +29,7 @@ import type { import { FabricTaskError } from "./types"; import { verifyExactTreeDiffV1 } from "./verifier"; +/** Options for running the bounded synthetic-patch fabric task executor. */ export interface RunFabricTaskOptions { routeSubject: RouteSubjectV1; producePatch: SyntheticPatchProducer; @@ -41,6 +42,7 @@ export interface RunFabricTaskOptions { sleep?: (ms: number) => Promise; } +/** Map a FabricTaskError into a ledger failure record. */ function failureFromError(error: FabricTaskError): FailureRecordV1 { return { class: error.code, @@ -50,6 +52,7 @@ function failureFromError(error: FabricTaskError): FailureRecordV1 { }; } +/** Map a failure record to the observation outcome kind for persistence. */ function outcomeFromFailure(failure: FailureRecordV1): FabricTaskOutcomeV1["outcome"] { if (failure.class === "behavioral_failure") return "fail"; if (failure.attribution === "harness") return "inconclusive"; @@ -253,6 +256,7 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) } } +/** Assemble an immutable FabricTaskOutcomeV1 from executor state. */ function sealOutcome(input: { taskSubject: FabricTaskOutcomeV1["taskSubject"]; subjectId: string; @@ -294,6 +298,7 @@ function sealOutcome(input: { return sealed; } +/** Arm total and inactivity deadlines around producer execution. */ function createTimeoutController( totalMs: number, inactivityMs: number, diff --git a/src/lab/fabric/index.ts b/src/lab/fabric/index.ts index 1e8f32470e..8a51e13235 100644 --- a/src/lab/fabric/index.ts +++ b/src/lab/fabric/index.ts @@ -1,3 +1,4 @@ +/** CL-07 fabric module public surface (explicit allowlist re-exports). */ export { FABRIC_SUITE_ID, FABRIC_SUITE_VERSION, diff --git a/src/lab/fabric/manifest.ts b/src/lab/fabric/manifest.ts index c0c8be1ee2..9e0a1ef806 100644 --- a/src/lab/fabric/manifest.ts +++ b/src/lab/fabric/manifest.ts @@ -14,6 +14,7 @@ import { } from "./constants"; import { verifierManifestDigest } from "./subject"; +/** In-memory authority for the frozen CL-07 fabric scenario catalogue. */ export interface FabricCaseAuthority { schemaVersion: number; sourceCommit: string; @@ -38,6 +39,7 @@ const FIXTURE_BYTES_UTF8 = JSON.stringify({ after: SYNTHETIC_AFTER_UTF8, }); +/** Frozen CL-07 fabric scenario authority loaded from compiled constants. */ export function loadFabricCaseAuthority(): FabricCaseAuthority { const fixtureBytes = new TextEncoder().encode(FIXTURE_BYTES_UTF8); const digest = fixtureDigest(fixtureBytes); @@ -101,6 +103,7 @@ export function loadFabricCaseAuthority(): FabricCaseAuthority { }; } +/** List fabric scenarios, optionally filtered by suite id. */ export function discoverFabricScenarios( authority: FabricCaseAuthority = loadFabricCaseAuthority(), suites?: readonly string[], @@ -108,6 +111,7 @@ export function discoverFabricScenarios( return authority.cases.filter((row) => !suites || suites.includes(row.suite)); } +/** Expand a fabric case into the canonical scenario manifest object. */ export function expandFabricScenario( caseRecord: CaseRecord, authority: FabricCaseAuthority = loadFabricCaseAuthority(), @@ -173,6 +177,7 @@ export function expandFabricScenario( }; } +/** Expand a fabric suite id into the canonical suite manifest object. */ export function expandFabricSuiteManifest( suiteId: string = FABRIC_SUITE_ID, authority: FabricCaseAuthority = loadFabricCaseAuthority(), @@ -200,6 +205,7 @@ export function expandFabricSuiteManifest( }; } +/** Digest of the expanded fabric scenario manifest for a case. */ export function fabricScenarioManifestDigest( caseRecord: CaseRecord = loadFabricCaseAuthority().cases[0]!, authority: FabricCaseAuthority = loadFabricCaseAuthority(), @@ -207,6 +213,7 @@ export function fabricScenarioManifestDigest( return scenarioManifestDigest(expandFabricScenario(caseRecord, authority)); } +/** Digest of the expanded fabric suite manifest. */ export function fabricSuiteManifestDigest( suiteId: string = FABRIC_SUITE_ID, authority: FabricCaseAuthority = loadFabricCaseAuthority(), diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 61aeb69653..5ce1fdd66e 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -33,6 +33,9 @@ import { import type { FabricLimitsV1, FabricTaskOutcomeV1 } from "./types"; import { FabricTaskError } from "./types"; +/** CL-07 fabric task outcome validation and observation persistence. */ + +/** Options for building or persisting a fabric observation event. */ export interface PersistFabricOptions { configDir?: string; recordedAt?: number; @@ -41,6 +44,7 @@ export interface PersistFabricOptions { attempt?: number; } +/** Result of appending a fabric observation to the compatibility ledger. */ export interface PersistedFabricObservation { event: ObservationEvent; ledgerPath: string; @@ -76,6 +80,7 @@ const LIMIT_KEYS = Object.keys(FABRIC_LIMITS) as Array; const FAILURE_KEYS = new Set(["class", "code", "retryable", "attribution"]); const FAILURE_ATTRIBUTIONS = new Set(["opencodex", "route", "environment", "harness"]); +/** Require a non-null plain object or throw a harness-class FabricTaskError. */ function assertPlainObject(value: unknown, label: string): Record { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new FabricTaskError(`malformed producer outcome: ${label}`, "malformed_producer_outcome", "harness"); @@ -83,6 +88,7 @@ function assertPlainObject(value: unknown, label: string): Record; } +/** Require a non-empty string field on a producer outcome object. */ function assertStringField(obj: Record, key: string): string { const value = obj[key]; if (typeof value !== "string" || value.length === 0) { @@ -91,6 +97,7 @@ function assertStringField(obj: Record, key: string): string { return value; } +/** Require an integer field on a producer outcome object. */ function assertIntegerField(obj: Record, key: string): number { const value = obj[key]; if (typeof value !== "number" || !Number.isInteger(value)) { @@ -99,6 +106,7 @@ function assertIntegerField(obj: Record, key: string): number { return value; } +/** Require a non-negative integer field on a producer outcome object. */ function assertNonNegativeIntegerField(obj: Record, key: string): number { const value = assertIntegerField(obj, key); if (value < 0) { @@ -107,6 +115,7 @@ function assertNonNegativeIntegerField(obj: Record, key: string return value; } +/** Map LabValidationError into FabricTaskError for producer outcome parsing. */ function wrapValidationError(error: unknown): void { if (error instanceof LabValidationError) { throw new FabricTaskError(error.message, "malformed_producer_outcome", "harness"); @@ -114,6 +123,7 @@ function wrapValidationError(error: unknown): void { throw error; } +/** Validate verifier payload shape and reject unknown nested verifier fields. */ function validateFabricVerifier(raw: Record): void { for (const key of Object.keys(raw)) { if (!VERIFIER_KEYS.has(key)) { @@ -160,18 +170,21 @@ function validateFabricVerifier(raw: Record): void { } } +/** Validate non-negative usage counters on a producer outcome. */ function validateFabricUsage(raw: Record): void { for (const key of USAGE_KEYS) { assertNonNegativeIntegerField(raw, key); } } +/** Validate non-negative limit fields on a producer outcome. */ function validateFabricLimits(raw: Record): void { for (const key of LIMIT_KEYS) { assertNonNegativeIntegerField(raw, key); } } +/** Validate failure record shape and known attribution values. */ function validateFailureRecord(raw: Record): void { for (const key of Object.keys(raw)) { if (!FAILURE_KEYS.has(key)) { @@ -192,10 +205,12 @@ function validateFailureRecord(raw: Record): void { } } +/** Compare top-level and nested route subjects by canonical JSON equality. */ function routeSubjectsMatch(top: RouteSubjectV1, nested: RouteSubjectV1): boolean { return JSON.stringify(top) === JSON.stringify(nested); } +/** Build an allowlisted verifier_summary artifact with sanitized diagnostics. */ function sanitizedVerifierSummary(outcome: FabricTaskOutcomeV1): Record { const verifier = { verifierId: outcome.verifier.verifierId, @@ -221,6 +236,7 @@ function sanitizedVerifierSummary(outcome: FabricTaskOutcomeV1): Record FABRIC_LIMITS.maxPatchOperations) { throw new FabricTaskError("too many patch operations", "budget_exhausted", "environment"); diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts index ff456c7063..0890b478b9 100644 --- a/src/lab/fabric/scratch.ts +++ b/src/lab/fabric/scratch.ts @@ -21,6 +21,8 @@ import { ensureLabDirs, ensureRestrictedDir, labScratchDir } from "../paths"; import { FABRIC_LIMITS, SYNTHETIC_BEFORE_UTF8, SYNTHETIC_VALUE_PATH } from "./constants"; import { FabricTaskError } from "./types"; +/** CL-07 restricted scratch tree IO for the synthetic-patch fabric task. */ + const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; const O_DIRECTORY = (fsConstants as { O_DIRECTORY?: number }).O_DIRECTORY; const FILE_MODE = 0o600; @@ -43,37 +45,44 @@ interface TrustedScratchDir { let scratchIoMode: ScratchIoMode | null = null; const trustedScratchRoots = new Map(); +/** Require stats to describe a regular file, not a symlink or special node. */ function assertRegularFile(stats: Stats, label: string): void { if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { throw new FabricTaskError(`${label} must be a regular file`, "sandbox_violation", "harness"); } } +/** Require stats to describe a real directory, not a symlink. */ function assertRealDirectory(stats: Stats, label: string): void { if (stats.isSymbolicLink() || !stats.isDirectory()) { throw new FabricTaskError(`${label} must be a real directory`, "sandbox_violation", "harness"); } } +/** Stable device/inode identity string for a filesystem node. */ function identityOf(stats: Stats): string { return `${stats.dev}:${stats.ino}`; } +/** Whether the host supports O_NOFOLLOW for scratch open operations. */ function platformSupportsNoFollow(): boolean { return typeof O_NOFOLLOW === "number" && O_NOFOLLOW !== 0; } +/** Compose open flags, optionally adding O_NOFOLLOW when supported. */ function openFlags(base: number, noFollow = true): number { if (noFollow && platformSupportsNoFollow()) return base | O_NOFOLLOW!; return base; } +/** Reject scratch relative names that traverse or embed forbidden characters. */ function assertScratchName(name: string): void { if (name.includes("/") || name.includes("\\") || name.includes("..") || name.includes("\0")) { throw new FabricTaskError("invalid scratch relative name", "sandbox_violation", "harness"); } } +/** Re-open scratch dir fd and confirm its identity has not changed. */ function revalidateScratchDir(dir: TrustedScratchDir): void { const stats = fstatSync(dir.fd); assertRealDirectory(stats, "scratch dir"); @@ -82,12 +91,14 @@ function revalidateScratchDir(dir: TrustedScratchDir): void { } } +/** Resolve a single path segment under a trusted scratch directory. */ function childScratchPath(dir: TrustedScratchDir, name: string): string { revalidateScratchDir(dir); assertScratchName(name); return join(dir.path, name); } +/** Detect whether scratch IO uses dirfd openat or win32 pinned path mode. */ function detectScratchIoMode(dir: TrustedScratchDir): ScratchIoMode { if (scratchIoMode !== null) return scratchIoMode; if (process.platform === "win32") { @@ -113,6 +124,7 @@ function detectScratchIoMode(dir: TrustedScratchDir): ScratchIoMode { return scratchIoMode; } +/** Open a single name relative to a trusted scratch directory fd. */ function openAtScratch(dir: TrustedScratchDir, name: string, flags: number, mode = 0, expectDirectory = false): number { revalidateScratchDir(dir); assertScratchName(name); @@ -134,6 +146,7 @@ function openAtScratch(dir: TrustedScratchDir, name: string, flags: number, mode return fd; } +/** Pin an absolute scratch root path into a trusted directory handle. */ function openTrustedScratchRoot(scratchRoot: string): TrustedScratchDir { const abs = resolve(scratchRoot); let fd: number; @@ -147,6 +160,7 @@ function openTrustedScratchRoot(scratchRoot: string): TrustedScratchDir { return { path: abs, fd, identity: identityOf(stats) }; } +/** Return a cached trusted scratch handle, opening the root on first use. */ function getTrustedScratch(scratchRoot: string): TrustedScratchDir { const abs = resolve(scratchRoot); let trusted = trustedScratchRoots.get(abs); @@ -158,6 +172,7 @@ function getTrustedScratch(scratchRoot: string): TrustedScratchDir { return trusted; } +/** Close and drop a cached trusted scratch directory handle. */ function releaseTrustedScratch(scratchRoot: string): void { const abs = resolve(scratchRoot); const trusted = trustedScratchRoots.get(abs); @@ -170,6 +185,7 @@ function releaseTrustedScratch(scratchRoot: string): void { } } +/** Open a relative scratch path by walking trusted directory handles. */ function openScratchRelativePath( root: TrustedScratchDir, relativePath: string, @@ -209,6 +225,7 @@ function openScratchRelativePath( } } +/** Read exactly size bytes from an open scratch file descriptor. */ function readAllFromFd(fd: number, size: number): string { const buf = Buffer.alloc(size); let offset = 0; @@ -244,6 +261,7 @@ export function assertSafeRelativePosixPath(raw: string): string { return normalized; } +/** Prove a resolved path remains under the scratch root prefix. */ function assertUnderScratchRoot(root: string, target: string): void { if (target !== root && !target.startsWith(root + sep)) { throw new FabricTaskError("path escapes scratch root", "sandbox_violation", "harness"); @@ -282,11 +300,13 @@ export function resolveInsideScratch(scratchRoot: string, relativePath: string, return current; } +/** Handle for a fabric scratch tree and its best-effort cleanup callback. */ export interface ScratchTree { root: string; cleanup: () => void; } +/** Create an isolated scratch tree with the frozen synthetic-patch fixture file. */ export function createSyntheticScratch(configDir?: string): ScratchTree { ensureLabDirs(configDir); const base = labScratchDir(configDir); @@ -324,6 +344,7 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { }; } +/** One regular file discovered during a bounded scratch tree walk. */ export interface WalkedFile { relativePosix: string; absolute: string; @@ -376,6 +397,7 @@ export function walkScratchFiles(scratchRoot: string): WalkedFile[] { return out; } +/** Read a scratch file as UTF-8 text under a byte budget. */ export function readScratchFileUtf8(scratchRoot: string, relativePath: string, maxBytes: number): string { resolveInsideScratch(scratchRoot, relativePath); const trusted = getTrustedScratch(scratchRoot); @@ -395,6 +417,7 @@ export function readScratchFileUtf8(scratchRoot: string, relativePath: string, m } } +/** Write UTF-8 content to a scratch file, replacing any existing bytes. */ export function writeScratchFileUtf8(scratchRoot: string, relativePath: string, contentUtf8: string, maxBytes: number): number { const bytes = Buffer.from(contentUtf8, "utf8"); if (bytes.byteLength > maxBytes) { diff --git a/src/lab/fabric/subject.ts b/src/lab/fabric/subject.ts index 1fc1fb7671..d4b2657c5b 100644 --- a/src/lab/fabric/subject.ts +++ b/src/lab/fabric/subject.ts @@ -16,10 +16,12 @@ const SANDBOX_DOMAIN = "ocx-lab:fabric-sandbox-profile:v1"; const VERIFIER_DOMAIN = "ocx-lab:fabric-verifier-manifest:v1"; const FIXTURE_DOMAIN = "ocx-lab:fabric-task-fixture:v1"; +/** Domain hash of the frozen fabric sandbox profile object. */ export function sandboxProfileDigest(profile: unknown = SANDBOX_PROFILE_V1): string { return domainHash(SANDBOX_DOMAIN, jcsStringify(profile)); } +/** Canonical exact-tree-diff verifier manifest object for CL-07. */ export function verifierManifestObject(): Record { return { schemaVersion: 1, @@ -39,10 +41,12 @@ export function verifierManifestObject(): Record { }; } +/** Domain hash of the fabric verifier manifest object. */ export function verifierManifestDigest(manifest: unknown = verifierManifestObject()): string { return domainHash(VERIFIER_DOMAIN, jcsStringify(manifest)); } +/** Canonical synthetic-patch task fixture object for CL-07. */ export function taskFixtureObject(): Record { return { schemaVersion: 1, @@ -63,10 +67,12 @@ export function taskFixtureObject(): Record { }; } +/** Domain hash of the fabric task fixture object. */ export function taskFixtureDigest(fixture: unknown = taskFixtureObject()): string { return domainHash(FIXTURE_DOMAIN, jcsStringify(fixture)); } +/** Build a frozen TaskSubjectV1 for the synthetic-patch fabric task. */ export function buildTaskSubjectV1(input: { routeSubject: RouteSubjectV1; taskClassId?: string; @@ -94,6 +100,7 @@ export function buildTaskSubjectV1(input: { }); } +/** Stable subject id for a fabric TaskSubjectV1. */ export function taskSubjectId(subject: TaskSubjectV1): string { return subjectIdForSubject(subject); } diff --git a/src/lab/fabric/types.ts b/src/lab/fabric/types.ts index 004f227760..48bfec1116 100644 --- a/src/lab/fabric/types.ts +++ b/src/lab/fabric/types.ts @@ -84,6 +84,7 @@ export type SyntheticPatchProducer = (ctx: { reportActivity: () => void; }) => SyntheticPatchV1 | Promise; +/** Typed error for fabric task producer, verifier, and sandbox failures. */ export class FabricTaskError extends Error { override readonly name = "FabricTaskError"; constructor( diff --git a/src/lab/fabric/verifier.ts b/src/lab/fabric/verifier.ts index 6b5bddb3b7..de7a993e15 100644 --- a/src/lab/fabric/verifier.ts +++ b/src/lab/fabric/verifier.ts @@ -11,6 +11,7 @@ import { verifierManifestDigest } from "./subject"; import type { ExactTreeDiffResultV1 } from "./types"; import { FabricTaskError } from "./types"; +/** SHA-256 hex digest of a UTF-8 string for verifier tree comparisons. */ function sha256Utf8(value: string): string { return createHash("sha256").update(value, "utf8").digest("hex"); } diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index a6af886d97..ac4eb82c55 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -33,6 +33,7 @@ interface LedgerLockMeta { createdAt: number; } +/** Block synchronously for the given duration (ledger lock retry only). */ function sleepSyncMs(ms: number): void { const end = Date.now() + ms; while (Date.now() < end) { @@ -40,6 +41,7 @@ function sleepSyncMs(ms: number): void { } } +/** Read pid and createdAt metadata from a ledger lock file, if well-formed. */ function readLedgerLockMeta(lockPath: string): LedgerLockMeta | null { try { const parsed = JSON.parse(readFileSync(lockPath, "utf8")) as LedgerLockMeta; @@ -50,12 +52,14 @@ function readLedgerLockMeta(lockPath: string): LedgerLockMeta | null { return null; } +/** Return true when a ledger lock file is missing metadata or older than the stale window. */ function isLedgerLockStale(lockPath: string): boolean { const meta = readLedgerLockMeta(lockPath); if (!meta) return true; return Date.now() - meta.createdAt > LEDGER_LOCK_STALE_MS; } +/** Create a ledger lock file exclusively, recovering stale locks when needed. */ function tryAcquireLedgerLock(lockPath: string): number { try { const fd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); @@ -74,6 +78,7 @@ function tryAcquireLedgerLock(lockPath: string): number { } } +/** Run a ledger mutation while holding the compatibility ledger lock file. */ function withLedgerLock(ledgerPath: string, fn: () => T): T { const lockPath = `${ledgerPath}.lock`; mkdirSync(dirname(ledgerPath), { recursive: true, mode: 0o700 }); @@ -99,6 +104,7 @@ function withLedgerLock(ledgerPath: string, fn: () => T): T { } } +/** Load or build the in-memory event-id index for a ledger path. */ function loadEventIdIndex(ledgerPath: string): Set { let index = eventIdIndexByLedger.get(ledgerPath); if (!index) { diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index 1872a93797..d93e2c7de3 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -41,6 +41,7 @@ export function isScenarioApplicable( return true; } +/** Match protocol subject fields against scenario requirement lists. */ function scenarioApplicableToRequirements( requirements: ScenarioRequirements, subject: ProtocolSubjectV1, @@ -92,6 +93,7 @@ export function taskSubjectApplicableToRequirements( return platformOk && featuresOk && preconditionsOk; } +/** Return true for non-negative integer values used in freshness parsing. */ function isNonNegativeInteger(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0; } From 1689466ac8530f2d073dd517cc90ddd5e4927a1d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:14:15 +0200 Subject: [PATCH 07/24] fix(lab): address third CodeRabbit round for CL-07 fabric producer Harden scratch IO and restricted dirs, improve ledger lock ownership, treat capability platform wildcard as universal, and strengthen regression tests. --- src/lab/fabric/observe.ts | 2 +- src/lab/fabric/scratch.ts | 220 +++++++++++++---------------- src/lab/ledger/store.ts | 92 ++++++++---- src/lab/paths.ts | 38 +++-- src/lab/projection/verification.ts | 6 +- tests/lab-fabric-task.test.ts | 57 ++++++-- 6 files changed, 242 insertions(+), 173 deletions(-) diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 5ce1fdd66e..2f078b219c 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -116,7 +116,7 @@ function assertNonNegativeIntegerField(obj: Record, key: string } /** Map LabValidationError into FabricTaskError for producer outcome parsing. */ -function wrapValidationError(error: unknown): void { +function wrapValidationError(error: unknown): never { if (error instanceof LabValidationError) { throw new FabricTaskError(error.message, "malformed_producer_outcome", "harness"); } diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts index 0890b478b9..9117a1ed84 100644 --- a/src/lab/fabric/scratch.ts +++ b/src/lab/fabric/scratch.ts @@ -6,12 +6,9 @@ import { lstatSync, mkdirSync, openSync, - readFileSync, readdirSync, readSync, - renameSync, rmSync, - unlinkSync, writeSync, type Stats, } from "node:fs"; @@ -23,28 +20,16 @@ import { FabricTaskError } from "./types"; /** CL-07 restricted scratch tree IO for the synthetic-patch fabric task. */ -const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; const O_DIRECTORY = (fsConstants as { O_DIRECTORY?: number }).O_DIRECTORY; +const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; const FILE_MODE = 0o600; -type OpenSyncWithDir = ( - path: string, - flags: number, - mode: number, - options: { dir: number }, -) => number; - -type ScratchIoMode = "dirfd" | "win32_pinned"; - interface TrustedScratchDir { path: string; fd: number; identity: string; } -let scratchIoMode: ScratchIoMode | null = null; -const trustedScratchRoots = new Map(); - /** Require stats to describe a regular file, not a symlink or special node. */ function assertRegularFile(stats: Stats, label: string): void { if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { @@ -82,9 +67,9 @@ function assertScratchName(name: string): void { } } -/** Re-open scratch dir fd and confirm its identity has not changed. */ +/** Confirm the scratch root path still refers to the same directory inode. */ function revalidateScratchDir(dir: TrustedScratchDir): void { - const stats = fstatSync(dir.fd); + const stats = lstatSync(dir.path); assertRealDirectory(stats, "scratch dir"); if (identityOf(stats) !== dir.identity) { throw new FabricTaskError("scratch directory identity changed", "sandbox_violation", "harness"); @@ -98,47 +83,18 @@ function childScratchPath(dir: TrustedScratchDir, name: string): string { return join(dir.path, name); } -/** Detect whether scratch IO uses dirfd openat or win32 pinned path mode. */ -function detectScratchIoMode(dir: TrustedScratchDir): ScratchIoMode { - if (scratchIoMode !== null) return scratchIoMode; - if (process.platform === "win32") { - scratchIoMode = "win32_pinned"; - return scratchIoMode; - } - const probe = `.dirfd-probe-${process.pid}`; - const finalName = `${probe}.ok`; - try { - const openWithDir = openSync as unknown as OpenSyncWithDir; - const fd = openWithDir(probe, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_RDWR, FILE_MODE, { dir: dir.fd }); - closeSync(fd); - renameSync(join(dir.path, probe), join(dir.path, finalName)); - if (!existsSync(join(dir.path, finalName))) { - scratchIoMode = "win32_pinned"; - return scratchIoMode; - } - unlinkSync(join(dir.path, finalName)); - scratchIoMode = "dirfd"; - } catch { - scratchIoMode = "win32_pinned"; - } - return scratchIoMode; -} - -/** Open a single name relative to a trusted scratch directory fd. */ +/** Open a single name relative to a trusted scratch directory using pinned paths. */ function openAtScratch(dir: TrustedScratchDir, name: string, flags: number, mode = 0, expectDirectory = false): number { revalidateScratchDir(dir); assertScratchName(name); - const mode_ = detectScratchIoMode(dir); - if (mode_ === "dirfd") { - return (openSync as unknown as OpenSyncWithDir)(name, flags, mode, { dir: dir.fd }); - } - const fd = openSync(childScratchPath(dir, name), flags, mode); + const path = childScratchPath(dir, name); + const fd = openSync(path, openFlags(flags), mode); const opened = fstatSync(fd); if (expectDirectory) { assertRealDirectory(opened, name); } else { assertRegularFile(opened, name); - const pathEntry = lstatSync(childScratchPath(dir, name)); + const pathEntry = lstatSync(path); if (identityOf(opened) !== identityOf(pathEntry)) { throw new FabricTaskError("scratch path redirection detected", "sandbox_violation", "harness"); } @@ -160,26 +116,10 @@ function openTrustedScratchRoot(scratchRoot: string): TrustedScratchDir { return { path: abs, fd, identity: identityOf(stats) }; } -/** Return a cached trusted scratch handle, opening the root on first use. */ -function getTrustedScratch(scratchRoot: string): TrustedScratchDir { - const abs = resolve(scratchRoot); - let trusted = trustedScratchRoots.get(abs); - if (!trusted) { - trusted = openTrustedScratchRoot(abs); - trustedScratchRoots.set(abs, trusted); - } - revalidateScratchDir(trusted); - return trusted; -} - -/** Close and drop a cached trusted scratch directory handle. */ -function releaseTrustedScratch(scratchRoot: string): void { - const abs = resolve(scratchRoot); - const trusted = trustedScratchRoots.get(abs); - if (!trusted) return; - trustedScratchRoots.delete(abs); +/** Close a trusted scratch directory handle. */ +function closeTrustedScratchRoot(dir: TrustedScratchDir): void { try { - closeSync(trusted.fd); + closeSync(dir.fd); } catch { /* ignore */ } @@ -210,7 +150,11 @@ function openScratchRelativePath( current = { path: join(current.path, part), fd: subFd, identity: identityOf(subStats) }; continue; } - return openAtScratch(current, part, flags, mode); + const finalFd = openAtScratch(current, part, flags, mode); + for (const fd of intermediateFds) { + closeSync(fd); + } + return finalFd; } throw new FabricTaskError("empty scratch path", "sandbox_violation", "harness"); } catch (error) { @@ -300,6 +244,28 @@ export function resolveInsideScratch(scratchRoot: string, relativePath: string, return current; } +/** Create a relative directory tree under scratch without following symlink components. */ +function ensureScratchRelativeDir(scratchRoot: string, relativeDir: string): void { + const safe = assertSafeRelativePosixPath(relativeDir); + const root = resolve(scratchRoot); + const rootStats = lstatSync(root); + assertRealDirectory(rootStats, "scratch root"); + let current = root; + for (const part of safe.split("/")) { + const next = join(current, part); + assertUnderScratchRoot(root, next); + if (!existsSync(next)) { + mkdirSync(next, { mode: 0o700 }); + } + const stats = lstatSync(next); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + assertRealDirectory(stats, next); + current = next; + } +} + /** Handle for a fabric scratch tree and its best-effort cleanup callback. */ export interface ScratchTree { root: string; @@ -314,34 +280,40 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { const root = join(base, `fabric-${Date.now().toString(36)}-${randomBytes(6).toString("hex")}`); ensureRestrictedDir(root); ensureRestrictedDir(join(root, dirname(SYNTHETIC_VALUE_PATH))); - const trusted = openTrustedScratchRoot(root); - trustedScratchRoots.set(trusted.path, trusted); - const bytes = Buffer.from(SYNTHETIC_BEFORE_UTF8, "utf8"); - if (bytes.byteLength > FABRIC_LIMITS.maxAggregateIoBytes) { - throw new FabricTaskError("fixture exceeds io budget", "budget_exhausted", "harness"); - } - const fd = openScratchRelativePath( - trusted, - SYNTHETIC_VALUE_PATH, - openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, true), - FILE_MODE, - ); + let trusted: TrustedScratchDir | undefined; try { - writeSync(fd, bytes); - } finally { - closeSync(fd); + trusted = openTrustedScratchRoot(root); + const bytes = Buffer.from(SYNTHETIC_BEFORE_UTF8, "utf8"); + if (bytes.byteLength > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("fixture exceeds io budget", "budget_exhausted", "harness"); + } + const fd = openScratchRelativePath( + trusted, + SYNTHETIC_VALUE_PATH, + openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, true), + FILE_MODE, + ); + try { + writeSync(fd, bytes); + } finally { + closeSync(fd); + } + const trustedForCleanup = trusted; + return { + root, + cleanup: () => { + closeTrustedScratchRoot(trustedForCleanup); + try { + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + } catch { + // best-effort cleanup + } + }, + }; + } catch (error) { + if (trusted) closeTrustedScratchRoot(trusted); + throw error; } - return { - root, - cleanup: () => { - releaseTrustedScratch(root); - try { - rmSync(root, { recursive: true, force: true, maxRetries: 3 }); - } catch { - // best-effort cleanup - } - }, - }; } /** One regular file discovered during a bounded scratch tree walk. */ @@ -400,20 +372,21 @@ export function walkScratchFiles(scratchRoot: string): WalkedFile[] { /** Read a scratch file as UTF-8 text under a byte budget. */ export function readScratchFileUtf8(scratchRoot: string, relativePath: string, maxBytes: number): string { resolveInsideScratch(scratchRoot, relativePath); - const trusted = getTrustedScratch(scratchRoot); - const fd = openScratchRelativePath(trusted, relativePath, openFlags(fsConstants.O_RDONLY, true), 0); + const trusted = openTrustedScratchRoot(scratchRoot); try { - const stats = fstatSync(fd); - if (stats.isSymbolicLink()) { - throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); - } - assertRegularFile(stats, relativePath); - if (stats.size > maxBytes) { - throw new FabricTaskError("file exceeds io budget", "budget_exhausted", "environment"); + const fd = openScratchRelativePath(trusted, relativePath, openFlags(fsConstants.O_RDONLY, true), 0); + try { + const stats = fstatSync(fd); + assertRegularFile(stats, relativePath); + if (stats.size > maxBytes) { + throw new FabricTaskError("file exceeds io budget", "budget_exhausted", "environment"); + } + return readAllFromFd(fd, stats.size); + } finally { + closeSync(fd); } - return readAllFromFd(fd, stats.size); } finally { - closeSync(fd); + closeTrustedScratchRoot(trusted); } } @@ -425,26 +398,27 @@ export function writeScratchFileUtf8(scratchRoot: string, relativePath: string, } const safe = assertSafeRelativePosixPath(relativePath); const parentPath = safe.includes("/") ? safe.slice(0, safe.lastIndexOf("/")) : ""; + resolveInsideScratch(scratchRoot, relativePath, { allowMissingFinal: true }); if (parentPath) { - ensureRestrictedDir(join(scratchRoot, parentPath.split("/").join(sep))); + ensureScratchRelativeDir(scratchRoot, parentPath); } - resolveInsideScratch(scratchRoot, relativePath, { allowMissingFinal: true }); - const trusted = getTrustedScratch(scratchRoot); - const fd = openScratchRelativePath( - trusted, - relativePath, - openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC, true), - FILE_MODE, - ); + const trusted = openTrustedScratchRoot(scratchRoot); try { - const stats = fstatSync(fd); - if (stats.isSymbolicLink()) { - throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + const fd = openScratchRelativePath( + trusted, + relativePath, + openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC, true), + FILE_MODE, + ); + try { + const stats = fstatSync(fd); + assertRegularFile(stats, relativePath); + writeSync(fd, bytes); + } finally { + closeSync(fd); } - assertRegularFile(stats, relativePath); - writeSync(fd, bytes); } finally { - closeSync(fd); + closeTrustedScratchRoot(trusted); } return bytes.byteLength; } diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index ac4eb82c55..c4af61ac86 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -11,6 +11,7 @@ import { unlinkSync, writeSync, } from "node:fs"; +import { randomBytes } from "node:crypto"; import { dirname } from "node:path"; import { jcsStringify } from "../digest"; import { MAX_SERIALIZED_EVENT_BYTES } from "../constants"; @@ -31,6 +32,7 @@ const LEDGER_LOCK_WAIT_MS = 5_000; interface LedgerLockMeta { pid: number; createdAt: number; + token: string; } /** Block synchronously for the given duration (ledger lock retry only). */ @@ -41,40 +43,83 @@ function sleepSyncMs(ms: number): void { } } -/** Read pid and createdAt metadata from a ledger lock file, if well-formed. */ +/** Read pid, createdAt, and token metadata from a ledger lock file, if well-formed. */ function readLedgerLockMeta(lockPath: string): LedgerLockMeta | null { try { const parsed = JSON.parse(readFileSync(lockPath, "utf8")) as LedgerLockMeta; - if (typeof parsed.pid === "number" && typeof parsed.createdAt === "number") return parsed; + if ( + typeof parsed.pid === "number" + && typeof parsed.createdAt === "number" + && typeof parsed.token === "string" + && parsed.token.length > 0 + ) { + return parsed; + } } catch { /* ignore */ } return null; } -/** Return true when a ledger lock file is missing metadata or older than the stale window. */ +/** Return true when the lock holder process is still running. */ +function isLockHolderAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +/** Return true when a ledger lock file is missing metadata or its holder is gone and stale. */ function isLedgerLockStale(lockPath: string): boolean { const meta = readLedgerLockMeta(lockPath); if (!meta) return true; + if (isLockHolderAlive(meta.pid)) return false; return Date.now() - meta.createdAt > LEDGER_LOCK_STALE_MS; } /** Create a ledger lock file exclusively, recovering stale locks when needed. */ -function tryAcquireLedgerLock(lockPath: string): number { - try { - const fd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); - writeSync(fd, Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now() }), "utf8")); - return fd; - } catch (error) { - if (existsSync(lockPath) && isLedgerLockStale(lockPath)) { +function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number; token: string } { + while (Date.now() < deadline) { + const token = randomBytes(16).toString("hex"); + try { + const fd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); try { - unlinkSync(lockPath); - } catch { - /* ignore */ + writeSync(fd, Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now(), token }), "utf8")); + } catch (error) { + closeSync(fd); + throw error; + } + return { fd, token }; + } catch (error) { + if (existsSync(lockPath) && isLedgerLockStale(lockPath)) { + try { + unlinkSync(lockPath); + } catch (unlinkError) { + if (Date.now() >= deadline) throw unlinkError; + } + continue; } - return tryAcquireLedgerLock(lockPath); + if (Date.now() >= deadline) throw error; + sleepSyncMs(10); } - throw error; + } + throw new Error("ledger lock acquisition timed out"); +} + +/** Release a ledger lock only when the token still matches the lock file. */ +function releaseLedgerLock(lockPath: string, lockFd: number, token: string): void { + try { + closeSync(lockFd); + } catch { + /* ignore */ + } + try { + const meta = readLedgerLockMeta(lockPath); + if (meta?.token === token) unlinkSync(lockPath); + } catch { + /* best-effort */ } } @@ -83,24 +128,11 @@ function withLedgerLock(ledgerPath: string, fn: () => T): T { const lockPath = `${ledgerPath}.lock`; mkdirSync(dirname(ledgerPath), { recursive: true, mode: 0o700 }); const deadline = Date.now() + LEDGER_LOCK_WAIT_MS; - let lockFd: number | undefined; - while (lockFd === undefined) { - try { - lockFd = tryAcquireLedgerLock(lockPath); - } catch (error) { - if (Date.now() >= deadline) throw error; - sleepSyncMs(10); - } - } + const { fd, token } = tryAcquireLedgerLock(lockPath, deadline); try { return fn(); } finally { - closeSync(lockFd); - try { - unlinkSync(lockPath); - } catch { - /* best-effort */ - } + releaseLedgerLock(lockPath, fd, token); } } diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 565a6a3b11..ce596e76b7 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -1,21 +1,43 @@ import { chmodSync, existsSync, mkdirSync, lstatSync } from "node:fs"; -import { join } from "node:path"; +import { join, resolve, sep } from "node:path"; import { getConfigDir } from "../config"; /** Create (or harden) a directory to mode 0o700 without following symlinks. */ export function ensureRestrictedDir(dir: string): void { - mkdirSync(dir, { recursive: true, mode: 0o700 }); - if (process.platform === "win32") return; - if (!existsSync(dir)) return; - const stats = lstatSync(dir); + if (process.platform === "win32") { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + return; + } + const abs = resolve(dir); + const parts = abs.split(sep); + let current = parts[0] === "" ? sep : parts[0]!; + for (const part of parts.slice(1)) { + if (part === "") continue; + current = join(current, part); + try { + mkdirSync(current, { mode: 0o700 }); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "EEXIST") throw error; + } + const componentStats = lstatSync(current); + if (componentStats.isSymbolicLink()) { + throw new Error(`restricted directory component is a symbolic link: ${current}`); + } + if (!componentStats.isDirectory()) { + throw new Error(`restricted path component is not a directory: ${current}`); + } + } + if (!existsSync(abs)) return; + const stats = lstatSync(abs); if (stats.isSymbolicLink()) { - throw new Error(`restricted directory is a symbolic link: ${dir}`); + throw new Error(`restricted directory is a symbolic link: ${abs}`); } if (!stats.isDirectory()) { - throw new Error(`restricted path is not a directory: ${dir}`); + throw new Error(`restricted path is not a directory: ${abs}`); } const mode = stats.mode & 0o777; - if (mode !== 0o700) chmodSync(dir, 0o700); + if (mode !== 0o700) chmodSync(abs, 0o700); } /** Canonical Compatibility Lab state root under the OpenCodex config dir. */ diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index d93e2c7de3..3659ee9a91 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -87,7 +87,9 @@ export function taskSubjectApplicableToRequirements( const platforms = requirements.platforms ?? ["*"]; const features = requirements.requiredHarnessFeatures ?? []; const preconditions = requirements.routePreconditions ?? []; - const platformOk = platforms.includes("*") || platforms.some((platform) => capability.platforms.includes(platform)); + const platformOk = platforms.includes("*") + || capability.platforms.includes("*") + || platforms.every((platform) => capability.platforms.includes(platform)); const featuresOk = features.every((feature) => capability.harnessFeatures.includes(feature)); const preconditionsOk = preconditions.every((item) => capability.routePreconditions.includes(item)); return platformOk && featuresOk && preconditionsOk; @@ -98,6 +100,7 @@ function isNonNegativeInteger(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0; } +/** Parse a freshness object from a scenario manifest fragment. */ function parseFreshness(value: unknown): { maxAgeMs: number | null } | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; const maxAgeMs = (value as { maxAgeMs?: unknown }).maxAgeMs; @@ -106,6 +109,7 @@ function parseFreshness(value: unknown): { maxAgeMs: number | null } | null { return { maxAgeMs }; } +/** Parse a string array requirement field from a manifest fragment. */ function parseStringArray(value: unknown): string[] | null { if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) return null; return value; diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index 6d0b80d4a6..4b4c5101b7 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -387,6 +387,15 @@ describe("CL-07 task effectiveness producer", () => { platforms: ["win32"], routePreconditions: ["exact-route-subject"], })).toBe(true); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: [], + platforms: ["linux"], + routePreconditions: [], + }, { + harnessFeatures: [], + platforms: ["win32", "*"], + routePreconditions: [], + })).toBe(true); }); test("malformed nested outcome fields throw FabricTaskError", async () => { @@ -396,15 +405,27 @@ describe("CL-07 task effectiveness producer", () => { configDir: home, producePatch: () => correctSyntheticPatch(), }); - expect(() => observationFromFabricOutcome({ ...base, verifier: {} })).toThrow(); - expect(() => observationFromFabricOutcome({ - ...base, - usage: { ...base.usage, inputBytes: -1 }, - })).toThrow(); - expect(() => observationFromFabricOutcome({ - ...base, - taskSubject: { ...base.taskSubject, subjectKind: "route" }, - })).toThrow(); + expect(() => observationFromFabricOutcome({ ...base, verifier: {} })).toThrow(FabricTaskError); + try { + observationFromFabricOutcome({ + ...base, + usage: { ...base.usage, inputBytes: -1 }, + }); + throw new Error("expected malformed usage to throw"); + } catch (error) { + expect(error).toBeInstanceOf(FabricTaskError); + expect((error as FabricTaskError).code).toBe("malformed_producer_outcome"); + } + try { + observationFromFabricOutcome({ + ...base, + routeSubject: routeSubject({ providerId: "other-provider" }), + }); + throw new Error("expected layer mismatch to throw"); + } catch (error) { + expect(error).toBeInstanceOf(FabricTaskError); + expect((error as FabricTaskError).code).toBe("layer_subject_mismatch"); + } }); test("verifier summary artifact omits raw credential diagnostics", async () => { @@ -448,7 +469,7 @@ describe("CL-07 task effectiveness producer", () => { process.env.OPENCODEX_HOME = home; mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); const lockPath = join(home, "lab", "compatibility.jsonl.lock"); - writeFileSync(lockPath, JSON.stringify({ pid: 99999, createdAt: Date.now() - 120_000 }), { mode: 0o600 }); + writeFileSync(lockPath, JSON.stringify({ pid: 99999, createdAt: Date.now() - 120_000, token: "stale-lock-token" }), { mode: 0o600 }); const outcome = await runFabricSyntheticPatchTask({ routeSubject: routeSubject(), configDir: home, @@ -474,6 +495,22 @@ describe("CL-07 task effectiveness producer", () => { expect(readdirSync(outside).some((name) => name.startsWith("fabric-"))).toBe(false); }); + test("symlinked lab root rejects directory escape via ensureLabDirs", () => { + if (process.platform === "win32") return; + const home = tempHome(); + const outside = join(home, "outside-lab"); + mkdirSync(outside, { recursive: true, mode: 0o700 }); + const labLink = join(home, "lab"); + try { + symlinkSync(outside, labLink); + } catch { + return; + } + expect(() => ensureLabDirs(home)).toThrow(); + expect(existsSync(join(outside, "scratch"))).toBe(false); + expect(existsSync(join(outside, "artifacts"))).toBe(false); + }); + test("duplicate outcome delivery is idempotent; distinct attempts remain distinct", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; From e00a6408e38ab62e9a8255ffc3e93d499f7eb0aa Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:29:43 +0200 Subject: [PATCH 08/24] fix(lab): address fourth CodeRabbit round for CL-07 fabric producer Limit symlink checks to Lab-owned paths, harden ledger lock recovery, consolidate scratch fd cleanup, and compare route subjects via JCS. --- src/lab/fabric/observe.ts | 6 ++--- src/lab/fabric/scratch.ts | 25 +++++++++++---------- src/lab/ledger/store.ts | 10 ++++++--- src/lab/paths.ts | 41 +++++++++++++++++++++++++---------- tests/lab-fabric-task.test.ts | 26 +++++++++++++++++++--- 5 files changed, 75 insertions(+), 33 deletions(-) diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 2f078b219c..5d8ee49ff1 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -8,7 +8,7 @@ import { OUTCOMES, } from "../constants"; import { FAILURE_CLASSIFICATIONS } from "../conformance/types"; -import { fixtureDigest, isSha256Hex } from "../digest"; +import { fixtureDigest, isSha256Hex, jcsStringify } from "../digest"; import type { ObservationEvent, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; import { LabValidationError } from "../events/errors"; import { assignEventId, validateSubject } from "../events/validate"; @@ -205,9 +205,9 @@ function validateFailureRecord(raw: Record): void { } } -/** Compare top-level and nested route subjects by canonical JSON equality. */ +/** Compare top-level and nested route subjects by canonical JCS representation. */ function routeSubjectsMatch(top: RouteSubjectV1, nested: RouteSubjectV1): boolean { - return JSON.stringify(top) === JSON.stringify(nested); + return jcsStringify(top) === jcsStringify(nested); } /** Build an allowlisted verifier_summary artifact with sanitized diagnostics. */ diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts index 9117a1ed84..0a7092b4e9 100644 --- a/src/lab/fabric/scratch.ts +++ b/src/lab/fabric/scratch.ts @@ -14,7 +14,7 @@ import { } from "node:fs"; import { dirname, join, posix, relative, resolve, sep } from "node:path"; import { randomBytes } from "node:crypto"; -import { ensureLabDirs, ensureRestrictedDir, labScratchDir } from "../paths"; +import { ensureLabDirs, ensureRestrictedDir, labRoot, labScratchDir } from "../paths"; import { FABRIC_LIMITS, SYNTHETIC_BEFORE_UTF8, SYNTHETIC_VALUE_PATH } from "./constants"; import { FabricTaskError } from "./types"; @@ -150,14 +150,10 @@ function openScratchRelativePath( current = { path: join(current.path, part), fd: subFd, identity: identityOf(subStats) }; continue; } - const finalFd = openAtScratch(current, part, flags, mode); - for (const fd of intermediateFds) { - closeSync(fd); - } - return finalFd; + return openAtScratch(current, part, flags, mode); } throw new FabricTaskError("empty scratch path", "sandbox_violation", "harness"); - } catch (error) { + } finally { for (const fd of intermediateFds) { try { closeSync(fd); @@ -165,7 +161,6 @@ function openScratchRelativePath( /* ignore */ } } - throw error; } } @@ -255,7 +250,12 @@ function ensureScratchRelativeDir(scratchRoot: string, relativeDir: string): voi const next = join(current, part); assertUnderScratchRoot(root, next); if (!existsSync(next)) { - mkdirSync(next, { mode: 0o700 }); + try { + mkdirSync(next, { mode: 0o700 }); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "EEXIST") throw error; + } } const stats = lstatSync(next); if (stats.isSymbolicLink()) { @@ -275,11 +275,12 @@ export interface ScratchTree { /** Create an isolated scratch tree with the frozen synthetic-patch fixture file. */ export function createSyntheticScratch(configDir?: string): ScratchTree { ensureLabDirs(configDir); + const labBoundary = labRoot(configDir); const base = labScratchDir(configDir); - ensureRestrictedDir(base); + ensureRestrictedDir(base, labBoundary); const root = join(base, `fabric-${Date.now().toString(36)}-${randomBytes(6).toString("hex")}`); - ensureRestrictedDir(root); - ensureRestrictedDir(join(root, dirname(SYNTHETIC_VALUE_PATH))); + ensureRestrictedDir(root, labBoundary); + ensureRestrictedDir(join(root, dirname(SYNTHETIC_VALUE_PATH)), labBoundary); let trusted: TrustedScratchDir | undefined; try { trusted = openTrustedScratchRoot(root); diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index c4af61ac86..6a7c084cf7 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -66,15 +66,18 @@ function isLockHolderAlive(pid: number): boolean { try { process.kill(pid, 0); return true; - } catch { + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ESRCH") return false; + if (code === "EPERM") return true; return false; } } -/** Return true when a ledger lock file is missing metadata or its holder is gone and stale. */ +/** Return true when a ledger lock file has dead metadata and can be recovered. */ function isLedgerLockStale(lockPath: string): boolean { const meta = readLedgerLockMeta(lockPath); - if (!meta) return true; + if (!meta) return false; if (isLockHolderAlive(meta.pid)) return false; return Date.now() - meta.createdAt > LEDGER_LOCK_STALE_MS; } @@ -98,6 +101,7 @@ function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number; unlinkSync(lockPath); } catch (unlinkError) { if (Date.now() >= deadline) throw unlinkError; + sleepSyncMs(10); } continue; } diff --git a/src/lab/paths.ts b/src/lab/paths.ts index ce596e76b7..1e2d2e6d71 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -2,13 +2,27 @@ import { chmodSync, existsSync, mkdirSync, lstatSync } from "node:fs"; import { join, resolve, sep } from "node:path"; import { getConfigDir } from "../config"; -/** Create (or harden) a directory to mode 0o700 without following symlinks. */ -export function ensureRestrictedDir(dir: string): void { +/** Return true when path is the Lab root or a descendant path component. */ +function isAtOrBelowLabBoundary(path: string, labRootBoundary: string): boolean { + const normalizedPath = resolve(path); + const normalizedBoundary = resolve(labRootBoundary); + if (normalizedPath === normalizedBoundary) return true; + const prefix = normalizedBoundary + sep; + return normalizedPath.startsWith(prefix) && normalizedPath.length > normalizedBoundary.length; +} + +/** + * Create (or harden) a directory to mode 0o700 without following symlinks on Lab-owned + * components. Ancestors above labRootBoundary are treated as infrastructure and are not + * inspected for symlinks (for example macOS /var → /private/var). + */ +export function ensureRestrictedDir(dir: string, labRootBoundary?: string): void { if (process.platform === "win32") { mkdirSync(dir, { recursive: true, mode: 0o700 }); return; } const abs = resolve(dir); + const boundary = resolve(labRootBoundary ?? abs); const parts = abs.split(sep); let current = parts[0] === "" ? sep : parts[0]!; for (const part of parts.slice(1)) { @@ -20,15 +34,18 @@ export function ensureRestrictedDir(dir: string): void { const code = (error as NodeJS.ErrnoException).code; if (code !== "EEXIST") throw error; } - const componentStats = lstatSync(current); - if (componentStats.isSymbolicLink()) { - throw new Error(`restricted directory component is a symbolic link: ${current}`); - } - if (!componentStats.isDirectory()) { - throw new Error(`restricted path component is not a directory: ${current}`); + if (isAtOrBelowLabBoundary(current, boundary)) { + const componentStats = lstatSync(current); + if (componentStats.isSymbolicLink()) { + throw new Error(`restricted directory component is a symbolic link: ${current}`); + } + if (!componentStats.isDirectory()) { + throw new Error(`restricted path component is not a directory: ${current}`); + } } } if (!existsSync(abs)) return; + if (!isAtOrBelowLabBoundary(abs, boundary)) return; const stats = lstatSync(abs); if (stats.isSymbolicLink()) { throw new Error(`restricted directory is a symbolic link: ${abs}`); @@ -83,10 +100,10 @@ export function ensureLabDirs(configDir = getConfigDir()): { const artifactsDir = labArtifactsDir(configDir); const scratchDir = labScratchDir(configDir); const exportDir = labExportDir(configDir); - ensureRestrictedDir(root); - ensureRestrictedDir(artifactsDir); - ensureRestrictedDir(scratchDir); - ensureRestrictedDir(exportDir); + ensureRestrictedDir(root, root); + ensureRestrictedDir(artifactsDir, root); + ensureRestrictedDir(scratchDir, root); + ensureRestrictedDir(exportDir, root); return { root, ledgerPath: labLedgerPath(configDir), diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index 4b4c5101b7..5c59b427eb 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -46,7 +46,7 @@ import { writeScratchFileUtf8, readScratchFileUtf8 } from "../src/lab/fabric/scr import { routingProfileIssues } from "../src/routing/profile"; import { buildInvalidationIndex } from "../src/lab/ledger/invalidation"; import { createArtifactStore } from "../src/lab/artifacts/store"; -import { ensureLabDirs } from "../src/lab/paths"; +import { ensureLabDirs, ensureRestrictedDir } from "../src/lab/paths"; import { verifyExactTreeDiffV1 } from "../src/lab/fabric/verifier"; import { parseSyntheticPatchV1 } from "../src/lab/fabric/patch"; import { FABRIC_LIMITS } from "../src/lab/fabric/constants"; @@ -469,7 +469,11 @@ describe("CL-07 task effectiveness producer", () => { process.env.OPENCODEX_HOME = home; mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); const lockPath = join(home, "lab", "compatibility.jsonl.lock"); - writeFileSync(lockPath, JSON.stringify({ pid: 99999, createdAt: Date.now() - 120_000, token: "stale-lock-token" }), { mode: 0o600 }); + writeFileSync(lockPath, JSON.stringify({ + pid: 4_000_000_000, + createdAt: Date.now() - 120_000, + token: "stale-lock-token", + }), { mode: 0o600 }); const outcome = await runFabricSyntheticPatchTask({ routeSubject: routeSubject(), configDir: home, @@ -506,11 +510,27 @@ describe("CL-07 task effectiveness producer", () => { } catch { return; } - expect(() => ensureLabDirs(home)).toThrow(); + expect(() => ensureLabDirs(home)).toThrow(/symbolic link/i); expect(existsSync(join(outside, "scratch"))).toBe(false); expect(existsSync(join(outside, "artifacts"))).toBe(false); }); + test("symlink under lab boundary is rejected by ensureRestrictedDir", () => { + if (process.platform === "win32") return; + const home = tempHome(); + const labDir = join(home, "lab"); + const outside = join(home, "outside-scratch"); + mkdirSync(labDir, { recursive: true, mode: 0o700 }); + mkdirSync(outside, { recursive: true, mode: 0o700 }); + const scratchLink = join(labDir, "scratch"); + try { + symlinkSync(outside, scratchLink); + } catch { + return; + } + expect(() => ensureRestrictedDir(join(labDir, "scratch", "nested"), labDir)).toThrow(/symbolic link/i); + }); + test("duplicate outcome delivery is idempotent; distinct attempts remain distinct", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; From 9edeeaf4c1fd8a91189bca0208a73e2fd8e636f8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:46:26 +0200 Subject: [PATCH 09/24] fix(lab): address fifth CodeRabbit round for CL-07 fabric producer - Clean up partial scratch dirs on createSyntheticScratch failure - Age out metadata-free ledger locks via mtime and unlink on write failure - Require lab root boundary for ensureRestrictedDir; validate reparse points on Windows - Use platforms.some for capability allow-list semantics - Add junction/symlink and scratch cleanup regression tests --- src/lab/fabric/scratch.ts | 9 +++- src/lab/ledger/store.ts | 13 ++++- src/lab/paths.ts | 45 +++++++++--------- src/lab/projection/verification.ts | 2 +- tests/lab-fabric-task.test.ts | 76 +++++++++++++++++++++--------- 5 files changed, 98 insertions(+), 47 deletions(-) diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts index 0a7092b4e9..6f0c9e8a60 100644 --- a/src/lab/fabric/scratch.ts +++ b/src/lab/fabric/scratch.ts @@ -279,10 +279,10 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { const base = labScratchDir(configDir); ensureRestrictedDir(base, labBoundary); const root = join(base, `fabric-${Date.now().toString(36)}-${randomBytes(6).toString("hex")}`); - ensureRestrictedDir(root, labBoundary); - ensureRestrictedDir(join(root, dirname(SYNTHETIC_VALUE_PATH)), labBoundary); let trusted: TrustedScratchDir | undefined; try { + ensureRestrictedDir(root, labBoundary); + ensureRestrictedDir(join(root, dirname(SYNTHETIC_VALUE_PATH)), labBoundary); trusted = openTrustedScratchRoot(root); const bytes = Buffer.from(SYNTHETIC_BEFORE_UTF8, "utf8"); if (bytes.byteLength > FABRIC_LIMITS.maxAggregateIoBytes) { @@ -313,6 +313,11 @@ export function createSyntheticScratch(configDir?: string): ScratchTree { }; } catch (error) { if (trusted) closeTrustedScratchRoot(trusted); + try { + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + } catch { + /* best-effort */ + } throw error; } } diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 6a7c084cf7..09ee1f9cbe 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -77,7 +77,13 @@ function isLockHolderAlive(pid: number): boolean { /** Return true when a ledger lock file has dead metadata and can be recovered. */ function isLedgerLockStale(lockPath: string): boolean { const meta = readLedgerLockMeta(lockPath); - if (!meta) return false; + if (!meta) { + try { + return Date.now() - statSync(lockPath).mtimeMs > LEDGER_LOCK_STALE_MS; + } catch { + return false; + } + } if (isLockHolderAlive(meta.pid)) return false; return Date.now() - meta.createdAt > LEDGER_LOCK_STALE_MS; } @@ -92,6 +98,11 @@ function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number; writeSync(fd, Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now(), token }), "utf8")); } catch (error) { closeSync(fd); + try { + unlinkSync(lockPath); + } catch { + /* best-effort */ + } throw error; } return { fd, token }; diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 1e2d2e6d71..03a815b488 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -1,4 +1,4 @@ -import { chmodSync, existsSync, mkdirSync, lstatSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, lstatSync, realpathSync } from "node:fs"; import { join, resolve, sep } from "node:path"; import { getConfigDir } from "../config"; @@ -11,18 +11,29 @@ function isAtOrBelowLabBoundary(path: string, labRootBoundary: string): boolean return normalizedPath.startsWith(prefix) && normalizedPath.length > normalizedBoundary.length; } +/** Reject symlink, non-directory, and reparse-point substitutions for a Lab-owned component. */ +function assertRestrictedDirectoryComponent(path: string): void { + const componentStats = lstatSync(path); + if (componentStats.isSymbolicLink()) { + throw new Error(`restricted directory component is a symbolic link: ${path}`); + } + if (!componentStats.isDirectory()) { + throw new Error(`restricted path component is not a directory: ${path}`); + } + const canonical = realpathSync.native(path); + if (resolve(canonical) !== resolve(path)) { + throw new Error(`restricted directory component contains a link or reparse-point substitution: ${path}`); + } +} + /** * Create (or harden) a directory to mode 0o700 without following symlinks on Lab-owned * components. Ancestors above labRootBoundary are treated as infrastructure and are not * inspected for symlinks (for example macOS /var → /private/var). */ -export function ensureRestrictedDir(dir: string, labRootBoundary?: string): void { - if (process.platform === "win32") { - mkdirSync(dir, { recursive: true, mode: 0o700 }); - return; - } +export function ensureRestrictedDir(dir: string, labRootBoundary: string): void { const abs = resolve(dir); - const boundary = resolve(labRootBoundary ?? abs); + const boundary = resolve(labRootBoundary); const parts = abs.split(sep); let current = parts[0] === "" ? sep : parts[0]!; for (const part of parts.slice(1)) { @@ -35,26 +46,16 @@ export function ensureRestrictedDir(dir: string, labRootBoundary?: string): void if (code !== "EEXIST") throw error; } if (isAtOrBelowLabBoundary(current, boundary)) { - const componentStats = lstatSync(current); - if (componentStats.isSymbolicLink()) { - throw new Error(`restricted directory component is a symbolic link: ${current}`); - } - if (!componentStats.isDirectory()) { - throw new Error(`restricted path component is not a directory: ${current}`); - } + assertRestrictedDirectoryComponent(current); } } if (!existsSync(abs)) return; if (!isAtOrBelowLabBoundary(abs, boundary)) return; - const stats = lstatSync(abs); - if (stats.isSymbolicLink()) { - throw new Error(`restricted directory is a symbolic link: ${abs}`); - } - if (!stats.isDirectory()) { - throw new Error(`restricted path is not a directory: ${abs}`); + assertRestrictedDirectoryComponent(abs); + if (process.platform !== "win32") { + const mode = lstatSync(abs).mode & 0o777; + if (mode !== 0o700) chmodSync(abs, 0o700); } - const mode = stats.mode & 0o777; - if (mode !== 0o700) chmodSync(abs, 0o700); } /** Canonical Compatibility Lab state root under the OpenCodex config dir. */ diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index 3659ee9a91..86ef9d6f08 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -89,7 +89,7 @@ export function taskSubjectApplicableToRequirements( const preconditions = requirements.routePreconditions ?? []; const platformOk = platforms.includes("*") || capability.platforms.includes("*") - || platforms.every((platform) => capability.platforms.includes(platform)); + || platforms.some((platform) => capability.platforms.includes(platform)); const featuresOk = features.every((feature) => capability.harnessFeatures.includes(feature)); const preconditionsOk = preconditions.every((item) => capability.routePreconditions.includes(item)); return platformOk && featuresOk && preconditionsOk; diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index 5c59b427eb..e28150ecc2 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import * as nodeFs from "node:fs"; import { existsSync, mkdirSync, @@ -61,6 +62,17 @@ function tempHome(): string { return dir; } +function linkDirectory(target: string, linkPath: string): boolean { + try { + symlinkSync(target, linkPath, process.platform === "win32" ? "junction" : "dir"); + return true; + } catch { + return false; + } +} + +const RESTRICTED_LINK_ERROR = /symbolic link|reparse-point substitution/i; + afterEach(() => { for (const dir of HOMES.splice(0)) { try { @@ -396,6 +408,24 @@ describe("CL-07 task effectiveness producer", () => { platforms: ["win32", "*"], routePreconditions: [], })).toBe(true); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: [], + platforms: ["linux", "darwin"], + routePreconditions: [], + }, { + harnessFeatures: [], + platforms: ["linux"], + routePreconditions: [], + })).toBe(true); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: [], + platforms: ["linux", "darwin"], + routePreconditions: [], + }, { + harnessFeatures: [], + platforms: ["win32"], + routePreconditions: [], + })).toBe(false); }); test("malformed nested outcome fields throw FabricTaskError", async () => { @@ -484,51 +514,55 @@ describe("CL-07 task effectiveness producer", () => { }); test("symlinked lab scratch dir is rejected", () => { - if (process.platform === "win32") return; const home = tempHome(); const outside = join(home, "outside-scratch"); mkdirSync(outside, { recursive: true, mode: 0o700 }); mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); const scratchLink = join(home, "lab", "scratch"); - try { - symlinkSync(outside, scratchLink); - } catch { - return; - } + if (!linkDirectory(outside, scratchLink)) return; expect(() => createSyntheticScratch(home)).toThrow(); expect(readdirSync(outside).some((name) => name.startsWith("fabric-"))).toBe(false); }); + test("failed scratch construction removes partial fabric directories", () => { + const home = tempHome(); + const originalWriteSync = nodeFs.writeSync; + const writeSpy = spyOn(nodeFs, "writeSync").mockImplementation((...args: Parameters) => { + const buffer = args[1]; + if (Buffer.isBuffer(buffer) && buffer.toString("utf8") === SYNTHETIC_BEFORE_UTF8) { + throw new Error("simulated fixture write failure"); + } + return originalWriteSync(...args); + }); + try { + expect(() => createSyntheticScratch(home)).toThrow("simulated fixture write failure"); + const scratchBase = join(home, "lab", "scratch"); + expect(readdirSync(scratchBase).some((name) => name.startsWith("fabric-"))).toBe(false); + } finally { + writeSpy.mockRestore(); + } + }); + test("symlinked lab root rejects directory escape via ensureLabDirs", () => { - if (process.platform === "win32") return; const home = tempHome(); const outside = join(home, "outside-lab"); mkdirSync(outside, { recursive: true, mode: 0o700 }); const labLink = join(home, "lab"); - try { - symlinkSync(outside, labLink); - } catch { - return; - } - expect(() => ensureLabDirs(home)).toThrow(/symbolic link/i); + if (!linkDirectory(outside, labLink)) return; + expect(() => ensureLabDirs(home)).toThrow(RESTRICTED_LINK_ERROR); expect(existsSync(join(outside, "scratch"))).toBe(false); expect(existsSync(join(outside, "artifacts"))).toBe(false); }); test("symlink under lab boundary is rejected by ensureRestrictedDir", () => { - if (process.platform === "win32") return; const home = tempHome(); const labDir = join(home, "lab"); const outside = join(home, "outside-scratch"); mkdirSync(labDir, { recursive: true, mode: 0o700 }); mkdirSync(outside, { recursive: true, mode: 0o700 }); const scratchLink = join(labDir, "scratch"); - try { - symlinkSync(outside, scratchLink); - } catch { - return; - } - expect(() => ensureRestrictedDir(join(labDir, "scratch", "nested"), labDir)).toThrow(/symbolic link/i); + if (!linkDirectory(outside, scratchLink)) return; + expect(() => ensureRestrictedDir(join(labDir, "scratch", "nested"), labDir)).toThrow(RESTRICTED_LINK_ERROR); }); test("duplicate outcome delivery is idempotent; distinct attempts remain distinct", async () => { From 006f880d31d21930da798eb9fc8e8f40ef813ad1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:14:07 +0200 Subject: [PATCH 10/24] fix(lab): propagate ledger lock metadata write failures immediately Separate openSync collision retries from writeSync metadata failures so appendLabEventIfAbsent surfaces the original error without waiting for the lock timeout. Add regression test. --- src/lab/ledger/store.ts | 28 ++++++++++++++------------ tests/lab-evidence-ledger.test.ts | 33 ++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 09ee1f9cbe..1bb29a04e2 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -92,20 +92,9 @@ function isLedgerLockStale(lockPath: string): boolean { function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number; token: string } { while (Date.now() < deadline) { const token = randomBytes(16).toString("hex"); + let fd: number; try { - const fd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); - try { - writeSync(fd, Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now(), token }), "utf8")); - } catch (error) { - closeSync(fd); - try { - unlinkSync(lockPath); - } catch { - /* best-effort */ - } - throw error; - } - return { fd, token }; + fd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); } catch (error) { if (existsSync(lockPath) && isLedgerLockStale(lockPath)) { try { @@ -118,7 +107,20 @@ function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number; } if (Date.now() >= deadline) throw error; sleepSyncMs(10); + continue; + } + try { + writeSync(fd, Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now(), token }), "utf8")); + } catch (error) { + closeSync(fd); + try { + unlinkSync(lockPath); + } catch { + /* best-effort */ + } + throw error; } + return { fd, token }; } throw new Error("ledger lock acquisition timed out"); } diff --git a/tests/lab-evidence-ledger.test.ts b/tests/lab-evidence-ledger.test.ts index 9678c9dd4d..7bd8bae67f 100644 --- a/tests/lab-evidence-ledger.test.ts +++ b/tests/lab-evidence-ledger.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import * as nodeFs from "node:fs"; import { chmodSync, existsSync, mkdirSync, writeFileSync, symlinkSync, linkSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -30,6 +31,7 @@ import { LAB_PROJECTION_SPEC_VERSION, behaviorFingerprintForCase, } from "../src/lab"; +import { appendLabEventIfAbsent } from "../src/lab/ledger/store"; import { ArtifactFsError, closeTrustedArtifactDir, putArtifactBytes, putNamedDigestBytes, readArtifactBytes, digestFileName } from "../src/lab/artifacts/secure-fs"; import { expandSuiteManifest } from "../src/lab/conformance/suite-manifest"; import { evaluateAllApplicableRequiredPassV1 } from "../src/lab/projection/verification"; @@ -279,6 +281,35 @@ describe("CL-02 ledger append/replay", () => { expect(replay.corruptions.some((c) => c.kind === "duplicate_event")).toBe(true); }); }); + + test("ledger lock metadata write failure propagates without lock-timeout wait", () => { + withHome((home) => { + const event = baseObservation(); + const ledger = join(home, "lab", "compatibility.jsonl"); + mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); + const metadataWriteError = new Error("simulated ledger lock metadata write failure"); + const originalWriteSync = nodeFs.writeSync; + const writeSpy = spyOn(nodeFs, "writeSync").mockImplementation((...args: Parameters) => { + const buffer = args[1]; + if ( + Buffer.isBuffer(buffer) + && buffer.includes('"pid"') + && buffer.includes('"createdAt"') + && buffer.includes('"token"') + ) { + throw metadataWriteError; + } + return originalWriteSync(...args); + }); + try { + const start = Date.now(); + expect(() => appendLabEventIfAbsent(ledger, event)).toThrow(metadataWriteError); + expect(Date.now() - start).toBeLessThan(500); + } finally { + writeSpy.mockRestore(); + } + }); + }); }); describe("CL-02 invalidation validation", () => { From f45cefcf218bd19e4ecd56e6784d9004ac99a052 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:29:17 +0200 Subject: [PATCH 11/24] fix(lab): reject incomplete ledger lock metadata writes Verify writeSync byte count for lock metadata and throw short_write so cleanup unlinks the lock. Add regression test for zero-byte metadata write. --- src/lab/ledger/store.ts | 6 +++++- tests/lab-evidence-ledger.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 1bb29a04e2..ff0ac451ec 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -110,7 +110,11 @@ function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number; continue; } try { - writeSync(fd, Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now(), token }), "utf8")); + const metadataBytes = Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now(), token }), "utf8"); + const written = writeSync(fd, metadataBytes); + if (written !== metadataBytes.byteLength) { + throw new LabValidationError("short_write", "ledger lock metadata write incomplete"); + } } catch (error) { closeSync(fd); try { diff --git a/tests/lab-evidence-ledger.test.ts b/tests/lab-evidence-ledger.test.ts index 7bd8bae67f..ed633c7b00 100644 --- a/tests/lab-evidence-ledger.test.ts +++ b/tests/lab-evidence-ledger.test.ts @@ -310,6 +310,34 @@ describe("CL-02 ledger append/replay", () => { } }); }); + + test("incomplete ledger lock metadata write removes lock file", () => { + withHome((home) => { + const event = baseObservation(); + const ledger = join(home, "lab", "compatibility.jsonl"); + const lockPath = join(home, "lab", "compatibility.jsonl.lock"); + mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); + const originalWriteSync = nodeFs.writeSync; + const writeSpy = spyOn(nodeFs, "writeSync").mockImplementation((...args: Parameters) => { + const buffer = args[1]; + if ( + Buffer.isBuffer(buffer) + && buffer.includes('"pid"') + && buffer.includes('"createdAt"') + && buffer.includes('"token"') + ) { + return 0; + } + return originalWriteSync(...args); + }); + try { + expect(() => appendLabEventIfAbsent(ledger, event)).toThrow(LabValidationError); + expect(existsSync(lockPath)).toBe(false); + } finally { + writeSpy.mockRestore(); + } + }); + }); }); describe("CL-02 invalidation validation", () => { From 34777e6346837752f0a31ce2b687244ce1ce310e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:11:04 +0200 Subject: [PATCH 12/24] fix(lab): CL-07 authoritative route execution and terminable producer Bind production task evidence to trusted route execution via buildRouteSubjectV1 and host-issued patch executors. Run patch producers in an isolated Bun child with hard kill on timeout. Separate harness vs trusted persistence, fix verifier infrastructure failure taxonomy, inactiveMs accounting, and privacy canary pattern. --- .../007_cl07_task_effectiveness.md | 38 +- src/lab/artifacts/sanitize.ts | 4 +- src/lab/fabric/executor.ts | 449 ++++++++++-------- src/lab/fabric/index.ts | 15 +- src/lab/fabric/observe.ts | 17 +- src/lab/fabric/producer-child.ts | 115 +++++ src/lab/fabric/producer-isolate.ts | 130 +++++ src/lab/fabric/types.ts | 36 +- src/lab/fabric/verifier.ts | 12 +- src/lib/fabric-task-execution-authority.ts | 7 + src/lib/fabric-task-host.ts | 29 ++ .../fabric-executors/correct-patch.ts | 9 + tests/helpers/fabric-task-test.ts | 145 ++++++ tests/lab-fabric-task.test.ts | 382 +++++++++++---- 14 files changed, 1083 insertions(+), 305 deletions(-) create mode 100644 src/lab/fabric/producer-child.ts create mode 100644 src/lab/fabric/producer-isolate.ts create mode 100644 src/lib/fabric-task-execution-authority.ts create mode 100644 src/lib/fabric-task-host.ts create mode 100644 tests/fixtures/fabric-executors/correct-patch.ts create mode 100644 tests/helpers/fabric-task-test.ts diff --git a/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md index 6eb05f43b8..4aedfaf16b 100644 --- a/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md +++ b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md @@ -105,9 +105,9 @@ No additional task classes in this phase. ### A.5 Patch producer seam -Execution does **not** embed a general coding agent. The executor accepts a closed `SyntheticPatchV1` (single `replace` of `src/value.txt` under byte bounds) from a caller-supplied producer function. +Execution does **not** embed a general coding agent. Production evidence requires a host-issued `TrustedFabricPatchExecutor` invoked through `runFabricSyntheticPatchTaskForRoute`, with `RouteSubjectV1` built from `routeContext` + `destination` via `buildRouteSubjectV1`. Patch producers run in an isolated Bun child with hard termination on timeout. -- Tests inject deterministic patches. +- Tests use `runFabricSyntheticPatchTaskHarness` with closed `FabricHarnessProducerKind` values or fixture executor modules; harness outcomes are not persistable via `persistFabricRunResult`. - A future live route adapter may call a provider **outside** the scratch sandbox and return only a validated `SyntheticPatchV1`; raw prompts/responses never enter Lab storage. - CL-07 does **not** ship automatic background execution (CL-08). @@ -232,15 +232,39 @@ Typecheck; focused fabric/subject/sandbox/verifier/observe/ledger/projection/rea ## M. Status - **Authorization:** IN PROGRESS from CL-06 merge `b66e33ce7207d91014644d99317e456c992a3418` -- **Implementation head:** `ed8e0794189b214398d835e1af828905f29fbc53` +- **Implementation head:** pending push (authoritative route execution + isolated producer child) - **Accepted head:** NOT YET - **PR:** [#1438](https://github.com/lidge-jun/opencodex/pull/1438) (draft) -### Local validation (pre-PR) +### Authoritative route execution boundary + +- Production evidence uses `runFabricSyntheticPatchTaskForRoute({ routeContext, destination, patchExecutor })`. +- `RouteSubjectV1` is built only via `buildRouteSubjectV1(routeContext, destination)` — callers cannot supply an independent route identity. +- Patch production requires a host-issued `TrustedFabricPatchExecutor` (`createHostIssuedFabricPatchExecutor` in `src/lib/fabric-task-host.ts`). +- `persistFabricRunResult` rejects harness-only runs (`executionAuthority !== "trusted_route"`). +- Harness runs (`runFabricSyntheticPatchTaskHarness`) use closed `FabricHarnessProducerKind` values only; they cannot create production ledger evidence. + +### Termination / sandbox model + +- Patch producers run in a dedicated Bun child (`producer-child.ts`) spawned by `producer-isolate.ts` with minimal env (`TZ`, `NO_COLOR`). +- Hard `SIGKILL` on total timeout; child process cannot block the OpenCodex event loop with a synchronous infinite loop. +- Scratch containment, symlink/path traversal rejection, byte/file limits, and cleanup remain in the parent executor after the child returns a validated `SyntheticPatchV1`. + +### Failure attribution + +- Semantic verifier mismatch → `fail` / `behavioral_failure` / `route`. +- Sandbox/containment (`FabricTaskError` from scratch/verifier infrastructure) → `blocked` or `inconclusive` / `sandbox_violation` / `harness` or `environment` — never route-attributed `behavioral_failure`. +- Timeouts / budget exhaustion → `blocked` / `environment`. +- Harness defects → `inconclusive` / `harness`. + +### Inactivity accounting + +- `inactiveMs` = `completedAt - lastActivityAt` where `lastActivityAt` resets on producer `reportActivity()` (trusted child executor path). + +### Local validation (blocker fix head) - `bun x tsc --noEmit`: passed -- `bun test tests/lab-fabric-task.test.ts`: 20/20 passed -- `bun test tests/lab-read-surfaces.test.ts tests/routing-compatibility.test.ts`: passed -- `bun test tests/repo-hygiene.test.ts`: 11/11 passed +- `bun test tests/lab-fabric-task.test.ts`: 39/39 passed - `bun run privacy:scan`: passed +- `bun test tests/lab-read-surfaces.test.ts tests/routing-compatibility.test.ts tests/lab-live-probe.test.ts`: passed - CL-08: **not started** diff --git a/src/lab/artifacts/sanitize.ts b/src/lab/artifacts/sanitize.ts index afb6386286..f989bf7377 100644 --- a/src/lab/artifacts/sanitize.ts +++ b/src/lab/artifacts/sanitize.ts @@ -8,8 +8,8 @@ import { jcsStringify } from "../digest"; import { redactSecretString } from "../../lib/redact"; const FORBIDDEN_KEY = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|content|body|url|hostname|baseUrl|path|account|alias)$/i; -const SECRETISH = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/i; -const SECRETISH_GLOBAL = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/gi; +const SECRETISH = /sk-[a-z0-9]{10,}|credential-canary-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/i; +const SECRETISH_GLOBAL = /sk-[a-z0-9]{10,}|credential-canary-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/gi; export function redactForArtifact(artifactClass: ArtifactClass, payload: unknown): unknown { if ( diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts index 5e10fb436c..7eb95cb4e1 100644 --- a/src/lab/fabric/executor.ts +++ b/src/lab/fabric/executor.ts @@ -1,5 +1,8 @@ import type { FailureRecordV1, RouteSubjectV1 } from "../events/types"; import { labSandboxEnvironment, rejectProxyEnvironment } from "../live/sandbox"; +import type { LabDestinationV1, LabRouteContext } from "../live/types"; +import { buildRouteSubjectV1 } from "../subject/route-subject"; +import { isTrustedFabricPatchExecutor } from "../../lib/fabric-task-execution-authority"; import { FABRIC_COMPATIBILITY_VERSION, FABRIC_LIMITS, @@ -12,6 +15,7 @@ import { SYNTHETIC_VALUE_PATH, } from "./constants"; import { applySyntheticPatch, parseSyntheticPatchV1 } from "./patch"; +import { fabricProducerIsolationLimits, runIsolatedFabricProducer } from "./producer-isolate"; import { assertNotUnderUserRepo, createSyntheticScratch, type ScratchTree } from "./scratch"; import { buildTaskSubjectV1, @@ -21,25 +25,36 @@ import { verifierManifestDigest, } from "./subject"; import type { + FabricExecutionAuthority, + FabricHarnessProducerKind, FabricTaskOutcomeV1, + FabricTaskRunResult, FabricUsageV1, - SyntheticPatchProducer, SyntheticPatchV1, + TrustedFabricPatchExecutor, } from "./types"; import { FabricTaskError } from "./types"; import { verifyExactTreeDiffV1 } from "./verifier"; -/** Options for running the bounded synthetic-patch fabric task executor. */ -export interface RunFabricTaskOptions { +/** Options for harness-only fabric task runs (not persistable as production evidence). */ +export interface RunFabricTaskHarnessOptions { routeSubject: RouteSubjectV1; - producePatch: SyntheticPatchProducer; + harnessKind: FabricHarnessProducerKind; + configDir?: string; + userRepoRoot?: string; + sourceRefs?: string[]; + now?: () => number; +} + +/** Options for authoritative route-bound fabric task execution. */ +export interface RunFabricTaskForRouteOptions { + routeContext: LabRouteContext; + destination: LabDestinationV1; + patchExecutor: TrustedFabricPatchExecutor; configDir?: string; - /** Optional absolute user repo path used only for containment proofs in tests. */ userRepoRoot?: string; sourceRefs?: string[]; now?: () => number; - /** Override clocks for inactivity/timeout tests. */ - sleep?: (ms: number) => Promise; } /** Map a FabricTaskError into a ledger failure record. */ @@ -68,19 +83,83 @@ export function correctSyntheticPatch(): SyntheticPatchV1 { } /** - * Bounded Lab-owned fabric task executor for fabric-core.task.synthetic-patch@1.0.0. - * Does not provide a general Agent Fabric platform. + * Authoritative CL-07 route-bound fabric task execution. + * Route identity is derived from routeContext + destination; patch production runs in an isolated child. + */ +export async function runFabricSyntheticPatchTaskForRoute( + options: RunFabricTaskForRouteOptions, +): Promise { + if (!isTrustedFabricPatchExecutor(options.patchExecutor)) { + throw new FabricTaskError("untrusted fabric patch executor", "sandbox_violation", "harness"); + } + const routeSubject = buildRouteSubjectV1(options.routeContext, options.destination, options.configDir); + return runFabricSyntheticPatchTaskInternal({ + routeSubject, + routeContext: options.routeContext, + destination: options.destination, + patchExecutor: options.patchExecutor, + harnessKind: undefined, + executionAuthority: "trusted_route", + configDir: options.configDir, + userRepoRoot: options.userRepoRoot, + sourceRefs: options.sourceRefs, + now: options.now, + }); +} + +/** Harness-only fabric task execution; outcomes are not persistable as production evidence. */ +export async function runFabricSyntheticPatchTaskHarness( + options: RunFabricTaskHarnessOptions, +): Promise { + return runFabricSyntheticPatchTaskInternal({ + routeSubject: options.routeSubject, + harnessKind: options.harnessKind, + executionAuthority: "harness", + configDir: options.configDir, + userRepoRoot: options.userRepoRoot, + sourceRefs: options.sourceRefs, + now: options.now, + }); +} + +/** + * @deprecated Use runFabricSyntheticPatchTaskHarness or runFabricSyntheticPatchTaskForRoute. */ -export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions): Promise { +export async function runFabricSyntheticPatchTask( + options: RunFabricTaskHarnessOptions & { producePatch?: never; harnessKind?: FabricHarnessProducerKind }, +): Promise { + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: options.routeSubject, + harnessKind: options.harnessKind ?? "deterministic_correct", + configDir: options.configDir, + userRepoRoot: options.userRepoRoot, + sourceRefs: options.sourceRefs, + now: options.now, + }); + return result.outcome; +} + +async function runFabricSyntheticPatchTaskInternal(input: { + routeSubject: RouteSubjectV1; + routeContext?: LabRouteContext; + destination?: LabDestinationV1; + patchExecutor?: TrustedFabricPatchExecutor; + harnessKind?: FabricHarnessProducerKind; + executionAuthority: FabricExecutionAuthority; + configDir?: string; + userRepoRoot?: string; + sourceRefs?: string[]; + now?: () => number; +}): Promise { rejectProxyEnvironment(); labSandboxEnvironment(); - const startedAt = options.now?.() ?? Date.now(); + const startedAt = input.now?.() ?? Date.now(); const fixtureDigest = taskFixtureDigest(); const verifierDigest = verifierManifestDigest(); const sandboxDigest = sandboxProfileDigest(); const taskSubject = buildTaskSubjectV1({ - routeSubject: options.routeSubject, + routeSubject: input.routeSubject, taskFixtureDigest: fixtureDigest, verifierManifestDigest: verifierDigest, sandboxProfileDigest: sandboxDigest, @@ -99,84 +178,138 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) }; let scratch: ScratchTree | undefined; + let producerCompletedAt = startedAt; + let lastActivityAt = startedAt; + try { - scratch = createSyntheticScratch(options.configDir); - if (options.userRepoRoot) { - assertNotUnderUserRepo(scratch.root, options.userRepoRoot); + scratch = createSyntheticScratch(input.configDir); + if (input.userRepoRoot) { + assertNotUnderUserRepo(scratch.root, input.userRepoRoot); } - const deadline = startedAt + FABRIC_LIMITS.totalTimeoutMs; - const produceStarted = options.now?.() ?? Date.now(); + const isolation = fabricProducerIsolationLimits(); let patchRaw: unknown; try { - const controller = createTimeoutController(FABRIC_LIMITS.totalTimeoutMs, FABRIC_LIMITS.inactivityTimeoutMs, options); - const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); - patchRaw = await controller.race( - sleep(0).then(() => options.producePatch({ - taskClassId: FABRIC_TASK_CLASS_ID, - taskClassVersion: FABRIC_TASK_CLASS_VERSION, - scratchRoot: scratch!.root, - reportActivity: controller.reportActivity, - })), - ); + if (input.harnessKind) { + lastActivityAt = input.now?.() ?? Date.now(); + patchRaw = await runIsolatedFabricProducer({ + harnessKind: input.harnessKind, + scratchRoot: scratch.root, + totalTimeoutMs: isolation.totalTimeoutMs, + inactivityTimeoutMs: isolation.inactivityTimeoutMs, + }); + producerCompletedAt = input.now?.() ?? Date.now(); + lastActivityAt = producerCompletedAt; + } else if (input.patchExecutor && input.routeContext && input.destination) { + const controller = new AbortController(); + const executorInput = { + routeContext: input.routeContext, + destination: input.destination, + routeSubject: input.routeSubject, + scratchRoot: scratch.root, + reportActivity: () => { + lastActivityAt = input.now?.() ?? Date.now(); + }, + signal: controller.signal, + }; + lastActivityAt = input.now?.() ?? Date.now(); + patchRaw = await runIsolatedFabricProducer({ + executorModulePath: input.patchExecutor.executorModulePath, + scratchRoot: scratch.root, + totalTimeoutMs: isolation.totalTimeoutMs, + inactivityTimeoutMs: isolation.inactivityTimeoutMs, + executorInput, + }); + producerCompletedAt = input.now?.() ?? Date.now(); + lastActivityAt = producerCompletedAt; + controller.abort(); + } else { + throw new FabricTaskError("fabric task run missing producer", "harness_failure", "harness"); + } } catch (error) { - const completedAt = options.now?.() ?? Date.now(); + const completedAt = input.now?.() ?? Date.now(); usage.elapsedMs = completedAt - startedAt; - usage.inactiveMs = Math.max(0, completedAt - produceStarted); + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); if (error instanceof FabricTaskError) { const failure = failureFromError(error); - return sealOutcome({ - taskSubject, - subjectId, - fixtureDigest, - verifierDigest, - sandboxDigest, - startedAt, - completedAt, - usage, - outcome: outcomeFromFailure(failure), - verifier: { - verifierId: FABRIC_VERIFIER_ID, - manifestDigest: verifierDigest, - passed: false, - pathSummaries: [], - reason: error.code, - }, - failure, - sourceRefs: options.sourceRefs, - }); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: infrastructureVerifier(verifierDigest, error.code), + failure, + sourceRefs: input.sourceRefs, + }), + }; } throw error; } - if ((options.now?.() ?? Date.now()) > deadline) { - throw new FabricTaskError("total timeout exceeded", "timeout", "environment"); - } - const patch = parseSyntheticPatchV1(patchRaw); const applied = applySyntheticPatch(scratch.root, patch); usage.outputBytes = applied.bytesWritten; usage.patchOperations = applied.patchOperations; usage.filesTouched = applied.filesTouched; - const verifier = verifyExactTreeDiffV1(scratch.root); - const completedAt = options.now?.() ?? Date.now(); + let verifier; + try { + verifier = verifyExactTreeDiffV1(scratch.root); + } catch (error) { + const completedAt = input.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: infrastructureVerifier(verifierDigest, error.code), + failure, + sourceRefs: input.sourceRefs, + }), + }; + } + throw error; + } + + const completedAt = input.now?.() ?? Date.now(); usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); if (verifier.passed) { - return sealOutcome({ - taskSubject, - subjectId, - fixtureDigest, - verifierDigest, - sandboxDigest, - startedAt, - completedAt, - usage, - outcome: "pass", - verifier, - sourceRefs: options.sourceRefs, - }); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: "pass", + verifier, + sourceRefs: input.sourceRefs, + }), + }; } const failure: FailureRecordV1 = { @@ -185,26 +318,9 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) retryable: false, attribution: "route", }; - return sealOutcome({ - taskSubject, - subjectId, - fixtureDigest, - verifierDigest, - sandboxDigest, - startedAt, - completedAt, - usage, - outcome: "fail", - verifier, - failure, - sourceRefs: options.sourceRefs, - }); - } catch (error) { - const completedAt = options.now?.() ?? Date.now(); - usage.elapsedMs = completedAt - startedAt; - if (error instanceof FabricTaskError) { - const failure = failureFromError(error); - return sealOutcome({ + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ taskSubject, subjectId, fixtureDigest, @@ -213,17 +329,35 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) startedAt, completedAt, usage, - outcome: outcomeFromFailure(failure), - verifier: { - verifierId: FABRIC_VERIFIER_ID, - manifestDigest: verifierDigest, - passed: false, - pathSummaries: [], - reason: error.code, - }, + outcome: "fail", + verifier, failure, - sourceRefs: options.sourceRefs, - }); + sourceRefs: input.sourceRefs, + }), + }; + } catch (error) { + const completedAt = input.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: infrastructureVerifier(verifierDigest, error.code), + failure, + sourceRefs: input.sourceRefs, + }), + }; } const failure: FailureRecordV1 = { class: "harness_failure", @@ -231,31 +365,38 @@ export async function runFabricSyntheticPatchTask(options: RunFabricTaskOptions) retryable: false, attribution: "harness", }; - return sealOutcome({ - taskSubject, - subjectId, - fixtureDigest, - verifierDigest, - sandboxDigest, - startedAt, - completedAt, - usage, - outcome: "inconclusive", - verifier: { - verifierId: FABRIC_VERIFIER_ID, - manifestDigest: verifierDigest, - passed: false, - pathSummaries: [], - reason: "harness_failure", - }, - failure, - sourceRefs: options.sourceRefs, - }); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: "inconclusive", + verifier: infrastructureVerifier(verifierDigest, "harness_failure"), + failure, + sourceRefs: input.sourceRefs, + }), + }; } finally { scratch?.cleanup(); } } +function infrastructureVerifier(manifestDigest: string, reason: string): FabricTaskOutcomeV1["verifier"] { + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [], + reason, + }; +} + /** Assemble an immutable FabricTaskOutcomeV1 from executor state. */ function sealOutcome(input: { taskSubject: FabricTaskOutcomeV1["taskSubject"]; @@ -271,7 +412,7 @@ function sealOutcome(input: { failure?: FailureRecordV1; sourceRefs?: string[]; }): FabricTaskOutcomeV1 { - const sealed: FabricTaskOutcomeV1 = { + return { schemaVersion: 1, taskClassId: FABRIC_TASK_CLASS_ID, taskClassVersion: FABRIC_TASK_CLASS_VERSION, @@ -295,76 +436,6 @@ function sealOutcome(input: { artifactDigests: [], ...(input.sourceRefs ? { sourceRefs: [...input.sourceRefs] } : {}), }; - return sealed; -} - -/** Arm total and inactivity deadlines around producer execution. */ -function createTimeoutController( - totalMs: number, - inactivityMs: number, - options: RunFabricTaskOptions, -): { - reportActivity: () => void; - race: (promise: Promise) => Promise; -} { - const sleep = options.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); - let settled = false; - let totalTimer: ReturnType | undefined; - let inactivityTimer: ReturnType | undefined; - let rejectRef: ((error: FabricTaskError) => void) | undefined; - - const clearTimers = () => { - if (totalTimer !== undefined) clearTimeout(totalTimer); - if (inactivityTimer !== undefined) clearTimeout(inactivityTimer); - totalTimer = undefined; - inactivityTimer = undefined; - }; - - const settleReject = (error: FabricTaskError) => { - if (settled) return; - settled = true; - clearTimers(); - rejectRef?.(error); - }; - - const armInactivity = () => { - if (inactivityTimer !== undefined) clearTimeout(inactivityTimer); - inactivityTimer = setTimeout(() => { - settleReject(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); - }, inactivityMs); - }; - - return { - reportActivity: () => { - if (settled) return; - armInactivity(); - }, - race: async (promise: Promise): Promise => { - return await new Promise((resolve, reject) => { - rejectRef = reject; - totalTimer = setTimeout(() => { - settleReject(new FabricTaskError("total timeout exceeded", "timeout", "environment")); - }, totalMs); - armInactivity(); - // Keep the injected sleep seam observable for tests without discarding work. - void sleep(0).catch(() => undefined); - promise.then( - (value) => { - if (settled) return; - settled = true; - clearTimers(); - resolve(value); - }, - (error) => { - if (settled) return; - settled = true; - clearTimers(); - reject(error); - }, - ); - }); - }, - }; } /** Declared sandbox policy for the frozen scratch producer (not runtime enforcement). */ diff --git a/src/lab/fabric/index.ts b/src/lab/fabric/index.ts index 8a51e13235..1bc219a33c 100644 --- a/src/lab/fabric/index.ts +++ b/src/lab/fabric/index.ts @@ -18,6 +18,7 @@ export { export type { FabricOutcomeKind, FabricFailureClass, + FabricExecutionAuthority, FabricLimitsV1, SyntheticPatchOperationV1, SyntheticPatchV1, @@ -25,7 +26,11 @@ export type { ExactTreeDiffPathSummaryV1, ExactTreeDiffResultV1, FabricTaskOutcomeV1, - SyntheticPatchProducer, + FabricTaskRunResult, + FabricHarnessProducerKind, + FabricPatchExecutorInput, + FabricPatchExecutor, + TrustedFabricPatchExecutor, } from "./types"; export { FabricTaskError } from "./types"; export { @@ -49,10 +54,15 @@ export { verifyExactTreeDiffV1 } from "./verifier"; export { correctSyntheticPatch, runFabricSyntheticPatchTask, + runFabricSyntheticPatchTaskForRoute, + runFabricSyntheticPatchTaskHarness, fabricDeclaredSandboxPolicy, fabricScratchCapabilityProbe, } from "./executor"; -export type { RunFabricTaskOptions } from "./executor"; +export type { + RunFabricTaskHarnessOptions, + RunFabricTaskForRouteOptions, +} from "./executor"; export { loadFabricCaseAuthority, discoverFabricScenarios, @@ -66,5 +76,6 @@ export { assertFabricOutcomeV1, observationFromFabricOutcome, persistFabricOutcome, + persistFabricRunResult, } from "./observe"; export type { PersistFabricOptions, PersistedFabricObservation } from "./observe"; diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 5d8ee49ff1..c3d28b1879 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -30,7 +30,7 @@ import { fabricSuiteManifestDigest, loadFabricCaseAuthority, } from "./manifest"; -import type { FabricLimitsV1, FabricTaskOutcomeV1 } from "./types"; +import type { FabricLimitsV1, FabricTaskOutcomeV1, FabricTaskRunResult } from "./types"; import { FabricTaskError } from "./types"; /** CL-07 fabric task outcome validation and observation persistence. */ @@ -437,3 +437,18 @@ export function persistFabricOutcome( if (ownsStore) store.close(); } } + +/** Persist production fabric evidence from a trusted-route run result. */ +export function persistFabricRunResult( + result: FabricTaskRunResult, + opts: PersistFabricOptions = {}, +): PersistedFabricObservation { + if (result.executionAuthority !== "trusted_route") { + throw new FabricTaskError( + "fabric evidence requires trusted route execution authority", + "malformed_producer_outcome", + "harness", + ); + } + return persistFabricOutcome(result.outcome, opts); +} diff --git a/src/lab/fabric/producer-child.ts b/src/lab/fabric/producer-child.ts new file mode 100644 index 0000000000..7dbe4511dd --- /dev/null +++ b/src/lab/fabric/producer-child.ts @@ -0,0 +1,115 @@ +import { + FABRIC_LIMITS, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, +} from "./constants"; +import type { FabricHarnessProducerKind, SyntheticPatchV1 } from "./types"; + +interface ChildRequest { + harnessKind?: FabricHarnessProducerKind; + executorModulePath?: string; + scratchRoot: string; + totalTimeoutMs: number; + inactivityTimeoutMs: number; + executorInput?: unknown; +} + +interface ChildResponse { + ok: boolean; + patch?: SyntheticPatchV1; + error?: { message: string; code: string; attribution: string }; +} + +function correctPatch(): SyntheticPatchV1 { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} + +function wrongPatch(): SyntheticPatchV1 { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: "wrong\n" }], + }; +} + +async function runHarness(kind: FabricHarnessProducerKind, scratchRoot: string): Promise { + switch (kind) { + case "deterministic_correct": + return correctPatch(); + case "deterministic_wrong": + return wrongPatch(); + case "infinite_sync": + while (true) { + /* hang until killed */ + } + case "never_resolve": + await new Promise(() => undefined); + throw new Error("never_resolve unexpectedly completed"); + case "mutate_after_delay": + await Bun.sleep(FABRIC_LIMITS.totalTimeoutMs + 5_000); + const { writeFileSync, mkdirSync } = await import("node:fs"); + const { join, dirname } = await import("node:path"); + mkdirSync(join(scratchRoot, dirname(SYNTHETIC_VALUE_PATH)), { recursive: true }); + writeFileSync(join(scratchRoot, SYNTHETIC_VALUE_PATH), "late\n"); + return correctPatch(); + default: + throw new Error(`unsupported harness kind: ${kind as string}`); + } +} + +async function runExecutorModule( + modulePath: string, + executorInput: unknown, + inactivityTimeoutMs: number, +): Promise { + const mod = await import(modulePath) as { execute: (input: unknown) => Promise | SyntheticPatchV1 }; + if (typeof mod.execute !== "function") throw new Error("executor module missing execute export"); + let lastActivity = Date.now(); + const input = { + ...(executorInput as Record), + reportActivity: () => { + lastActivity = Date.now(); + }, + }; + const inactivity = setInterval(() => { + if (Date.now() - lastActivity > inactivityTimeoutMs) { + throw new Error("inactivity_timeout"); + } + }, 50); + try { + return await Promise.resolve(mod.execute(input)); + } finally { + clearInterval(inactivity); + } +} + +async function main(): Promise { + const raw = await Bun.stdin.text(); + const req = JSON.parse(raw) as ChildRequest; + let patch: SyntheticPatchV1; + if (req.harnessKind) { + patch = await runHarness(req.harnessKind, req.scratchRoot); + } else if (req.executorModulePath) { + patch = await runExecutorModule(req.executorModulePath, req.executorInput, req.inactivityTimeoutMs); + } else { + throw new Error("child request missing harnessKind or executorModulePath"); + } + const response: ChildResponse = { ok: true, patch }; + process.stdout.write(JSON.stringify(response)); +} + +main().catch((error: unknown) => { + const response: ChildResponse = { + ok: false, + error: { + message: error instanceof Error ? error.message : String(error), + code: error instanceof Error && error.message === "inactivity_timeout" ? "inactivity_timeout" : "harness_failure", + attribution: "harness", + }, + }; + process.stdout.write(JSON.stringify(response)); + process.exit(1); +}); diff --git a/src/lab/fabric/producer-isolate.ts b/src/lab/fabric/producer-isolate.ts new file mode 100644 index 0000000000..842921f0de --- /dev/null +++ b/src/lab/fabric/producer-isolate.ts @@ -0,0 +1,130 @@ +import { spawn } from "node:child_process"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { FABRIC_LIMITS } from "./constants"; +import type { FabricHarnessProducerKind, FabricPatchExecutorInput, SyntheticPatchV1 } from "./types"; +import { FabricTaskError } from "./types"; + +const CHILD_ENTRY = join(dirname(fileURLToPath(import.meta.url)), "producer-child.ts"); + +interface IsolateRequest { + harnessKind?: FabricHarnessProducerKind; + executorModulePath?: string; + scratchRoot: string; + totalTimeoutMs: number; + inactivityTimeoutMs: number; + executorInput?: FabricPatchExecutorInput; +} + +interface IsolateResponse { + ok: boolean; + patch?: SyntheticPatchV1; + error?: { message: string; code: string; attribution: string }; +} + +function minimalChildEnv(): Record { + return { TZ: "UTC", NO_COLOR: "1" }; +} + +/** Run a fabric patch producer in an isolated child process with hard termination. */ +export async function runIsolatedFabricProducer(request: IsolateRequest): Promise { + return await new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["run", CHILD_ENTRY], { + env: minimalChildEnv(), + stdio: ["pipe", "pipe", "pipe"], + }); + + let stdout = ""; + let settled = false; + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(totalTimer); + fn(); + }; + + const totalTimer = setTimeout(() => { + try { + child.kill("SIGKILL"); + } catch { + /* ignore */ + } + finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); + }, request.totalTimeoutMs); + + child.stdout.on("data", (chunk: Buffer | string) => { + stdout += chunk.toString("utf8"); + }); + + child.stderr.on("data", () => { + /* discard */ + }); + + child.on("error", (error) => { + finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + }); + + child.on("close", (code, signal) => { + if (settled) return; + if (signal === "SIGKILL") { + finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); + return; + } + try { + const trimmed = stdout.trim(); + if (!trimmed) { + finish(() => reject(new FabricTaskError("isolated producer returned no output", "harness_failure", "harness"))); + return; + } + const parsed = JSON.parse(trimmed) as IsolateResponse; + if (!parsed.ok || !parsed.patch) { + const code = parsed.error?.code ?? "harness_failure"; + const attribution = parsed.error?.attribution === "environment" ? "environment" : "harness"; + const fabricCode = code === "inactivity_timeout" + ? "inactivity_timeout" + : code === "timeout" + ? "timeout" + : code === "sandbox_violation" + ? "sandbox_violation" + : "harness_failure"; + finish(() => reject(new FabricTaskError(parsed.error?.message ?? "isolated producer failed", fabricCode, attribution))); + return; + } + finish(() => resolve(parsed.patch!)); + } catch (error) { + finish(() => reject(new FabricTaskError( + error instanceof Error ? error.message : String(error), + "harness_failure", + "harness", + ))); + } + void code; + }); + + const payload = JSON.stringify({ + harnessKind: request.harnessKind, + executorModulePath: request.executorModulePath, + scratchRoot: request.scratchRoot, + totalTimeoutMs: request.totalTimeoutMs, + inactivityTimeoutMs: request.inactivityTimeoutMs, + executorInput: request.executorInput + ? { + routeContext: request.executorInput.routeContext, + destination: request.executorInput.destination, + routeSubject: request.executorInput.routeSubject, + scratchRoot: request.executorInput.scratchRoot, + } + : undefined, + }); + child.stdin.write(payload); + child.stdin.end(); + }); +} + +/** Default isolation limits for fabric producer child processes. */ +export function fabricProducerIsolationLimits(): { totalTimeoutMs: number; inactivityTimeoutMs: number } { + return { + totalTimeoutMs: FABRIC_LIMITS.totalTimeoutMs, + inactivityTimeoutMs: FABRIC_LIMITS.inactivityTimeoutMs, + }; +} diff --git a/src/lab/fabric/types.ts b/src/lab/fabric/types.ts index 48bfec1116..e1f72c8af8 100644 --- a/src/lab/fabric/types.ts +++ b/src/lab/fabric/types.ts @@ -1,11 +1,14 @@ import type { FailureClassification } from "../conformance/types"; import type { ObservationOutcome } from "../constants"; import type { FailureRecordV1, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; +import type { LabDestinationV1, LabRouteContext } from "../live/types"; export type FabricOutcomeKind = ObservationOutcome; export type FabricFailureClass = FailureClassification; +export type FabricExecutionAuthority = "trusted_route" | "harness"; + export interface FabricLimitsV1 { maxFiles: number; maxScratchFiles: number; @@ -76,11 +79,42 @@ export interface FabricTaskOutcomeV1 { sourceRefs?: string[]; } +/** Authoritative route execution input for trusted fabric patch producers. */ +export interface FabricPatchExecutorInput { + routeContext: LabRouteContext; + destination: LabDestinationV1; + routeSubject: RouteSubjectV1; + scratchRoot: string; + reportActivity: () => void; + signal: AbortSignal; +} + +export type FabricPatchExecutor = (input: FabricPatchExecutorInput) => SyntheticPatchV1 | Promise; + +/** Opaque host-issued capability for exact-route fabric patch production. */ +export interface TrustedFabricPatchExecutor { + execute(input: FabricPatchExecutorInput): Promise; + readonly executorModulePath: string; +} + +/** Sealed fabric run result including evidence authority (not part of outcome JSON). */ +export interface FabricTaskRunResult { + outcome: FabricTaskOutcomeV1; + executionAuthority: FabricExecutionAuthority; +} + +export type FabricHarnessProducerKind = + | "deterministic_correct" + | "deterministic_wrong" + | "infinite_sync" + | "never_resolve" + | "mutate_after_delay"; + +/** @deprecated Test-only harness producer; use FabricHarnessProducerKind via runFabricSyntheticPatchTaskHarness. */ export type SyntheticPatchProducer = (ctx: { taskClassId: string; taskClassVersion: string; scratchRoot: string; - /** Reset the inactivity deadline while the producer is making progress. */ reportActivity: () => void; }) => SyntheticPatchV1 | Promise; diff --git a/src/lab/fabric/verifier.ts b/src/lab/fabric/verifier.ts index de7a993e15..8ae70fee9d 100644 --- a/src/lab/fabric/verifier.ts +++ b/src/lab/fabric/verifier.ts @@ -98,17 +98,7 @@ export function verifyExactTreeDiffV1(scratchRoot: string): ExactTreeDiffResultV }; } catch (error) { if (error instanceof FabricTaskError) { - return { - verifierId: FABRIC_VERIFIER_ID, - manifestDigest, - passed: false, - pathSummaries: [{ - path: SYNTHETIC_VALUE_PATH, - kind: "rejected", - reason: error.code, - }], - reason: error.code, - }; + throw error; } throw error; } diff --git a/src/lib/fabric-task-execution-authority.ts b/src/lib/fabric-task-execution-authority.ts new file mode 100644 index 0000000000..0de6d5bacf --- /dev/null +++ b/src/lib/fabric-task-execution-authority.ts @@ -0,0 +1,7 @@ +import type { TrustedFabricPatchExecutor } from "../lab/fabric/types"; +import { isHostIssuedFabricPatchExecutor } from "./fabric-task-host"; + +/** Trusted exact-route fabric patch executor recognition facade. */ +export function isTrustedFabricPatchExecutor(value: unknown): value is TrustedFabricPatchExecutor { + return isHostIssuedFabricPatchExecutor(value); +} diff --git a/src/lib/fabric-task-host.ts b/src/lib/fabric-task-host.ts new file mode 100644 index 0000000000..15ff341fae --- /dev/null +++ b/src/lib/fabric-task-host.ts @@ -0,0 +1,29 @@ +import type { FabricPatchExecutor, FabricPatchExecutorInput, TrustedFabricPatchExecutor } from "../lab/fabric/types"; + +const HOST_ISSUED_EXECUTORS = new WeakSet(); + +/** + * Trusted host integration boundary for CL-07 exact-route fabric patch production. + * + * @internal host integration only + */ +export function createHostIssuedFabricPatchExecutor( + executorModulePath: string, + execute: FabricPatchExecutor, +): TrustedFabricPatchExecutor { + const capability: TrustedFabricPatchExecutor = Object.freeze({ + executorModulePath, + execute: (input: FabricPatchExecutorInput) => Promise.resolve(execute(input)), + }); + HOST_ISSUED_EXECUTORS.add(capability); + return capability; +} + +/** Internal recognition check consumed by the read-only authority facade. */ +export function isHostIssuedFabricPatchExecutor(value: unknown): value is TrustedFabricPatchExecutor { + if (typeof value !== "object" || value === null || !HOST_ISSUED_EXECUTORS.has(value as object)) return false; + const capability = value as TrustedFabricPatchExecutor; + return typeof capability.executorModulePath === "string" + && capability.executorModulePath.length > 0 + && typeof capability.execute === "function"; +} diff --git a/tests/fixtures/fabric-executors/correct-patch.ts b/tests/fixtures/fabric-executors/correct-patch.ts new file mode 100644 index 0000000000..5d5be44407 --- /dev/null +++ b/tests/fixtures/fabric-executors/correct-patch.ts @@ -0,0 +1,9 @@ +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "../../../src/lab/fabric/types"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "../../../src/lab/fabric/constants"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} diff --git a/tests/helpers/fabric-task-test.ts b/tests/helpers/fabric-task-test.ts new file mode 100644 index 0000000000..0a8e175869 --- /dev/null +++ b/tests/helpers/fabric-task-test.ts @@ -0,0 +1,145 @@ +import { writeFileSync, mkdirSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createLabDestination } from "../../src/lab"; +import { runFabricSyntheticPatchTaskForRoute } from "../../src/lab/fabric/executor"; +import type { FabricTaskRunResult } from "../../src/lab/fabric/types"; +import { createHostIssuedFabricPatchExecutor } from "../../src/lib/fabric-task-host"; +import type { LabBehaviorValues, LabRouteContext } from "../../src/lab/live/types"; +import type { TrustedFabricPatchExecutor } from "../../src/lab/fabric/types"; +import { correctSyntheticPatch } from "../../src/lab/fabric/executor"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const FIXTURE_DIR = join(REPO_ROOT, "tests", "fixtures", "fabric-executors"); + +function repoImport(subpath: string): string { + return join(REPO_ROOT, subpath).replace(/\\/g, "/"); +} + +export function fabricBehavior(adapter: string, upstreamProtocol: string): LabBehaviorValues { + return { + "wire.adapter": { source: "lab_forced", value: adapter }, + "wire.upstreamProtocol": { source: "lab_forced", value: upstreamProtocol }, + "auth.mode": { source: "provider_config", value: "api_key" }, + "auth.transport": { source: "provider_config", value: "authorization_bearer" }, + "mcp.nativeLocalExec": { source: "lab_forced", value: false }, + "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, + "runtime.platform": { source: "lab_forced", value: process.platform }, + "runtime.arch": { source: "lab_forced", value: process.arch }, + "runtime.streamMode": { source: "lab_forced", value: "auto" }, + "runtime.fastMode": { source: "lab_forced", value: false }, + "runtime.effortCap": { source: "lab_forced", value: null }, + "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) }, + }; +} + +export function fabricMockRoute(overrides: Partial = {}): LabRouteContext { + const base = { + providerId: "provider-a", + providerInstanceKey: "fixture-provider-instance", + clientModelId: "model-a", + upstreamModelId: "model-a", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + baseUrl: "https://api.example.com/v1", + opencodexCompatibilityVersion: "a".repeat(64), + labRunApproval: true, + allowPrivateNetwork: false, + requiredClaims: [], + availableHarnessFeatures: ["fabric-scratch-v1"], + }; + const merged = { ...base, ...overrides }; + return { + ...merged, + behaviorValues: overrides.behaviorValues ?? fabricBehavior(merged.effectiveAdapter, merged.upstreamProtocol), + }; +} + +export function fabricCorrectPatchExecutor(): TrustedFabricPatchExecutor { + const modulePath = join(FIXTURE_DIR, "correct-patch.ts"); + return createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()); +} + +export function fabricRouteBoundPatchExecutor(expectedProviderId: string): TrustedFabricPatchExecutor { + const dir = join(FIXTURE_DIR, "generated"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, `route-bound-${expectedProviderId}.ts`); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +const expectedProviderId = "${expectedProviderId}"; + +export async function execute(input: FabricPatchExecutorInput): Promise { + if (input.routeSubject.providerId !== expectedProviderId) { + throw new Error("route subject mismatch"); + } + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async (input) => { + if (input.routeSubject.providerId !== expectedProviderId) { + throw new Error("route subject mismatch"); + } + return correctSyntheticPatch(); + }); +} + +export function fabricWrongPatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "wrong-patch.ts"); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + return { schemaVersion: 1, operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: "wrong\\n" }] }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => ({ + schemaVersion: 1, + operations: [{ op: "replace", path: "src/value.txt", contentUtf8: "wrong\n" }], + })); +} + +export function fabricOversizedPatchExecutor(home: string, huge: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "oversized-patch.ts"); + writeFileSync(modulePath, `import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; +const huge = ${JSON.stringify(huge)}; +export async function execute(_input: FabricPatchExecutorInput): Promise { + return { schemaVersion: 1, operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: huge }] }; +}`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => ({ + schemaVersion: 1, + operations: [{ op: "replace", path: "src/value.txt", contentUtf8: huge }], + })); +} + +export async function runTrustedFabricTask( + home: string, + overrides: Partial = {}, + now?: () => number, +): Promise { + const destination = await createLabDestination({ + baseUrl: overrides.baseUrl ?? "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + return runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(overrides), + destination, + patchExecutor: fabricCorrectPatchExecutor(), + configDir: home, + now, + }); +} diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index e28150ecc2..96e5cc0a7d 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -10,7 +10,8 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { appendLabEvent, assertNotUnderUserRepo, @@ -18,6 +19,7 @@ import { assignEventId, buildTaskSubjectV1, correctSyntheticPatch, + createLabDestination, createSyntheticScratch, fabricDeclaredSandboxPolicy, FABRIC_VERIFIER_ID, @@ -25,11 +27,13 @@ import { LAB_PRODUCER, observationFromFabricOutcome, persistFabricOutcome, + persistFabricRunResult, queryLabCatalog, readVerdictSnapshot, rebuildLabProjection, replayLabLedger, - runFabricSyntheticPatchTask, + runFabricSyntheticPatchTaskForRoute, + runFabricSyntheticPatchTaskHarness, sandboxProfileDigest, subjectIdForSubject, SYNTHETIC_AFTER_UTF8, @@ -52,6 +56,17 @@ import { verifyExactTreeDiffV1 } from "../src/lab/fabric/verifier"; import { parseSyntheticPatchV1 } from "../src/lab/fabric/patch"; import { FABRIC_LIMITS } from "../src/lab/fabric/constants"; import { taskSubjectApplicableToRequirements } from "../src/lab/projection/verification"; +import { createHostIssuedFabricPatchExecutor } from "../src/lib/fabric-task-host"; +import type { TrustedFabricPatchExecutor } from "../src/lab/fabric/types"; +import { + fabricCorrectPatchExecutor, + fabricMockRoute, + fabricOversizedPatchExecutor, + runTrustedFabricTask, +} from "./helpers/fabric-task-test"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const CREDENTIAL_CANARY = "credential-canary-abcdefghijklmnopqrstuvwxyz1234567890"; const HOMES: string[] = []; @@ -62,6 +77,105 @@ function tempHome(): string { return dir; } +function repoImport(subpath: string): string { + return join(REPO_ROOT, subpath).replace(/\\/g, "/"); +} + +async function fabricDestination(home: string) { + return createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); +} + +function fabricActivityPatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "activity-patch.ts"); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +export async function execute(input: FabricPatchExecutorInput): Promise { + for (let i = 0; i < 3; i++) { + input.reportActivity(); + await Bun.sleep(Math.floor(${FABRIC_LIMITS.inactivityTimeoutMs} * 0.6)); + } + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async (input) => { + for (let i = 0; i < 3; i++) { + input.reportActivity(); + await Bun.sleep(Math.floor(FABRIC_LIMITS.inactivityTimeoutMs * 0.6)); + } + return correctSyntheticPatch(); + }); +} + +function fabricInactivePatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "inactive-patch.ts"); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + await Bun.sleep(${FABRIC_LIMITS.inactivityTimeoutMs} + 50); + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => { + await Bun.sleep(FABRIC_LIMITS.inactivityTimeoutMs + 50); + return correctSyntheticPatch(); + }); +} + +function fabricSymlinkSandboxExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "symlink-sandbox.ts"); + writeFileSync(modulePath, ` +import { symlinkSync } from "node:fs"; +import { join } from "node:path"; +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +export async function execute(input: FabricPatchExecutorInput): Promise { + const target = join(input.scratchRoot, "src", "value.txt"); + const link = join(input.scratchRoot, "src", "link.txt"); + try { + symlinkSync(target, link); + } catch { + /* platform may reject symlink creation */ + } + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async (input) => { + const target = join(input.scratchRoot, "src", "value.txt"); + const link = join(input.scratchRoot, "src", "link.txt"); + try { + symlinkSync(target, link); + } catch { + /* platform may reject symlink creation */ + } + return correctSyntheticPatch(); + }); +} + function linkDirectory(target: string, linkPath: string): boolean { try { symlinkSync(target, linkPath, process.platform === "win32" ? "junction" : "dir"); @@ -143,16 +257,48 @@ describe("CL-07 task effectiveness producer", () => { expect(sandboxProfileDigest()).toMatch(/^[0-9a-f]{64}$/); }); - test("exact synthetic patch passes", async () => { + test("exact synthetic patch passes via trusted route", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; - const outcome = await runFabricSyntheticPatchTask({ + const result = await runTrustedFabricTask(home); + expect(result.executionAuthority).toBe("trusted_route"); + expect(result.outcome.outcome).toBe("pass"); + expect(result.outcome.verifier.passed).toBe(true); + }); + + test("deterministic_correct harness passes without trusted authority", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ routeSubject: routeSubject(), + harnessKind: "deterministic_correct", configDir: home, - producePatch: () => correctSyntheticPatch(), }); - expect(outcome.outcome).toBe("pass"); - expect(outcome.verifier.passed).toBe(true); + expect(result.executionAuthority).toBe("harness"); + expect(result.outcome.outcome).toBe("pass"); + }); + + test("harness execution cannot be persisted as production evidence", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "deterministic_correct", + configDir: home, + }); + expect(() => persistFabricRunResult(result, { configDir: home })).toThrow(FabricTaskError); + try { + persistFabricRunResult(result, { configDir: home }); + } catch (error) { + expect((error as FabricTaskError).code).toBe("malformed_producer_outcome"); + } + }); + + test("trusted route execution can be persisted as production evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runTrustedFabricTask(home); + const persisted = persistFabricRunResult(result, { configDir: home }); + expect(existsSync(join(home, "lab", "compatibility.jsonl"))).toBe(true); + expect(persisted.event.evidenceLayer).toBe("task_effectiveness"); }); test("unchanged / wrong / added / deleted trees fail verifier", () => { @@ -257,50 +403,137 @@ describe("CL-07 task effectiveness producer", () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; const huge = "x".repeat(FABRIC_LIMITS.maxAggregateIoBytes + 8); - const outcome = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricOversizedPatchExecutor(home, huge), configDir: home, - producePatch: () => ({ - schemaVersion: 1, - operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: huge }], - }), }); - expect(outcome.outcome).not.toBe("pass"); - expect(outcome.failure?.code).toBe("budget_exhausted"); + expect(result.outcome.outcome).not.toBe("pass"); + expect(result.outcome.failure?.code).toBe("budget_exhausted"); }); - test("execution and inactivity timeouts are bounded", async () => { + test("never_resolve harness terminates within total budget", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "never_resolve", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); + }, 40_000); + + test("infinite_sync harness terminates within total budget", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "infinite_sync", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); + }, 40_000); + + test("mutate_after_delay harness kills producer and cleans scratch", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; - const outcome = await runFabricSyntheticPatchTask({ + const result = await runFabricSyntheticPatchTaskHarness({ routeSubject: routeSubject(), + harnessKind: "mutate_after_delay", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); + const scratchBase = join(home, "lab", "scratch"); + await Bun.sleep(6_000); + if (existsSync(scratchBase)) { + expect(readdirSync(scratchBase).some((name) => name.startsWith("fabric-"))).toBe(false); + } + for (const name of existsSync(scratchBase) ? readdirSync(scratchBase) : []) { + if (!name.startsWith("fabric-")) continue; + const latePath = join(scratchBase, name, SYNTHETIC_VALUE_PATH); + if (existsSync(latePath)) { + expect(readFileSync(latePath, "utf8")).not.toBe("late\n"); + } + } + }, 45_000); + + test("inactivity timeout is bounded for trusted route executors", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricInactivePatchExecutor(home), configDir: home, - producePatch: async () => { - await Bun.sleep(FABRIC_LIMITS.inactivityTimeoutMs + 50); - return correctSyntheticPatch(); - }, }); - expect(["blocked", "inconclusive"]).toContain(outcome.outcome); - expect(["timeout", "inactivity_timeout"]).toContain(outcome.failure?.code ?? ""); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); }, 20_000); test("activity resets inactivity deadline within total budget", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; - const outcome = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricActivityPatchExecutor(home), configDir: home, - producePatch: async ({ reportActivity }) => { - for (let i = 0; i < 3; i++) { - reportActivity(); - await Bun.sleep(Math.floor(FABRIC_LIMITS.inactivityTimeoutMs * 0.6)); - } - return correctSyntheticPatch(); - }, }); - expect(outcome.outcome).toBe("pass"); + expect(result.outcome.outcome).toBe("pass"); }, 30_000); + test("inactiveMs on success is meaningful with controlled clock", async () => { + const home = tempHome(); + let tick = 10_000; + const now = () => tick; + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "deterministic_correct", + configDir: home, + now, + }); + expect(result.outcome.outcome).toBe("pass"); + expect(result.outcome.usage.inactiveMs).toBeGreaterThanOrEqual(0); + expect(result.outcome.usage.inactiveMs).toBeLessThanOrEqual(result.outcome.usage.elapsedMs); + tick += 2_500; + const trusted = await runTrustedFabricTask(home, {}, now); + expect(trusted.outcome.outcome).toBe("pass"); + expect(trusted.outcome.usage.inactiveMs).toBeGreaterThanOrEqual(0); + expect(trusted.outcome.usage.inactiveMs).toBeLessThanOrEqual(trusted.outcome.usage.elapsedMs); + }); + + test("sandbox violations do not become behavioral_failure", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricSymlinkSandboxExecutor(home), + configDir: home, + }); + if (result.outcome.failure?.code === "sandbox_violation") { + expect(result.outcome.failure.class).not.toBe("behavioral_failure"); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + return; + } + const scratch = createSyntheticScratch(home); + try { + symlinkSync(join(scratch.root, "src", "value.txt"), join(scratch.root, "src", "link.txt")); + expect(() => verifyExactTreeDiffV1(scratch.root)).toThrow(FabricTaskError); + try { + verifyExactTreeDiffV1(scratch.root); + } catch (error) { + expect((error as FabricTaskError).code).toBe("sandbox_violation"); + } + } catch { + /* symlink unsupported on this platform */ + } finally { + scratch.cleanup(); + } + }); + test("declared sandbox policy denies network MCP shell and user repo", () => { expect(fabricDeclaredSandboxPolicy()).toEqual({ network: false, @@ -430,11 +663,7 @@ describe("CL-07 task effectiveness producer", () => { test("malformed nested outcome fields throw FabricTaskError", async () => { const home = tempHome(); - const base = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), - configDir: home, - producePatch: () => correctSyntheticPatch(), - }); + const base = (await runTrustedFabricTask(home)).outcome; expect(() => observationFromFabricOutcome({ ...base, verifier: {} })).toThrow(FabricTaskError); try { observationFromFabricOutcome({ @@ -462,33 +691,28 @@ describe("CL-07 task effectiveness producer", () => { const home = tempHome(); const paths = ensureLabDirs(home); const store = createArtifactStore(paths.artifactsDir); - const base = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), - configDir: home, - producePatch: () => correctSyntheticPatch(), - }); - const secret = "sk-abcdefghijklmnopqrstuvwxyz1234567890"; + const base = (await runTrustedFabricTask(home)).outcome; try { const { artifacts } = observationFromFabricOutcome({ ...base, verifier: { ...base.verifier, - reason: `failed ${secret}`, + reason: `failed ${CREDENTIAL_CANARY}`, }, }, { configDir: home, artifactStore: store }); const summaryRef = artifacts.find((row) => row.artifactClass === "verifier_summary"); expect(summaryRef).toBeDefined(); const text = new TextDecoder().decode(store.get(summaryRef!.digest)); - expect(text.includes(secret)).toBe(false); + expect(text.includes(CREDENTIAL_CANARY)).toBe(false); persistFabricOutcome({ ...base, verifier: { ...base.verifier, - reason: `failed ${secret}`, + reason: `failed ${CREDENTIAL_CANARY}`, }, }, { configDir: home }); const ledger = readFileSync(join(home, "lab", "compatibility.jsonl"), "utf8"); - expect(ledger.includes(secret)).toBe(false); + expect(ledger.includes(CREDENTIAL_CANARY)).toBe(false); } finally { store.close(); } @@ -504,12 +728,8 @@ describe("CL-07 task effectiveness producer", () => { createdAt: Date.now() - 120_000, token: "stale-lock-token", }), { mode: 0o600 }); - const outcome = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), - configDir: home, - producePatch: () => correctSyntheticPatch(), - }); - persistFabricOutcome(outcome, { configDir: home }); + const result = await runTrustedFabricTask(home); + persistFabricRunResult(result, { configDir: home }); expect(existsSync(join(home, "lab", "compatibility.jsonl"))).toBe(true); }); @@ -568,45 +788,31 @@ describe("CL-07 task effectiveness producer", () => { test("duplicate outcome delivery is idempotent; distinct attempts remain distinct", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; - const outcome = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), - configDir: home, - producePatch: () => correctSyntheticPatch(), - now: () => 1_000, - }); - const first = persistFabricOutcome(outcome, { configDir: home, recordedAt: 1_000, attempt: 1 }); - const second = persistFabricOutcome(outcome, { configDir: home, recordedAt: 1_000, attempt: 1 }); + const firstRun = await runTrustedFabricTask(home, {}, () => 1_000); + const first = persistFabricRunResult(firstRun, { configDir: home, recordedAt: 1_000, attempt: 1 }); + const second = persistFabricRunResult(firstRun, { configDir: home, recordedAt: 1_000, attempt: 1 }); expect(second.event.eventId).toBe(first.event.eventId); const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); const ids = replay.events.filter((row) => row.eventKind === "observation").map((row) => row.eventId); expect(ids).toEqual([first.event.eventId]); expect(replay.corruptions.filter((row) => row.kind === "duplicate_event")).toEqual([]); - const secondAttempt = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), - configDir: home, - producePatch: () => correctSyntheticPatch(), - now: () => 2_000, - }); - const third = persistFabricOutcome(secondAttempt, { configDir: home, recordedAt: 2_000, attempt: 2 }); + const secondAttempt = await runTrustedFabricTask(home, {}, () => 2_000); + const third = persistFabricRunResult(secondAttempt, { configDir: home, recordedAt: 2_000, attempt: 2 }); expect(third.event.eventId).not.toBe(first.event.eventId); }); test("structured result converts to fabric Lab observation", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; - const outcome = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), - configDir: home, - producePatch: () => correctSyntheticPatch(), - }); + const outcome = (await runTrustedFabricTask(home)).outcome; const { event } = observationFromFabricOutcome(outcome, { configDir: home }); expect(event.evidenceLayer).toBe("task_effectiveness"); expect(event.executionMode).toBe("fabric"); expect(event.subject.subjectKind).toBe("task"); expect(event.suiteId).toBe("fabric-core"); const serialized = JSON.stringify(event); - expect(serialized.includes("sk-")).toBe(false); + expect(serialized.includes(CREDENTIAL_CANARY)).toBe(false); expect(serialized.includes("OPENAI_API_KEY")).toBe(false); }); @@ -648,13 +854,8 @@ describe("CL-07 task effectiveness producer", () => { test("projection rebuild and invalidation work for task evidence", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; - const outcome = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), - configDir: home, - producePatch: () => correctSyntheticPatch(), - now: () => 5_000, - }); - const persisted = persistFabricOutcome(outcome, { configDir: home, recordedAt: 5_000 }); + const result = await runTrustedFabricTask(home, {}, () => 5_000); + const persisted = persistFabricRunResult(result, { configDir: home, recordedAt: 5_000 }); const rebuilt = rebuildLabProjection(home); expect(rebuilt.events).toBeGreaterThan(0); expect(rebuilt.corruptions).toEqual([]); @@ -684,11 +885,7 @@ describe("CL-07 task effectiveness producer", () => { test("artifacts are bounded and catalog exposes task evidence", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; - const outcome = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), - configDir: home, - producePatch: () => correctSyntheticPatch(), - }); + const outcome = (await runTrustedFabricTask(home)).outcome; const { event, artifacts } = observationFromFabricOutcome(outcome, { configDir: home }); expect(artifacts.length).toBeGreaterThan(0); expect(artifacts.every((row) => row.byteCount <= 256 * 1024)).toBe(true); @@ -718,15 +915,16 @@ describe("CL-07 task effectiveness producer", () => { test("ledger lines omit prompts and credentials", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; - const outcome = await runFabricSyntheticPatchTask({ - routeSubject: routeSubject(), + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricCorrectPatchExecutor(), configDir: home, - producePatch: () => correctSyntheticPatch(), sourceRefs: ["routeDecision:abcd1234"], }); - persistFabricOutcome(outcome, { configDir: home }); + persistFabricRunResult(result, { configDir: home }); const text = readFileSync(join(home, "lab", "compatibility.jsonl"), "utf8"); expect(text.includes("system prompt")).toBe(false); - expect(text.includes("sk-")).toBe(false); + expect(text.includes(CREDENTIAL_CANARY)).toBe(false); }); }); From d17b7172aa0f30cdedf347e5325b3a6bf296d925 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:21:40 +0200 Subject: [PATCH 13/24] test(lab): expect infrastructure throws only for sandbox verifier cases --- tests/lab-fabric-task.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index 96e5cc0a7d..cc06772098 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -358,7 +358,7 @@ describe("CL-07 task effectiveness producer", () => { } catch { return; } - expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(() => verifyExactTreeDiffV1(scratch.root)).toThrow(FabricTaskError); } finally { scratch.cleanup(); } @@ -372,7 +372,7 @@ describe("CL-07 task effectiveness producer", () => { const fifo = join(scratch.root, "src", "fifo"); const created = Bun.spawnSync(["mkfifo", fifo], { stdout: "pipe", stderr: "pipe" }); if (created.exitCode !== 0) return; - expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(() => verifyExactTreeDiffV1(scratch.root)).toThrow(FabricTaskError); } finally { scratch.cleanup(); } From 0a10d0b5c9c66123c08f2518422d5904705cada4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:41:00 +0200 Subject: [PATCH 14/24] fix(lab): close CL-07 producer isolation and persistence gaps Seal production task-effectiveness ingestion behind persistFabricRunResult, add parent-owned NDJSON child IPC with bounded protocol output, and move timeout authority to the parent with deterministic classification tests. --- .../007_cl07_task_effectiveness.md | 31 ++- src/lab/fabric/executor.ts | 72 ++++-- src/lab/fabric/index.ts | 1 - src/lab/fabric/observe.ts | 2 +- src/lab/fabric/patch.ts | 8 +- src/lab/fabric/producer-child.ts | 96 ++++---- src/lab/fabric/producer-isolate.ts | 233 +++++++++++++----- src/lab/fabric/producer-protocol.ts | 58 +++++ src/lab/fabric/types.ts | 5 +- tests/helpers/fabric-task-test.ts | 4 +- tests/lab-fabric-task.test.ts | 168 ++++++++++++- 11 files changed, 527 insertions(+), 151 deletions(-) create mode 100644 src/lab/fabric/producer-protocol.ts diff --git a/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md index 4aedfaf16b..35c818bc56 100644 --- a/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md +++ b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md @@ -143,7 +143,7 @@ Nested `RouteSubjectV1` must come from existing CL-03/CL-06 builders (`buildRout ## E. Lab ingestion -`observationFromFabricOutcome` / `persistFabricOutcome`: +`observationFromFabricOutcome` / internal `persistFabricOutcome` (not public): - `evidenceLayer: "task_effectiveness"` - `executionMode: "fabric"` @@ -241,30 +241,41 @@ Typecheck; focused fabric/subject/sandbox/verifier/observe/ledger/projection/rea - Production evidence uses `runFabricSyntheticPatchTaskForRoute({ routeContext, destination, patchExecutor })`. - `RouteSubjectV1` is built only via `buildRouteSubjectV1(routeContext, destination)` — callers cannot supply an independent route identity. - Patch production requires a host-issued `TrustedFabricPatchExecutor` (`createHostIssuedFabricPatchExecutor` in `src/lib/fabric-task-host.ts`). -- `persistFabricRunResult` rejects harness-only runs (`executionAuthority !== "trusted_route"`). +- **Public ingestion:** `persistFabricRunResult` only; it rejects harness runs (`executionAuthority !== "trusted_route"`). +- `persistFabricOutcome` is internal to `observe.ts` and is **not** exported from the fabric public surface. - Harness runs (`runFabricSyntheticPatchTaskHarness`) use closed `FabricHarnessProducerKind` values only; they cannot create production ledger evidence. -### Termination / sandbox model +### Child isolation / IPC / timeouts -- Patch producers run in a dedicated Bun child (`producer-child.ts`) spawned by `producer-isolate.ts` with minimal env (`TZ`, `NO_COLOR`). -- Hard `SIGKILL` on total timeout; child process cannot block the OpenCodex event loop with a synchronous infinite loop. -- Scratch containment, symlink/path traversal rejection, byte/file limits, and cleanup remain in the parent executor after the child returns a validated `SyntheticPatchV1`. +- Patch producers run in a dedicated Bun child (`producer-child.ts`) spawned by `producer-isolate.ts` with minimal env (`TZ`, `NO_COLOR`, `OCX_FABRIC_SCRATCH_ROOT`). +- Parent↔child protocol is newline-delimited JSON on stdout only (`activity`, `result`, `error` in `producer-protocol.ts`). Arbitrary logging is not mixed into protocol output. +- Parent owns **both** total and inactivity timeouts; both terminate the child via `SIGKILL`. Classification: `timeout` vs `inactivity_timeout`. +- Child stdout is capped at 64 KiB protocol bytes; stderr diagnostic capture capped at 4 KiB. Exceeding protocol limits → `budget_exhausted` and child kill. +- `lastActivityAt` is authoritative in the parent; child `reportActivity()` emits `activity` IPC messages that reset the inactivity deadline. +- `infinite_sync` harness disables inactivity ceiling extension so synchronous CPU spin is classified under total timeout. + +### Sandbox enforcement (honest scope) + +- Scratch containment, symlink/path-traversal/special-file rejection, byte/file limits, and cleanup are enforced in the parent via `scratch.ts`, `patch.ts` (`assertSafeRelativePosixPath`), and `applySyntheticPatch`. +- Child isolation strips proxy env vars on the parent path and runs producers in a separate process with minimal env — **not** an OS-level network/shell sandbox. +- Host-issued executor modules may still perform direct host filesystem operations outside the scratch tree; that is outside the scratch-apply boundary and is not claimed as blocked. +- Declared deny flags (`fabricDeclaredSandboxPolicy`) document intent; runtime enforcement matches the scratch/patch/verifier containment above. ### Failure attribution - Semantic verifier mismatch → `fail` / `behavioral_failure` / `route`. -- Sandbox/containment (`FabricTaskError` from scratch/verifier infrastructure) → `blocked` or `inconclusive` / `sandbox_violation` / `harness` or `environment` — never route-attributed `behavioral_failure`. +- Sandbox/containment (`FabricTaskError` from scratch/verifier/patch infrastructure) → `blocked` or `inconclusive` / `sandbox_violation` / `harness` or `environment` — never route-attributed `behavioral_failure`. - Timeouts / budget exhaustion → `blocked` / `environment`. - Harness defects → `inconclusive` / `harness`. ### Inactivity accounting -- `inactiveMs` = `completedAt - lastActivityAt` where `lastActivityAt` resets on producer `reportActivity()` (trusted child executor path). +- `inactiveMs` = `completedAt - lastActivityAt` where `lastActivityAt` is updated only from parent-received `activity` IPC (or initial start). ### Local validation (blocker fix head) - `bun x tsc --noEmit`: passed -- `bun test tests/lab-fabric-task.test.ts`: 39/39 passed +- `bun test tests/lab-fabric-task.test.ts`: 46/46 passed - `bun run privacy:scan`: passed -- `bun test tests/lab-read-surfaces.test.ts tests/routing-compatibility.test.ts tests/lab-live-probe.test.ts`: passed +- Windows sqlite projection flakes in `lab-evidence-ledger.test.ts` (EBUSY) — environmental, not CL-07 - CL-08: **not started** diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts index 7eb95cb4e1..8068d0dac6 100644 --- a/src/lab/fabric/executor.ts +++ b/src/lab/fabric/executor.ts @@ -188,41 +188,44 @@ async function runFabricSyntheticPatchTaskInternal(input: { } const isolation = fabricProducerIsolationLimits(); + const inactivityTimeoutMs = + input.harnessKind === "infinite_sync" + ? isolation.totalTimeoutMs + 1_000 + : isolation.inactivityTimeoutMs; let patchRaw: unknown; try { if (input.harnessKind) { lastActivityAt = input.now?.() ?? Date.now(); - patchRaw = await runIsolatedFabricProducer({ + const isolated = await runIsolatedFabricProducer({ harnessKind: input.harnessKind, scratchRoot: scratch.root, totalTimeoutMs: isolation.totalTimeoutMs, - inactivityTimeoutMs: isolation.inactivityTimeoutMs, + inactivityTimeoutMs, + now: input.now, }); + patchRaw = isolated.patch; + lastActivityAt = isolated.lastActivityAt; producerCompletedAt = input.now?.() ?? Date.now(); - lastActivityAt = producerCompletedAt; } else if (input.patchExecutor && input.routeContext && input.destination) { - const controller = new AbortController(); - const executorInput = { - routeContext: input.routeContext, - destination: input.destination, - routeSubject: input.routeSubject, - scratchRoot: scratch.root, - reportActivity: () => { - lastActivityAt = input.now?.() ?? Date.now(); - }, - signal: controller.signal, - }; lastActivityAt = input.now?.() ?? Date.now(); - patchRaw = await runIsolatedFabricProducer({ + const isolated = await runIsolatedFabricProducer({ executorModulePath: input.patchExecutor.executorModulePath, scratchRoot: scratch.root, totalTimeoutMs: isolation.totalTimeoutMs, inactivityTimeoutMs: isolation.inactivityTimeoutMs, - executorInput, + executorInput: { + routeContext: input.routeContext, + destination: input.destination, + routeSubject: input.routeSubject, + scratchRoot: scratch.root, + reportActivity: () => undefined, + signal: new AbortController().signal, + }, + now: input.now, }); + patchRaw = isolated.patch; + lastActivityAt = isolated.lastActivityAt; producerCompletedAt = input.now?.() ?? Date.now(); - lastActivityAt = producerCompletedAt; - controller.abort(); } else { throw new FabricTaskError("fabric task run missing producer", "harness_failure", "harness"); } @@ -253,8 +256,37 @@ async function runFabricSyntheticPatchTaskInternal(input: { throw error; } - const patch = parseSyntheticPatchV1(patchRaw); - const applied = applySyntheticPatch(scratch.root, patch); + let patch; + let applied; + try { + patch = parseSyntheticPatchV1(patchRaw); + applied = applySyntheticPatch(scratch.root, patch); + } catch (error) { + const completedAt = input.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: infrastructureVerifier(verifierDigest, error.code), + failure, + sourceRefs: input.sourceRefs, + }), + }; + } + throw error; + } usage.outputBytes = applied.bytesWritten; usage.patchOperations = applied.patchOperations; usage.filesTouched = applied.filesTouched; diff --git a/src/lab/fabric/index.ts b/src/lab/fabric/index.ts index 1bc219a33c..e6f9acca82 100644 --- a/src/lab/fabric/index.ts +++ b/src/lab/fabric/index.ts @@ -75,7 +75,6 @@ export type { FabricCaseAuthority } from "./manifest"; export { assertFabricOutcomeV1, observationFromFabricOutcome, - persistFabricOutcome, persistFabricRunResult, } from "./observe"; export type { PersistFabricOptions, PersistedFabricObservation } from "./observe"; diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index c3d28b1879..78955ab86e 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -421,7 +421,7 @@ export function observationFromFabricOutcome( } } -/** Persist a fabric outcome observation to the compatibility ledger when absent. */ +/** @internal Ledger append helper; production evidence must use {@link persistFabricRunResult}. */ export function persistFabricOutcome( outcome: FabricTaskOutcomeV1, opts: PersistFabricOptions = {}, diff --git a/src/lab/fabric/patch.ts b/src/lab/fabric/patch.ts index 81b222b2ae..9ac196f5b8 100644 --- a/src/lab/fabric/patch.ts +++ b/src/lab/fabric/patch.ts @@ -1,5 +1,5 @@ import { FABRIC_LIMITS, SYNTHETIC_VALUE_PATH } from "./constants"; -import { writeScratchFileUtf8 } from "./scratch"; +import { assertSafeRelativePosixPath, writeScratchFileUtf8 } from "./scratch"; import { FabricTaskError, type SyntheticPatchV1 } from "./types"; const ALLOWED_KEYS = new Set(["schemaVersion", "operations"]); @@ -38,6 +38,12 @@ export function parseSyntheticPatchV1(raw: unknown): SyntheticPatchV1 { if (op.op !== "replace" || typeof op.path !== "string" || typeof op.contentUtf8 !== "string") { throw new FabricTaskError("malformed producer outcome: replace shape", "malformed_producer_outcome", "harness"); } + try { + assertSafeRelativePosixPath(op.path); + } catch (error) { + if (error instanceof FabricTaskError) throw error; + throw new FabricTaskError("patch path escapes scratch", "sandbox_violation", "harness"); + } if (op.path !== SYNTHETIC_VALUE_PATH) { throw new FabricTaskError("patch path must be src/value.txt", "behavioral_failure", "route"); } diff --git a/src/lab/fabric/producer-child.ts b/src/lab/fabric/producer-child.ts index 7dbe4511dd..6b299fd193 100644 --- a/src/lab/fabric/producer-child.ts +++ b/src/lab/fabric/producer-child.ts @@ -4,21 +4,16 @@ import { SYNTHETIC_BEFORE_UTF8, SYNTHETIC_VALUE_PATH, } from "./constants"; +import { + encodeProducerProtocolLine, + FABRIC_PRODUCER_PROTOCOL_MAX_BYTES, + FABRIC_PRODUCER_REQUEST_MAX_BYTES, + type ProducerParentRequest, +} from "./producer-protocol"; import type { FabricHarnessProducerKind, SyntheticPatchV1 } from "./types"; -interface ChildRequest { - harnessKind?: FabricHarnessProducerKind; - executorModulePath?: string; - scratchRoot: string; - totalTimeoutMs: number; - inactivityTimeoutMs: number; - executorInput?: unknown; -} - -interface ChildResponse { - ok: boolean; - patch?: SyntheticPatchV1; - error?: { message: string; code: string; attribution: string }; +function writeLine(message: Parameters[0]): void { + process.stdout.write(encodeProducerProtocolLine(message)); } function correctPatch(): SyntheticPatchV1 { @@ -43,11 +38,12 @@ async function runHarness(kind: FabricHarnessProducerKind, scratchRoot: string): return wrongPatch(); case "infinite_sync": while (true) { - /* hang until killed */ + /* terminated by parent */ } case "never_resolve": - await new Promise(() => undefined); - throw new Error("never_resolve unexpectedly completed"); + while (true) { + await Bun.sleep(1_000); + } case "mutate_after_delay": await Bun.sleep(FABRIC_LIMITS.totalTimeoutMs + 5_000); const { writeFileSync, mkdirSync } = await import("node:fs"); @@ -55,6 +51,21 @@ async function runHarness(kind: FabricHarnessProducerKind, scratchRoot: string): mkdirSync(join(scratchRoot, dirname(SYNTHETIC_VALUE_PATH)), { recursive: true }); writeFileSync(join(scratchRoot, SYNTHETIC_VALUE_PATH), "late\n"); return correctPatch(); + case "periodic_activity": + for (let i = 0; i < 20; i++) { + writeLine({ type: "activity" }); + await Bun.sleep(200); + } + return correctPatch(); + case "activity_until_total": + while (true) { + writeLine({ type: "activity" }); + await Bun.sleep(50); + } + case "flood_stdout": + while (true) { + process.stdout.write("x".repeat(4096)); + } default: throw new Error(`unsupported harness kind: ${kind as string}`); } @@ -63,53 +74,48 @@ async function runHarness(kind: FabricHarnessProducerKind, scratchRoot: string): async function runExecutorModule( modulePath: string, executorInput: unknown, - inactivityTimeoutMs: number, + reportActivity: () => void, ): Promise { const mod = await import(modulePath) as { execute: (input: unknown) => Promise | SyntheticPatchV1 }; if (typeof mod.execute !== "function") throw new Error("executor module missing execute export"); - let lastActivity = Date.now(); const input = { ...(executorInput as Record), - reportActivity: () => { - lastActivity = Date.now(); - }, + reportActivity, }; - const inactivity = setInterval(() => { - if (Date.now() - lastActivity > inactivityTimeoutMs) { - throw new Error("inactivity_timeout"); - } - }, 50); - try { - return await Promise.resolve(mod.execute(input)); - } finally { - clearInterval(inactivity); - } + return await Promise.resolve(mod.execute(input)); } async function main(): Promise { const raw = await Bun.stdin.text(); - const req = JSON.parse(raw) as ChildRequest; + if (Buffer.byteLength(raw, "utf8") > FABRIC_PRODUCER_REQUEST_MAX_BYTES) { + writeLine({ + type: "error", + code: "budget_exhausted", + message: "request payload exceeds protocol limit", + attribution: "environment", + }); + process.exit(1); + return; + } + const req = JSON.parse(raw) as ProducerParentRequest; let patch: SyntheticPatchV1; + const reportActivity = () => writeLine({ type: "activity" }); if (req.harnessKind) { - patch = await runHarness(req.harnessKind, req.scratchRoot); + patch = await runHarness(req.harnessKind as FabricHarnessProducerKind, req.scratchRoot); } else if (req.executorModulePath) { - patch = await runExecutorModule(req.executorModulePath, req.executorInput, req.inactivityTimeoutMs); + patch = await runExecutorModule(req.executorModulePath, req.executorInput, reportActivity); } else { throw new Error("child request missing harnessKind or executorModulePath"); } - const response: ChildResponse = { ok: true, patch }; - process.stdout.write(JSON.stringify(response)); + writeLine({ type: "result", patch }); } main().catch((error: unknown) => { - const response: ChildResponse = { - ok: false, - error: { - message: error instanceof Error ? error.message : String(error), - code: error instanceof Error && error.message === "inactivity_timeout" ? "inactivity_timeout" : "harness_failure", - attribution: "harness", - }, - }; - process.stdout.write(JSON.stringify(response)); + writeLine({ + type: "error", + code: "harness_failure", + message: error instanceof Error ? error.message : String(error), + attribution: "harness", + }); process.exit(1); }); diff --git a/src/lab/fabric/producer-isolate.ts b/src/lab/fabric/producer-isolate.ts index 842921f0de..a8bac8afe9 100644 --- a/src/lab/fabric/producer-isolate.ts +++ b/src/lab/fabric/producer-isolate.ts @@ -1,7 +1,14 @@ -import { spawn } from "node:child_process"; +import { spawn, type ChildProcess } from "node:child_process"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { FABRIC_LIMITS } from "./constants"; +import { + FABRIC_PRODUCER_PROTOCOL_MAX_BYTES, + FABRIC_PRODUCER_REQUEST_MAX_BYTES, + FABRIC_PRODUCER_STDERR_MAX_BYTES, + parseProducerProtocolLine, + type IsolatedProducerResult, +} from "./producer-protocol"; import type { FabricHarnessProducerKind, FabricPatchExecutorInput, SyntheticPatchV1 } from "./types"; import { FabricTaskError } from "./types"; @@ -14,99 +21,164 @@ interface IsolateRequest { totalTimeoutMs: number; inactivityTimeoutMs: number; executorInput?: FabricPatchExecutorInput; + now?: () => number; } -interface IsolateResponse { - ok: boolean; - patch?: SyntheticPatchV1; - error?: { message: string; code: string; attribution: string }; +function minimalChildEnv(scratchRoot: string): Record { + return { + TZ: "UTC", + NO_COLOR: "1", + OCX_FABRIC_SCRATCH_ROOT: scratchRoot, + }; } -function minimalChildEnv(): Record { - return { TZ: "UTC", NO_COLOR: "1" }; +function killChild(child: ChildProcess): void { + try { + child.kill("SIGKILL"); + } catch { + /* ignore */ + } } -/** Run a fabric patch producer in an isolated child process with hard termination. */ -export async function runIsolatedFabricProducer(request: IsolateRequest): Promise { - return await new Promise((resolve, reject) => { - const child = spawn(process.execPath, ["run", CHILD_ENTRY], { - env: minimalChildEnv(), - stdio: ["pipe", "pipe", "pipe"], - }); +/** Run a fabric patch producer in an isolated child process with parent-owned timeouts. */ +export async function runIsolatedFabricProducer(request: IsolateRequest): Promise { + const now = request.now ?? (() => Date.now()); + let lastActivityAt = now(); + + return await new Promise((resolve, reject) => { + let child: ChildProcess; + try { + child = spawn(process.execPath, ["run", CHILD_ENTRY], { + env: minimalChildEnv(request.scratchRoot), + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (error) { + reject(new FabricTaskError( + error instanceof Error ? error.message : String(error), + "harness_failure", + "harness", + )); + return; + } - let stdout = ""; + let stdoutBuffer = ""; + let stderrBytes = 0; let settled = false; + let receivedResult: SyntheticPatchV1 | undefined; + let killReason: FabricTaskError | undefined; + const finish = (fn: () => void) => { if (settled) return; settled = true; clearTimeout(totalTimer); + clearTimeout(inactivityTimer); fn(); }; - const totalTimer = setTimeout(() => { - try { - child.kill("SIGKILL"); - } catch { - /* ignore */ - } - finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); - }, request.totalTimeoutMs); + const settleTimeout = (error: FabricTaskError) => { + if (settled) return; + killReason = error; + killChild(child); + }; - child.stdout.on("data", (chunk: Buffer | string) => { - stdout += chunk.toString("utf8"); - }); + const armInactivity = () => { + clearTimeout(inactivityTimer); + inactivityTimer = setTimeout(() => { + settleTimeout(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); + }, request.inactivityTimeoutMs); + }; - child.stderr.on("data", () => { - /* discard */ - }); + let inactivityTimer: ReturnType = setTimeout(() => { + settleTimeout(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); + }, request.inactivityTimeoutMs); - child.on("error", (error) => { - finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); - }); + const totalTimer = setTimeout(() => { + settleTimeout(new FabricTaskError("total timeout exceeded", "timeout", "environment")); + }, request.totalTimeoutMs); - child.on("close", (code, signal) => { - if (settled) return; - if (signal === "SIGKILL") { - finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); - return; - } + const handleProtocolLine = (line: string) => { try { - const trimmed = stdout.trim(); - if (!trimmed) { - finish(() => reject(new FabricTaskError("isolated producer returned no output", "harness_failure", "harness"))); + const message = parseProducerProtocolLine(line); + if (message.type === "activity") { + lastActivityAt = now(); + armInactivity(); + return; + } + if (message.type === "result") { + if (settled) return; + receivedResult = message.patch; + finish(() => resolve({ patch: message.patch, lastActivityAt })); return; } - const parsed = JSON.parse(trimmed) as IsolateResponse; - if (!parsed.ok || !parsed.patch) { - const code = parsed.error?.code ?? "harness_failure"; - const attribution = parsed.error?.attribution === "environment" ? "environment" : "harness"; - const fabricCode = code === "inactivity_timeout" + if (message.type === "error") { + const fabricCode = message.code === "inactivity_timeout" ? "inactivity_timeout" - : code === "timeout" + : message.code === "timeout" ? "timeout" - : code === "sandbox_violation" + : message.code === "sandbox_violation" ? "sandbox_violation" - : "harness_failure"; - finish(() => reject(new FabricTaskError(parsed.error?.message ?? "isolated producer failed", fabricCode, attribution))); + : message.code === "budget_exhausted" + ? "budget_exhausted" + : "harness_failure"; + const attribution = message.attribution === "environment" ? "environment" : "harness"; + const fabricError = new FabricTaskError(message.message, fabricCode, attribution); + finish(() => reject(fabricError)); + killChild(child); return; } - finish(() => resolve(parsed.patch!)); } catch (error) { - finish(() => reject(new FabricTaskError( + settleTimeout(new FabricTaskError( error instanceof Error ? error.message : String(error), "harness_failure", "harness", - ))); + )); + } + }; + + const consumeStdout = (chunk: string) => { + stdoutBuffer += chunk; + if (Buffer.byteLength(stdoutBuffer, "utf8") > FABRIC_PRODUCER_PROTOCOL_MAX_BYTES) { + settleTimeout(new FabricTaskError("producer protocol output exceeded limit", "budget_exhausted", "environment")); + stdoutBuffer = ""; + return; } - void code; + let newlineIdx = stdoutBuffer.indexOf("\n"); + while (newlineIdx >= 0) { + const line = stdoutBuffer.slice(0, newlineIdx); + stdoutBuffer = stdoutBuffer.slice(newlineIdx + 1); + handleProtocolLine(line); + newlineIdx = stdoutBuffer.indexOf("\n"); + } + }; + + child.stdout?.on("data", (chunk: Buffer | string) => { + consumeStdout(chunk.toString("utf8")); + }); + + child.stdout?.on("error", (error) => { + if (settled) return; + finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + }); + + child.stderr?.on("data", (chunk: Buffer | string) => { + stderrBytes += Buffer.byteLength(chunk.toString("utf8"), "utf8"); + if (stderrBytes > FABRIC_PRODUCER_STDERR_MAX_BYTES) { + stderrBytes = FABRIC_PRODUCER_STDERR_MAX_BYTES; + } + }); + + child.on("error", (error) => { + finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + }); + + child.stdin?.on("error", () => { + /* EPIPE after close is expected */ }); const payload = JSON.stringify({ harnessKind: request.harnessKind, executorModulePath: request.executorModulePath, scratchRoot: request.scratchRoot, - totalTimeoutMs: request.totalTimeoutMs, - inactivityTimeoutMs: request.inactivityTimeoutMs, executorInput: request.executorInput ? { routeContext: request.executorInput.routeContext, @@ -116,8 +188,53 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis } : undefined, }); - child.stdin.write(payload); - child.stdin.end(); + if (Buffer.byteLength(payload, "utf8") > FABRIC_PRODUCER_REQUEST_MAX_BYTES) { + killChild(child); + finish(() => reject(new FabricTaskError("producer request exceeds protocol limit", "budget_exhausted", "environment"))); + return; + } + + try { + child.stdin?.write(payload); + child.stdin?.end(); + } catch (error) { + killChild(child); + finish(() => reject(new FabricTaskError( + error instanceof Error ? error.message : String(error), + "harness_failure", + "harness", + ))); + return; + } + + child.on("close", (code, signal) => { + if (settled) return; + if (killReason) { + finish(() => reject(killReason!)); + return; + } + if (receivedResult) { + finish(() => resolve({ patch: receivedResult!, lastActivityAt })); + return; + } + if (stdoutBuffer.trim()) { + try { + handleProtocolLine(stdoutBuffer.trim()); + if (receivedResult) return; + } catch { + /* fall through */ + } + } + if (signal === "SIGKILL") { + finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); + return; + } + finish(() => reject(new FabricTaskError( + code === 0 ? "isolated producer returned no result" : `isolated producer exited (${code ?? signal ?? "unknown"})`, + "harness_failure", + "harness", + ))); + }); }); } diff --git a/src/lab/fabric/producer-protocol.ts b/src/lab/fabric/producer-protocol.ts new file mode 100644 index 0000000000..ab20548467 --- /dev/null +++ b/src/lab/fabric/producer-protocol.ts @@ -0,0 +1,58 @@ +import type { SyntheticPatchV1 } from "./types"; + +/** Maximum bytes accepted from child stdout for the fabric producer protocol. */ +export const FABRIC_PRODUCER_PROTOCOL_MAX_BYTES = 64 * 1024; + +/** Maximum stderr bytes retained for harness diagnostics. */ +export const FABRIC_PRODUCER_STDERR_MAX_BYTES = 4 * 1024; + +/** Maximum request payload size on child stdin. */ +export const FABRIC_PRODUCER_REQUEST_MAX_BYTES = 64 * 1024; + +export type ProducerChildMessage = + | { type: "activity" } + | { type: "result"; patch: SyntheticPatchV1 } + | { type: "error"; code: string; message: string; attribution: "harness" | "environment" }; + +export interface ProducerParentRequest { + harnessKind?: string; + executorModulePath?: string; + scratchRoot: string; + executorInput?: unknown; +} + +export interface IsolatedProducerResult { + patch: SyntheticPatchV1; + lastActivityAt: number; +} + +/** Serialize one protocol line to stdout (newline-delimited JSON). */ +export function encodeProducerProtocolLine(message: ProducerChildMessage): string { + return `${JSON.stringify(message)}\n`; +} + +/** Parse one protocol line from the child. Malformed lines throw. */ +export function parseProducerProtocolLine(line: string): ProducerChildMessage { + const trimmed = line.trim(); + if (!trimmed) { + throw new Error("empty protocol line"); + } + const parsed = JSON.parse(trimmed) as ProducerChildMessage; + if (!parsed || typeof parsed !== "object" || typeof parsed.type !== "string") { + throw new Error("malformed protocol message"); + } + switch (parsed.type) { + case "activity": + return { type: "activity" }; + case "result": + if (!parsed.patch || typeof parsed.patch !== "object") throw new Error("malformed result patch"); + return parsed as ProducerChildMessage; + case "error": + if (typeof parsed.code !== "string" || typeof parsed.message !== "string") { + throw new Error("malformed error message"); + } + return parsed as ProducerChildMessage; + default: + throw new Error(`unknown protocol type: ${(parsed as { type: string }).type}`); + } +} diff --git a/src/lab/fabric/types.ts b/src/lab/fabric/types.ts index e1f72c8af8..de2780a6f2 100644 --- a/src/lab/fabric/types.ts +++ b/src/lab/fabric/types.ts @@ -108,7 +108,10 @@ export type FabricHarnessProducerKind = | "deterministic_wrong" | "infinite_sync" | "never_resolve" - | "mutate_after_delay"; + | "mutate_after_delay" + | "periodic_activity" + | "activity_until_total" + | "flood_stdout"; /** @deprecated Test-only harness producer; use FabricHarnessProducerKind via runFabricSyntheticPatchTaskHarness. */ export type SyntheticPatchProducer = (ctx: { diff --git a/tests/helpers/fabric-task-test.ts b/tests/helpers/fabric-task-test.ts index 0a8e175869..264d69a83c 100644 --- a/tests/helpers/fabric-task-test.ts +++ b/tests/helpers/fabric-task-test.ts @@ -62,8 +62,8 @@ export function fabricCorrectPatchExecutor(): TrustedFabricPatchExecutor { return createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()); } -export function fabricRouteBoundPatchExecutor(expectedProviderId: string): TrustedFabricPatchExecutor { - const dir = join(FIXTURE_DIR, "generated"); +export function fabricRouteBoundPatchExecutor(home: string, expectedProviderId: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); mkdirSync(dir, { recursive: true }); const modulePath = join(dir, `route-bound-${expectedProviderId}.ts`); writeFileSync(modulePath, ` diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index cc06772098..e63c19fa9d 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -26,7 +26,6 @@ import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, observationFromFabricOutcome, - persistFabricOutcome, persistFabricRunResult, queryLabCatalog, readVerdictSnapshot, @@ -62,6 +61,7 @@ import { fabricCorrectPatchExecutor, fabricMockRoute, fabricOversizedPatchExecutor, + fabricRouteBoundPatchExecutor, runTrustedFabricTask, } from "./helpers/fabric-task-test"; @@ -140,6 +140,51 @@ export async function execute(_input: FabricPatchExecutorInput): Promise { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: "../outside.txt", contentUtf8: "evil\\n" }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => ({ + schemaVersion: 1, + operations: [{ op: "replace", path: "../outside.txt", contentUtf8: "evil\n" }], + })); +} + +function fabricOutsideScratchWriteExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const outside = join(home, "outside-scratch"); + mkdirSync(outside, { recursive: true }); + const modulePath = join(dir, "outside-write.ts"); + writeFileSync(modulePath, ` +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +const outside = "${outside.replace(/\\/g, "/")}"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + writeFileSync(join(outside, "evil.txt"), "evil\\n"); + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()); +} + function fabricSymlinkSandboxExecutor(home: string): TrustedFabricPatchExecutor { const dir = join(home, "fabric-executors"); mkdirSync(dir, { recursive: true }); @@ -290,6 +335,8 @@ describe("CL-07 task effectiveness producer", () => { } catch (error) { expect((error as FabricTaskError).code).toBe("malformed_producer_outcome"); } + const labExports = await import("../src/lab"); + expect("persistFabricOutcome" in labExports).toBe(false); }); test("trusted route execution can be persisted as production evidence", async () => { @@ -413,7 +460,7 @@ describe("CL-07 task effectiveness producer", () => { expect(result.outcome.failure?.code).toBe("budget_exhausted"); }); - test("never_resolve harness terminates within total budget", async () => { + test("never_resolve harness terminates on inactivity timeout", async () => { const home = tempHome(); const result = await runFabricSyntheticPatchTaskHarness({ routeSubject: routeSubject(), @@ -421,10 +468,10 @@ describe("CL-07 task effectiveness producer", () => { configDir: home, }); expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); - expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); - }, 40_000); + expect(result.outcome.failure?.code).toBe("inactivity_timeout"); + }, 20_000); - test("infinite_sync harness terminates within total budget", async () => { + test("infinite_sync harness terminates on total timeout", async () => { const home = tempHome(); const result = await runFabricSyntheticPatchTaskHarness({ routeSubject: routeSubject(), @@ -432,9 +479,41 @@ describe("CL-07 task effectiveness producer", () => { configDir: home, }); expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); - expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); + expect(result.outcome.failure?.code).toBe("timeout"); + }, 40_000); + + test("periodic_activity harness survives inactivity within total budget", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "periodic_activity", + configDir: home, + }); + expect(result.outcome.outcome).toBe("pass"); + }, 20_000); + + test("activity_until_total harness terminates on total timeout", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "activity_until_total", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(result.outcome.failure?.code).toBe("timeout"); }, 40_000); + test("flood_stdout harness terminates on protocol byte limit", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "flood_stdout", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(result.outcome.failure?.code).toBe("budget_exhausted"); + }, 20_000); + test("mutate_after_delay harness kills producer and cleans scratch", async () => { const home = tempHome(); process.env.OPENCODEX_HOME = home; @@ -469,7 +548,7 @@ describe("CL-07 task effectiveness producer", () => { configDir: home, }); expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); - expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); + expect(result.outcome.failure?.code).toBe("inactivity_timeout"); }, 20_000); test("activity resets inactivity deadline within total budget", async () => { @@ -543,6 +622,67 @@ describe("CL-07 task effectiveness producer", () => { }); }); + test("untrusted patch executor is rejected before isolation", async () => { + const home = tempHome(); + const fakeExecutor = { + executorModulePath: join(home, "evil.ts"), + execute: async () => correctSyntheticPatch(), + }; + await expect(runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fakeExecutor as TrustedFabricPatchExecutor, + configDir: home, + })).rejects.toThrow(FabricTaskError); + }); + + test("authoritative route subject is bound into trusted outcomes", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const routeContext = fabricMockRoute({ providerId: "provider-bound" }); + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext, + destination: await fabricDestination(home), + patchExecutor: fabricRouteBoundPatchExecutor(home, "provider-bound"), + configDir: home, + }); + expect(result.outcome.routeSubject.providerId).toBe("provider-bound"); + expect(result.outcome.taskSubject.routeSubject.providerId).toBe("provider-bound"); + expect(result.outcome.routeSubject).toEqual(result.outcome.taskSubject.routeSubject); + }); + + test("patch path traversal is rejected at scratch apply boundary", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricTraversalPatchExecutor(home), + configDir: home, + }); + expect(result.outcome.outcome).not.toBe("pass"); + expect(result.outcome.failure?.code).toBe("sandbox_violation"); + expect(existsSync(join(home, "outside.txt"))).toBe(false); + }); + + test("adversarial executor direct write outside scratch is not production evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outside = join(home, "outside-scratch"); + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricOutsideScratchWriteExecutor(home), + configDir: home, + }); + expect(result.outcome.outcome).toBe("pass"); + expect(existsSync(join(outside, "evil.txt"))).toBe(true); + expect(() => persistFabricRunResult({ + ...result, + executionAuthority: "harness", + }, { configDir: home })).toThrow(FabricTaskError); + }); + test("user repository cannot host the scratch root", () => { const home = tempHome(); const repo = join(home, "user-repo"); @@ -704,11 +844,15 @@ describe("CL-07 task effectiveness producer", () => { expect(summaryRef).toBeDefined(); const text = new TextDecoder().decode(store.get(summaryRef!.digest)); expect(text.includes(CREDENTIAL_CANARY)).toBe(false); - persistFabricOutcome({ - ...base, - verifier: { - ...base.verifier, - reason: `failed ${CREDENTIAL_CANARY}`, + const trustedRun = await runTrustedFabricTask(home); + persistFabricRunResult({ + ...trustedRun, + outcome: { + ...trustedRun.outcome, + verifier: { + ...trustedRun.outcome.verifier, + reason: `failed ${CREDENTIAL_CANARY}`, + }, }, }, { configDir: home }); const ledger = readFileSync(join(home, "lab", "compatibility.jsonl"), "utf8"); From e7c6337e380de982b56fbb2f2689364c85e2c691 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:57:12 +0200 Subject: [PATCH 15/24] test(lab): reproduce infrastructure symlink path alias --- tests/lab-paths-security.test.ts | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/lab-paths-security.test.ts diff --git a/tests/lab-paths-security.test.ts b/tests/lab-paths-security.test.ts new file mode 100644 index 0000000000..d93580b585 --- /dev/null +++ b/tests/lab-paths-security.test.ts @@ -0,0 +1,51 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ensureRestrictedDir } from "../src/lab/paths"; + +const ROOTS: string[] = []; + +function tempRoot(): string { + const root = mkdtempSync(join(tmpdir(), "ocx-lab-paths-")); + ROOTS.push(root); + return root; +} + +afterEach(() => { + for (const root of ROOTS.splice(0)) { + try { rmSync(root, { recursive: true, force: true }); } catch { /* ignore */ } + } +}); + +test("restricted Lab paths allow a symlinked infrastructure ancestor above the Lab boundary", () => { + const root = tempRoot(); + const actualInfrastructure = join(root, "actual-infrastructure"); + const infrastructureAlias = join(root, "infrastructure-alias"); + mkdirSync(actualInfrastructure, { recursive: true }); + + try { + symlinkSync(actualInfrastructure, infrastructureAlias, process.platform === "win32" ? "junction" : "dir"); + } catch { + return; + } + + const lab = join(infrastructureAlias, "opencodex", "lab"); + expect(() => ensureRestrictedDir(lab, lab)).not.toThrow(); + expect(realpathSync.native(lab)).toBe(realpathSync.native(join(actualInfrastructure, "opencodex", "lab"))); +}); + +test("restricted Lab paths still reject a symlink at the Lab boundary itself", () => { + const root = tempRoot(); + const actualLab = join(root, "actual-lab"); + const substitutedLab = join(root, "lab"); + mkdirSync(actualLab, { recursive: true }); + + try { + symlinkSync(actualLab, substitutedLab, process.platform === "win32" ? "junction" : "dir"); + } catch { + return; + } + + expect(() => ensureRestrictedDir(substitutedLab, substitutedLab)).toThrow(/symbolic link|link or reparse-point substitution/); +}); From 3596fa56400edafd0b906b53117998abedcabfb6 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:58:05 +0200 Subject: [PATCH 16/24] fix(lab): allow canonical infrastructure aliases above Lab root --- src/lab/paths.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 03a815b488..754b78bb4d 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -1,5 +1,5 @@ import { chmodSync, existsSync, mkdirSync, lstatSync, realpathSync } from "node:fs"; -import { join, resolve, sep } from "node:path"; +import { basename, dirname, join, resolve, sep } from "node:path"; import { getConfigDir } from "../config"; /** Return true when path is the Lab root or a descendant path component. */ @@ -20,8 +20,10 @@ function assertRestrictedDirectoryComponent(path: string): void { if (!componentStats.isDirectory()) { throw new Error(`restricted path component is not a directory: ${path}`); } - const canonical = realpathSync.native(path); - if (resolve(canonical) !== resolve(path)) { + const canonical = resolve(realpathSync.native(path)); + const canonicalParent = resolve(realpathSync.native(dirname(path))); + const expectedCanonical = resolve(join(canonicalParent, basename(path))); + if (canonical !== expectedCanonical) { throw new Error(`restricted directory component contains a link or reparse-point substitution: ${path}`); } } From 700303f0d4b3db994808b23f04926853efae9a8d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:35:40 +0200 Subject: [PATCH 17/24] test(lab): guard fabric persistence authority boundary --- tests/lab-fabric-persistence-boundary.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/lab-fabric-persistence-boundary.test.ts diff --git a/tests/lab-fabric-persistence-boundary.test.ts b/tests/lab-fabric-persistence-boundary.test.ts new file mode 100644 index 0000000000..572c7ed6cf --- /dev/null +++ b/tests/lab-fabric-persistence-boundary.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, test } from "bun:test"; + +describe("CL-07 fabric persistence export boundary", () => { + test("authority-free persistence helper is module-private", async () => { + const observeExports = await import("../src/lab/fabric/observe"); + + expect("persistFabricOutcome" in observeExports).toBe(false); + expect(typeof observeExports.persistFabricRunResult).toBe("function"); + }); +}); From 26575d0a2509da410fad354ac9bec3933006ba4d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:36:40 +0200 Subject: [PATCH 18/24] fix(lab): close fabric persistence authority bypass --- src/lab/fabric/observe.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 78955ab86e..426a4a5769 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -197,7 +197,7 @@ function validateFailureRecord(raw: Record): void { } assertStringField(raw, "code"); if (typeof raw.retryable !== "boolean") { - throw new FabricTaskError("malformed producer outcome: failure.retryable", "malformed_producer_outcome", "harness"); + throw new FabricTaskError("malformed producer outcome: failure.retryable", "harness"); } const attribution = assertStringField(raw, "attribution"); if (!FAILURE_ATTRIBUTIONS.has(attribution)) { @@ -421,8 +421,8 @@ export function observationFromFabricOutcome( } } -/** @internal Ledger append helper; production evidence must use {@link persistFabricRunResult}. */ -export function persistFabricOutcome( +/** Module-private ledger append helper; production evidence must use {@link persistFabricRunResult}. */ +function persistFabricOutcome( outcome: FabricTaskOutcomeV1, opts: PersistFabricOptions = {}, ): PersistedFabricObservation { From 5607ea5330f57840ceee706f30e9aa4c13b12038 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:38:15 +0200 Subject: [PATCH 19/24] fix(lab): restore fabric validation error code --- src/lab/fabric/observe.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 426a4a5769..7e93d13fd3 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -197,7 +197,7 @@ function validateFailureRecord(raw: Record): void { } assertStringField(raw, "code"); if (typeof raw.retryable !== "boolean") { - throw new FabricTaskError("malformed producer outcome: failure.retryable", "harness"); + throw new FabricTaskError("malformed producer outcome: failure.retryable", "malformed_producer_outcome", "harness"); } const attribution = assertStringField(raw, "attribution"); if (!FAILURE_ATTRIBUTIONS.has(attribution)) { From a06066937e21df44989bded008b4a34b1612fd6d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:06:49 +0200 Subject: [PATCH 20/24] fix(lab): scope delayed producer case --- src/lab/fabric/producer-child.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lab/fabric/producer-child.ts b/src/lab/fabric/producer-child.ts index 6b299fd193..6a2f23349f 100644 --- a/src/lab/fabric/producer-child.ts +++ b/src/lab/fabric/producer-child.ts @@ -44,13 +44,14 @@ async function runHarness(kind: FabricHarnessProducerKind, scratchRoot: string): while (true) { await Bun.sleep(1_000); } - case "mutate_after_delay": + case "mutate_after_delay": { await Bun.sleep(FABRIC_LIMITS.totalTimeoutMs + 5_000); const { writeFileSync, mkdirSync } = await import("node:fs"); const { join, dirname } = await import("node:path"); mkdirSync(join(scratchRoot, dirname(SYNTHETIC_VALUE_PATH)), { recursive: true }); writeFileSync(join(scratchRoot, SYNTHETIC_VALUE_PATH), "late\n"); return correctPatch(); + } case "periodic_activity": for (let i = 0; i < 20; i++) { writeLine({ type: "activity" }); From 54e4c4e773adf9a79759a9826273c55d18309b67 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:07:07 +0200 Subject: [PATCH 21/24] fix(lab): simplify exact tree verifier --- src/lab/fabric/verifier.ts | 131 ++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 69 deletions(-) diff --git a/src/lab/fabric/verifier.ts b/src/lab/fabric/verifier.ts index 8ae70fee9d..88c58f7b89 100644 --- a/src/lab/fabric/verifier.ts +++ b/src/lab/fabric/verifier.ts @@ -21,85 +21,78 @@ function sha256Utf8(value: string): string { */ export function verifyExactTreeDiffV1(scratchRoot: string): ExactTreeDiffResultV1 { const manifestDigest = verifierManifestDigest(); - try { - const files = walkScratchFiles(scratchRoot); - let aggregate = 0; - for (const file of files) { - aggregate += file.byteLength; - if (aggregate > FABRIC_LIMITS.maxAggregateIoBytes) { - throw new FabricTaskError("tree exceeds io budget", "budget_exhausted", "environment"); - } - } - if (files.length !== 1 || files[0]!.relativePosix !== SYNTHETIC_VALUE_PATH) { - const unexpected = files - .filter((row) => row.relativePosix !== SYNTHETIC_VALUE_PATH) - .map((row) => row.relativePosix); - return { - verifierId: FABRIC_VERIFIER_ID, - manifestDigest, - passed: false, - pathSummaries: [ - ...unexpected.map((path) => ({ - path, - kind: "added" as const, - reason: "unexpected_file", - })), - ...(files.some((row) => row.relativePosix === SYNTHETIC_VALUE_PATH) - ? [] - : [{ path: SYNTHETIC_VALUE_PATH, kind: "deleted" as const, reason: "missing_required_file" }]), - ], - reason: files.length === 0 ? "deleted_required_file" : "unexpected_tree_shape", - }; - } - const after = readScratchFileUtf8(scratchRoot, SYNTHETIC_VALUE_PATH, FABRIC_LIMITS.maxAggregateIoBytes); - const beforeDigest = sha256Utf8(SYNTHETIC_BEFORE_UTF8); - const afterDigest = sha256Utf8(after); - const expectedDigest = sha256Utf8(SYNTHETIC_AFTER_UTF8); - if (after === SYNTHETIC_BEFORE_UTF8) { - return { - verifierId: FABRIC_VERIFIER_ID, - manifestDigest, - passed: false, - pathSummaries: [{ - path: SYNTHETIC_VALUE_PATH, - kind: "unchanged", - beforeDigest, - afterDigest, - reason: "unchanged", - }], - reason: "unchanged_file", - }; - } - if (after !== SYNTHETIC_AFTER_UTF8) { - return { - verifierId: FABRIC_VERIFIER_ID, - manifestDigest, - passed: false, - pathSummaries: [{ - path: SYNTHETIC_VALUE_PATH, - kind: "modified", - beforeDigest, - afterDigest, - reason: "wrong_final_bytes", - }], - reason: `expected_digest_${expectedDigest}`, - }; + const files = walkScratchFiles(scratchRoot); + let aggregate = 0; + for (const file of files) { + aggregate += file.byteLength; + if (aggregate > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("tree exceeds io budget", "budget_exhausted", "environment"); } + } + if (files.length !== 1 || files[0]!.relativePosix !== SYNTHETIC_VALUE_PATH) { + const unexpected = files + .filter((row) => row.relativePosix !== SYNTHETIC_VALUE_PATH) + .map((row) => row.relativePosix); + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [ + ...unexpected.map((path) => ({ + path, + kind: "added" as const, + reason: "unexpected_file", + })), + ...(files.some((row) => row.relativePosix === SYNTHETIC_VALUE_PATH) + ? [] + : [{ path: SYNTHETIC_VALUE_PATH, kind: "deleted" as const, reason: "missing_required_file" }]), + ], + reason: files.length === 0 ? "deleted_required_file" : "unexpected_tree_shape", + }; + } + const after = readScratchFileUtf8(scratchRoot, SYNTHETIC_VALUE_PATH, FABRIC_LIMITS.maxAggregateIoBytes); + const beforeDigest = sha256Utf8(SYNTHETIC_BEFORE_UTF8); + const afterDigest = sha256Utf8(after); + const expectedDigest = sha256Utf8(SYNTHETIC_AFTER_UTF8); + if (after === SYNTHETIC_BEFORE_UTF8) { + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "unchanged", + beforeDigest, + afterDigest, + reason: "unchanged", + }], + reason: "unchanged_file", + }; + } + if (after !== SYNTHETIC_AFTER_UTF8) { return { verifierId: FABRIC_VERIFIER_ID, manifestDigest, - passed: true, + passed: false, pathSummaries: [{ path: SYNTHETIC_VALUE_PATH, kind: "modified", beforeDigest, afterDigest, + reason: "wrong_final_bytes", }], + reason: `expected_digest_${expectedDigest}`, }; - } catch (error) { - if (error instanceof FabricTaskError) { - throw error; - } - throw error; } + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: true, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "modified", + beforeDigest, + afterDigest, + }], + }; } From d4defb00b36b41bc83c99e20d6b136cdc55a055f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:08:31 +0200 Subject: [PATCH 22/24] fix(lab): allow default deprecated harness kind --- src/lab/fabric/executor.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts index 8068d0dac6..66c9e2eb25 100644 --- a/src/lab/fabric/executor.ts +++ b/src/lab/fabric/executor.ts @@ -126,7 +126,10 @@ export async function runFabricSyntheticPatchTaskHarness( * @deprecated Use runFabricSyntheticPatchTaskHarness or runFabricSyntheticPatchTaskForRoute. */ export async function runFabricSyntheticPatchTask( - options: RunFabricTaskHarnessOptions & { producePatch?: never; harnessKind?: FabricHarnessProducerKind }, + options: Omit & { + producePatch?: never; + harnessKind?: RunFabricTaskHarnessOptions["harnessKind"]; + }, ): Promise { const result = await runFabricSyntheticPatchTaskHarness({ routeSubject: options.routeSubject, From 24287e74df5488603497ee77465fe756570397d8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:10:28 +0200 Subject: [PATCH 23/24] refactor(lab): derive global secret pattern --- src/lab/artifacts/sanitize.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lab/artifacts/sanitize.ts b/src/lab/artifacts/sanitize.ts index f989bf7377..8f64ed14ca 100644 --- a/src/lab/artifacts/sanitize.ts +++ b/src/lab/artifacts/sanitize.ts @@ -9,7 +9,7 @@ import { redactSecretString } from "../../lib/redact"; const FORBIDDEN_KEY = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|content|body|url|hostname|baseUrl|path|account|alias)$/i; const SECRETISH = /sk-[a-z0-9]{10,}|credential-canary-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/i; -const SECRETISH_GLOBAL = /sk-[a-z0-9]{10,}|credential-canary-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/gi; +const SECRETISH_GLOBAL = new RegExp(SECRETISH.source, "gi"); export function redactForArtifact(artifactClass: ArtifactClass, payload: unknown): unknown { if ( @@ -583,4 +583,4 @@ export function sanitizeDiagnostic(value: unknown): string { export function sanitizedJsonBytes(value: unknown): Uint8Array { return new TextEncoder().encode(jcsStringify(scrubValue(value, 0))); -} +} \ No newline at end of file From 0efe2c69514d3baefee686383fe740e4ecb37d83 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:11:42 +0200 Subject: [PATCH 24/24] fix(lab): surface non-EPIPE producer stdin errors --- src/lab/fabric/producer-isolate.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lab/fabric/producer-isolate.ts b/src/lab/fabric/producer-isolate.ts index a8bac8afe9..c808de0e12 100644 --- a/src/lab/fabric/producer-isolate.ts +++ b/src/lab/fabric/producer-isolate.ts @@ -171,8 +171,10 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); }); - child.stdin?.on("error", () => { - /* EPIPE after close is expected */ + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (settled || error.code === "EPIPE") return; + killChild(child); + finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); }); const payload = JSON.stringify({ @@ -244,4 +246,4 @@ export function fabricProducerIsolationLimits(): { totalTimeoutMs: number; inact totalTimeoutMs: FABRIC_LIMITS.totalTimeoutMs, inactivityTimeoutMs: FABRIC_LIMITS.inactivityTimeoutMs, }; -} +} \ No newline at end of file