diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index b6269ca4ba..06d1eddb30 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -26,7 +26,7 @@ independent review, blockers, and whether a later phase is authorized. | CL-04 | `feat/cl-04-lab-read-surfaces` | `68c71a4e9cdf882d812f09fd94783a28749db629` | NOT RECORDED | [#1378](https://github.com/lidge-jun/opencodex/pull/1378) | MERGED TO `dev` at `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | | CL-05 | `feat/cl-05-compatibility-matrix-ui` | `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | `2a159b8b7` (Models tab placement) | [#1384](https://github.com/lidge-jun/opencodex/pull/1384) | MERGED TO `dev` at `1072b9c39c48a4982229131613ac300560740742` | | CL-06 | `feat/cl-06-routing-profile-compatibility` | `1072b9c39c48a4982229131613ac300560740742` | `b96eae83f2a6d1654472aeeef84799070743aeb8` | [#1394](https://github.com/lidge-jun/opencodex/pull/1394) | MERGED TO `dev` at `b66e33ce7207d91014644d99317e456c992a3418`; ACCEPTED/CLOSED | - +| CL-07 | `feat/cl-07-task-effectiveness-producer` | `b66e33ce7207d91014644d99317e456c992a3418` | NOT RECORDED | [#1438](https://github.com/lidge-jun/opencodex/pull/1438) | DRAFT / PENDING CI; implementation head `ed8e0794189b214398d835e1af828905f29fbc53`; plan `007_cl07_task_effectiveness.md` | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -162,7 +162,20 @@ Claims cannot produce `PROBED`/`VERIFIED`. - CL-03: **ACCEPTED/CLOSED** via [#1352](https://github.com/lidge-jun/opencodex/pull/1352), merged to `dev` at `68c71a4e9cdf882d812f09fd94783a28749db629`. - CL-04: **MERGED** via #1378 at `d517161aeaa3a974ad3c0360ff0c97b03b4c4520`. - CL-05: **MERGED** via #1384 at `1072b9c39c48a4982229131613ac300560740742`. -- CL-06: **AUTHORIZED / IN PROGRESS** from CL-05 merge `1072b9c39c48a4982229131613ac300560740742`; branch `feat/cl-06-routing-profile-compatibility`; plan `006_cl06_routing_compatibility.md`. +- CL-06: **ACCEPTED/CLOSED** via [#1394](https://github.com/lidge-jun/opencodex/pull/1394), merged to `dev` at `b66e33ce7207d91014644d99317e456c992a3418`. +- CL-07: **AUTHORIZED / IN PROGRESS** from CL-06 merge `b66e33ce7207d91014644d99317e456c992a3418`; branch `feat/cl-07-task-effectiveness-producer`; plan `007_cl07_task_effectiveness.md`. +- CL-08: **not started**. + +## CL-07 start log + +- **Starting/base SHA:** `b66e33ce7207d91014644d99317e456c992a3418` (exact CL-06 merge #1394) +- **Branch:** `feat/cl-07-task-effectiveness-producer` +- **Scope:** bounded Lab-owned task-effectiveness producer for `fabric-core` / + `fabric-core.task.synthetic-patch@1.0.0`; `exact-tree-diff-v1` verifier; + scratch sandbox; observation ingestion with `executionMode: fabric`; catalog + discovery for the task layer. No general Agent Fabric product API. +- **Explicitly out of scope:** CL-08 automation/background execution; CL-06 + routing semantic changes; user repositories/prompts; arbitrary shell. ## CL-03 implementation log (2026-08-09) diff --git a/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md new file mode 100644 index 0000000000..35c818bc56 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/007_cl07_task_effectiveness.md @@ -0,0 +1,281 @@ +# CL-07 implementation record — Task-effectiveness evidence producer + +## Programme position + +| Field | Value | +|---|---| +| **Phase** | CL-07 | +| **Starting SHA** | `b66e33ce7207d91014644d99317e456c992a3418` (CL-06 merge #1394) | +| **Branch** | `feat/cl-07-task-effectiveness-producer` | +| **Target** | `lidge-jun/opencodex:dev` | +| **CL-08** | **Not started** (explicit non-goal) | + +## 0. Audit summary (repository reality) + +Inspected at starting SHA `b66e33ce7207d91014644d99317e456c992a3418`: + +| Area | Reality | +|---|---| +| `TaskSubjectV1` | Defined and validated (`src/lab/events/types.ts`, `validate.ts`); query DTO mapped | +| Agent Fabric product | **Absent**. Only `devlog/_plan/800_agent-fabric/` planning notes and CL-00 reserved consumer semantics | +| Lab ledger / artifacts / projection | Present; `task_effectiveness` already a legal evidence layer | +| CL-04 catalog | Protocol + live only — no fabric suite discovery yet | +| CL-05 matrix | Already lists `task_effectiveness` column; read-only | +| CL-06 routing | `requiredSuites.evidenceLayer` is **only** `protocol_conformance` \| `live_route_compatibility` | +| Live sandbox | Counter/env limits for probes; **not** a synthetic scratch-tree executor | +| `fabric-core` / `synthetic-patch` / `exact-tree-diff-v1` | Reserved in `020_scenario_contract_and_catalogue.md`; **no runtime implementation** | + +**Decision:** CL-07 implements a **bounded Lab-owned task producer** under `src/lab/fabric/`. It does **not** invent a general Agent Fabric platform, ACP/A2A orchestration, background grind, or user-worktree execution. + +## A. Frozen producer boundary + +### A.1 Subject identity + +Evidence uses existing `TaskSubjectV1` only (no alternate task identity): + +```text +subjectSchemaVersion 1 +subjectKind task +routeSubject RouteSubjectV1 # exact nested route from CL-03/CL-06 builders +taskClassId +taskClassVersion +taskFixtureDigest +verifierManifestDigest +fabricCompatibilityVersion +sandboxProfileDigest +``` + +`subjectId = subjectIdForSubject(taskSubject)` (existing digest helper). + +Any material change to route, task class, fixture, verifier, fabric compatibility version, or sandbox profile yields a distinct subject. Evidence must not reuse across subjects. + +### A.2 Producer outcome schema (`FabricTaskOutcomeV1`) + +Closed, fail-closed on unknown fields. Schema version `1`. + +Required fields: + +| Field | Role | +|---|---| +| `schemaVersion` | `1` | +| `taskClassId` / `taskClassVersion` | Exact class | +| `routeSubject` / `taskSubject` / `subjectId` | Exact identities | +| `taskFixtureDigest` | Fixture identity | +| `verifierManifestDigest` | Verifier identity | +| `fabricCompatibilityVersion` | Executor contract version | +| `sandboxProfileDigest` | Sandbox policy identity | +| `startedAt` / `completedAt` | Execution window (ms epoch integers) | +| `limits` | Declared ceilings | +| `usage` | Actual resource counters used for limit checks | +| `outcome` | Normalized: `pass` \| `fail` \| `blocked` \| `inconclusive` | +| `verifier` | Bounded `exact-tree-diff-v1` result | +| `failure` | Optional typed failure (`class`, `code`, `retryable`, `attribution`) | +| `artifactDigests` | Content-addressed digests only | +| `sourceRefs` | Optional safe IDs (request / route-decision / attempt) — never bodies | + +**Forbidden in outcome and artifacts:** user repositories, arbitrary file trees, prompts, hidden reasoning, credentials, env secrets, host paths, unrestricted logs/stdout/stderr, raw model transcripts, arbitrary response bodies. + +### A.3 V1 executable scope (only) + +| Item | Value | +|---|---| +| Suite | `fabric-core@1.0.0` | +| Scenario | `fabric-core.task.synthetic-patch@1.0.0` | +| Evidence layer | `task_effectiveness` | +| Execution mode | `fabric` | +| Verifier | `exact-tree-diff-v1` | +| Fixture | Lab-owned scratch with `src/value.txt` = UTF-8 `before\n`; requested final `after\n` | + +No additional task classes in this phase. + +### A.4 Limits (non-weakened) + +| Limit | Value | +|---|---| +| Files touched | 1 | +| Aggregate input/output | 64 KiB | +| Patch operations | 1 | +| Total timeout | 30 s | +| Inactivity timeout | 5 s | +| Aggregate artifacts | 1 MiB | +| Network | denied in scratch | +| User MCP | unavailable | +| Arbitrary shell | unavailable | +| User repository | unreachable | + +### A.5 Patch producer seam + +Execution does **not** embed a general coding agent. Production evidence requires a host-issued `TrustedFabricPatchExecutor` invoked through `runFabricSyntheticPatchTaskForRoute`, with `RouteSubjectV1` built from `routeContext` + `destination` via `buildRouteSubjectV1`. Patch producers run in an isolated Bun child with hard termination on timeout. + +- Tests use `runFabricSyntheticPatchTaskHarness` with closed `FabricHarnessProducerKind` values or fixture executor modules; harness outcomes are not persistable via `persistFabricRunResult`. +- A future live route adapter may call a provider **outside** the scratch sandbox and return only a validated `SyntheticPatchV1`; raw prompts/responses never enter Lab storage. +- CL-07 does **not** ship automatic background execution (CL-08). + +## B. Verifier: `exact-tree-diff-v1` + +Deterministic, no LLM. + +1. Walk only the bounded scratch root (no follow). +2. Reject symlinks, special files, path traversal (`..`, absolute, drive prefixes), unexpected paths. +3. Sort repository-relative POSIX paths by UTF-8 bytes. +4. Hash/read allowed file bytes under bounds. +5. Pass iff the sole change is `src/value.txt: before\n → after\n` with no add/delete/rename. +6. Emit bounded structured result: `{ verifierId, manifestDigest, passed, pathSummaries[], reason? }`. + +Verifier manifest bytes participate in `verifierManifestDigest` → `TaskSubjectV1`. Behavior changes require a new digest; historical observations are never reinterpreted with current bytes. + +## C. Sandbox / execution boundary + +Minimal deny-by-default scratch executor: + +- Create ephemeral Lab-owned directory under the Lab paths tree (not the user repo). +- Materialize fixture files only. +- Apply at most one validated patch operation via direct file write (no shell). +- Enforce byte/time/inactivity ceilings. +- Cleanup on success, failure, and timeout. +- Freeze `sandboxProfileDigest` from a versioned sandbox profile object. + +Reuse patterns from `src/lab/artifacts/secure-fs.ts` / live sandbox env stripping where applicable; do not reuse live provider network transport inside the scratch. + +## D. Route identity + +Nested `RouteSubjectV1` must come from existing CL-03/CL-06 builders (`buildRouteSubjectV1` / policy subject helpers). Approximate `provider/model` strings are forbidden. Optional `sourceRefs` may cite `routeDecisionId` / attempt IDs without copying request content. + +## E. Lab ingestion + +`observationFromFabricOutcome` / internal `persistFabricOutcome` (not public): + +- `evidenceLayer: "task_effectiveness"` +- `executionMode: "fabric"` +- Exact scenario/suite IDs + manifest digests + fixture digests +- Exact `TaskSubjectV1` + `subjectId` +- Assertions from verifier +- Typed `failure` attribution +- Bounded environment metadata +- Sanitized artifact refs via existing artifact store + +No second ledger, mutable task DB, or separate verdict store. JSONL canonical; SQLite rebuildable; verdicts projected. + +### E.1 Idempotency + +Event identity uses existing `assignEventId` content-addressing. Replaying the same outcome (same event payload identity) must not create contradictory evidence. A legitimate second attempt uses a distinct `attempt` / timing / outcome payload and remains distinct evidence. + +## F. Verdict / failure mapping + +Layer remains independent of protocol/live. + +| Condition | Observation outcome | Failure class → attribution | Projection effect | +|---|---|---|---| +| Verifier pass (current required task) | `pass` | — | May contribute to `PROBED`/`VERIFIED` per suite rule | +| Verifier semantic fail | `fail` | `behavioral_failure` → `route` | `DEGRADED` path per suite | +| Sandbox violation / containment | `blocked` or `inconclusive` | `sandbox_violation` → `harness`/`environment` | `BLOCKED` / none | +| Timeout / inactivity / budget | `blocked` | matching blocker → `environment` | none (not route incompatibility) | +| Harness/executor defect | `inconclusive` | `harness_failure` → `harness` | none | +| Malformed producer outcome | reject (no event) | — | — | +| Artifact integrity failure | reject or invalidate | `integrity_failure` → `harness` | invalidate | + +Registry/provider claims **cannot** create task-effectiveness `CLAIMED`. + +## G. Freshness / invalidation / artifacts + +Reuse existing Lab freshness, invalidation, sensitive purge, and rebuild. Missing historical contract artifacts make evidence unusable. + +Allowed artifacts: verifier summary, normalized tree-diff summary, bounded execution metadata, sanitized failure summary. Never file bodies, prompts, or raw logs. + +## H. Read surfaces / UI / CL-06 interaction + +- Extend CL-04 `queryLabCatalog` to discover `fabric-core` scenarios. +- Projection/query already accept `task_effectiveness` subjects. +- CL-05 matrix remains read-only; show task layer if already wired. +- **CL-06 unchanged:** do **not** add `task_effectiveness` to routing `requiredSuites`. `TaskSubjectV1` cannot be resolved pre-dispatch (task class/fixture/verifier/sandbox unknown until execution). Historical task evidence may appear in Lab reads; it must not silently alter production routing without a future explicit contract (out of CL-07 scope). + +## I. Rejected alternatives + +| Alternative | Why rejected | +|---|---| +| General Agent Fabric API / worktree runner | Out of scope; unsafe surface area | +| Shell-based patch apply | Violates no-arbitrary-shell; use direct write | +| Pre-dispatch routing on task suites | Subject unknown before execution | +| Numerical effectiveness score / leaderboard | Forbidden by CL-00 | +| Second ledger / mutable verified flag | Breaks Lab persistence authority | +| Embedding prompts in observations | Privacy violation | +| Weakening byte/time limits for tests | Contract non-negotiable | +| CL-08 auto/background probing | Explicit exclusion | + +## J. Security review checklist + +Scratch-root containment; symlink/path-traversal/special-file rejection; max files/bytes; artifact publication bounds; no network/MCP/shell in scratch; minimal env; no credentials/config exposure; cleanup; concurrent isolation; hostile patch paths. Fail closed. Adversarial tests required (§18 of programme request). + +## K. Implementation layout (proposed) + +```text +src/lab/fabric/ + constants.ts + types.ts + subject.ts + fixture.ts + sandbox-profile.ts + scratch.ts + patch.ts + verifier.ts + executor.ts + manifest.ts + observe.ts + index.ts +tests/lab-fabric-task.test.ts +``` + +## L. Validation plan + +Typecheck; focused fabric/subject/sandbox/verifier/observe/ledger/projection/read-surface tests; routing regressions proving CL-06 unchanged; privacy scan; hygiene; GUI only if touched. + +## M. Status + +- **Authorization:** IN PROGRESS from CL-06 merge `b66e33ce7207d91014644d99317e456c992a3418` +- **Implementation head:** pending push (authoritative route execution + isolated producer child) +- **Accepted head:** NOT YET +- **PR:** [#1438](https://github.com/lidge-jun/opencodex/pull/1438) (draft) + +### Authoritative route execution boundary + +- Production evidence uses `runFabricSyntheticPatchTaskForRoute({ routeContext, destination, patchExecutor })`. +- `RouteSubjectV1` is built only via `buildRouteSubjectV1(routeContext, destination)` — callers cannot supply an independent route identity. +- Patch production requires a host-issued `TrustedFabricPatchExecutor` (`createHostIssuedFabricPatchExecutor` in `src/lib/fabric-task-host.ts`). +- **Public ingestion:** `persistFabricRunResult` only; it rejects harness runs (`executionAuthority !== "trusted_route"`). +- `persistFabricOutcome` is internal to `observe.ts` and is **not** exported from the fabric public surface. +- Harness runs (`runFabricSyntheticPatchTaskHarness`) use closed `FabricHarnessProducerKind` values only; they cannot create production ledger evidence. + +### Child isolation / IPC / timeouts + +- Patch producers run in a dedicated Bun child (`producer-child.ts`) spawned by `producer-isolate.ts` with minimal env (`TZ`, `NO_COLOR`, `OCX_FABRIC_SCRATCH_ROOT`). +- Parent↔child protocol is newline-delimited JSON on stdout only (`activity`, `result`, `error` in `producer-protocol.ts`). Arbitrary logging is not mixed into protocol output. +- Parent owns **both** total and inactivity timeouts; both terminate the child via `SIGKILL`. Classification: `timeout` vs `inactivity_timeout`. +- Child stdout is capped at 64 KiB protocol bytes; stderr diagnostic capture capped at 4 KiB. Exceeding protocol limits → `budget_exhausted` and child kill. +- `lastActivityAt` is authoritative in the parent; child `reportActivity()` emits `activity` IPC messages that reset the inactivity deadline. +- `infinite_sync` harness disables inactivity ceiling extension so synchronous CPU spin is classified under total timeout. + +### Sandbox enforcement (honest scope) + +- Scratch containment, symlink/path-traversal/special-file rejection, byte/file limits, and cleanup are enforced in the parent via `scratch.ts`, `patch.ts` (`assertSafeRelativePosixPath`), and `applySyntheticPatch`. +- Child isolation strips proxy env vars on the parent path and runs producers in a separate process with minimal env — **not** an OS-level network/shell sandbox. +- Host-issued executor modules may still perform direct host filesystem operations outside the scratch tree; that is outside the scratch-apply boundary and is not claimed as blocked. +- Declared deny flags (`fabricDeclaredSandboxPolicy`) document intent; runtime enforcement matches the scratch/patch/verifier containment above. + +### Failure attribution + +- Semantic verifier mismatch → `fail` / `behavioral_failure` / `route`. +- Sandbox/containment (`FabricTaskError` from scratch/verifier/patch infrastructure) → `blocked` or `inconclusive` / `sandbox_violation` / `harness` or `environment` — never route-attributed `behavioral_failure`. +- Timeouts / budget exhaustion → `blocked` / `environment`. +- Harness defects → `inconclusive` / `harness`. + +### Inactivity accounting + +- `inactiveMs` = `completedAt - lastActivityAt` where `lastActivityAt` is updated only from parent-received `activity` IPC (or initial start). + +### Local validation (blocker fix head) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-fabric-task.test.ts`: 46/46 passed +- `bun run privacy:scan`: passed +- Windows sqlite projection flakes in `lab-evidence-ledger.test.ts` (EBUSY) — environmental, not CL-07 +- CL-08: **not started** diff --git a/src/lab/artifacts/sanitize.ts b/src/lab/artifacts/sanitize.ts index afb6386286..8f64ed14ca 100644 --- a/src/lab/artifacts/sanitize.ts +++ b/src/lab/artifacts/sanitize.ts @@ -8,8 +8,8 @@ import { jcsStringify } from "../digest"; import { redactSecretString } from "../../lib/redact"; const FORBIDDEN_KEY = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|content|body|url|hostname|baseUrl|path|account|alias)$/i; -const SECRETISH = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/i; -const SECRETISH_GLOBAL = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/gi; +const SECRETISH = /sk-[a-z0-9]{10,}|credential-canary-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/i; +const SECRETISH_GLOBAL = new RegExp(SECRETISH.source, "gi"); export function redactForArtifact(artifactClass: ArtifactClass, payload: unknown): unknown { if ( @@ -583,4 +583,4 @@ export function sanitizeDiagnostic(value: unknown): string { export function sanitizedJsonBytes(value: unknown): Uint8Array { return new TextEncoder().encode(jcsStringify(scrubValue(value, 0))); -} +} \ No newline at end of file diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 30cc0c9637..affc5e4a65 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -9,7 +9,11 @@ export type VerificationRole = "required" | "supplemental" | "negative_control"; export type FailureClassification = | "harness_failure" | "timeout" + | "inactivity_timeout" | "budget_exhausted" + | "sandbox_violation" + | "malformed_producer_outcome" + | "layer_subject_mismatch" | "protocol_failure" | "capability_failure" | "behavioral_failure" @@ -20,6 +24,25 @@ export type FailureClassification = | "provider_transient" | "inconclusive"; +export const FAILURE_CLASSIFICATIONS = [ + "harness_failure", + "timeout", + "inactivity_timeout", + "budget_exhausted", + "sandbox_violation", + "malformed_producer_outcome", + "layer_subject_mismatch", + "protocol_failure", + "capability_failure", + "behavioral_failure", + "authentication_blocked", + "quota_blocked", + "region_blocked", + "network_failure", + "provider_transient", + "inconclusive", +] as const satisfies readonly FailureClassification[]; + export interface FixtureRecord { id: string; role: "client_request" | "upstream_response" | "adapter_vector" | "synthetic_tool"; diff --git a/src/lab/events/types.ts b/src/lab/events/types.ts index 69ed120db0..293d9a7c5d 100644 --- a/src/lab/events/types.ts +++ b/src/lab/events/types.ts @@ -9,6 +9,7 @@ import type { ObservationOutcome, PurgeAction, } from "../constants"; +import type { FailureClassification } from "../conformance/types"; export interface LabProducer { name: string; @@ -90,7 +91,7 @@ export interface AssertionRecordV1 { } export interface FailureRecordV1 { - class: string; + class: FailureClassification; code: string; retryable: boolean; attribution: "opencodex" | "route" | "environment" | "harness"; diff --git a/src/lab/events/validate.ts b/src/lab/events/validate.ts index 1bf3c137fc..df9d15f95a 100644 --- a/src/lab/events/validate.ts +++ b/src/lab/events/validate.ts @@ -29,6 +29,7 @@ import { jcsStringify, subjectIdForSubject, } from "../digest"; +import { FAILURE_CLASSIFICATIONS } from "../conformance/types"; import type { ArtifactRefV1, ClaimCapabilityFactsV1, @@ -408,7 +409,7 @@ function validateObservation(raw: Record): ObservationEvent { if (raw.failure !== undefined) { if (!isPlainObject(raw.failure)) throw new LabValidationError("invalid_failure", "failure"); event.failure = { - class: assertString(raw.failure.class, "failure.class"), + class: assertClosed(raw.failure.class, "failure.class", FAILURE_CLASSIFICATIONS), code: assertString(raw.failure.code, "failure.code"), retryable: raw.failure.retryable === true, attribution: assertClosed(raw.failure.attribution, "failure.attribution", [ diff --git a/src/lab/fabric/constants.ts b/src/lab/fabric/constants.ts new file mode 100644 index 0000000000..d15e814e87 --- /dev/null +++ b/src/lab/fabric/constants.ts @@ -0,0 +1,40 @@ +/** CL-07 frozen fabric-core / synthetic-patch constants. */ + +export const FABRIC_SUITE_ID = "fabric-core"; +export const FABRIC_SUITE_VERSION = "1.0.0"; +export const FABRIC_SCENARIO_ID = "fabric-core.task.synthetic-patch"; +export const FABRIC_SCENARIO_VERSION = "1.0.0"; +export const FABRIC_TASK_CLASS_ID = "fabric-core.task.synthetic-patch"; +export const FABRIC_TASK_CLASS_VERSION = "1.0.0"; +export const FABRIC_COMPATIBILITY_VERSION = "ocx-fabric-producer-v1"; +export const FABRIC_VERIFIER_ID = "exact-tree-diff-v1"; +export const FABRIC_EVIDENCE_LAYER = "task_effectiveness" as const; + +export const SYNTHETIC_VALUE_PATH = "src/value.txt"; +export const SYNTHETIC_BEFORE_UTF8 = "before\n"; +export const SYNTHETIC_AFTER_UTF8 = "after\n"; + +export const FABRIC_LIMITS = Object.freeze({ + maxFiles: 1, + /** Upper bound for scratch-tree walks (verifier may see unexpected extras). */ + maxScratchFiles: 8, + maxAggregateIoBytes: 64 * 1024, + maxPatchOperations: 1, + totalTimeoutMs: 30_000, + inactivityTimeoutMs: 5_000, + aggregateArtifactBytes: 1024 * 1024, + maxArtifacts: 8, +}); + +export const SANDBOX_PROFILE_V1 = Object.freeze({ + schemaVersion: 1, + profileId: "lab-fabric-scratch-v1", + allowNetwork: false, + allowUserMcp: false, + allowShell: false, + allowUserRepository: false, + followSymlinks: false, + allowSpecialFiles: false, + scratchOnly: true, + limits: FABRIC_LIMITS, +}); diff --git a/src/lab/fabric/executor.ts b/src/lab/fabric/executor.ts new file mode 100644 index 0000000000..66c9e2eb25 --- /dev/null +++ b/src/lab/fabric/executor.ts @@ -0,0 +1,492 @@ +import type { FailureRecordV1, RouteSubjectV1 } from "../events/types"; +import { labSandboxEnvironment, rejectProxyEnvironment } from "../live/sandbox"; +import type { LabDestinationV1, LabRouteContext } from "../live/types"; +import { buildRouteSubjectV1 } from "../subject/route-subject"; +import { isTrustedFabricPatchExecutor } from "../../lib/fabric-task-execution-authority"; +import { + FABRIC_COMPATIBILITY_VERSION, + FABRIC_LIMITS, + FABRIC_TASK_CLASS_ID, + FABRIC_TASK_CLASS_VERSION, + FABRIC_VERIFIER_ID, + SANDBOX_PROFILE_V1, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, +} from "./constants"; +import { applySyntheticPatch, parseSyntheticPatchV1 } from "./patch"; +import { fabricProducerIsolationLimits, runIsolatedFabricProducer } from "./producer-isolate"; +import { assertNotUnderUserRepo, createSyntheticScratch, type ScratchTree } from "./scratch"; +import { + buildTaskSubjectV1, + sandboxProfileDigest, + taskFixtureDigest, + taskSubjectId, + verifierManifestDigest, +} from "./subject"; +import type { + FabricExecutionAuthority, + FabricHarnessProducerKind, + FabricTaskOutcomeV1, + FabricTaskRunResult, + FabricUsageV1, + SyntheticPatchV1, + TrustedFabricPatchExecutor, +} from "./types"; +import { FabricTaskError } from "./types"; +import { verifyExactTreeDiffV1 } from "./verifier"; + +/** Options for harness-only fabric task runs (not persistable as production evidence). */ +export interface RunFabricTaskHarnessOptions { + routeSubject: RouteSubjectV1; + harnessKind: FabricHarnessProducerKind; + configDir?: string; + userRepoRoot?: string; + sourceRefs?: string[]; + now?: () => number; +} + +/** Options for authoritative route-bound fabric task execution. */ +export interface RunFabricTaskForRouteOptions { + routeContext: LabRouteContext; + destination: LabDestinationV1; + patchExecutor: TrustedFabricPatchExecutor; + configDir?: string; + userRepoRoot?: string; + sourceRefs?: string[]; + now?: () => number; +} + +/** Map a FabricTaskError into a ledger failure record. */ +function failureFromError(error: FabricTaskError): FailureRecordV1 { + return { + class: error.code, + code: error.code, + retryable: error.code === "timeout" || error.code === "inactivity_timeout" || error.code === "budget_exhausted", + attribution: error.attribution, + }; +} + +/** Map a failure record to the observation outcome kind for persistence. */ +function outcomeFromFailure(failure: FailureRecordV1): FabricTaskOutcomeV1["outcome"] { + if (failure.class === "behavioral_failure") return "fail"; + if (failure.attribution === "harness") return "inconclusive"; + return "blocked"; +} + +/** Closed successful patch for deterministic harness injection. */ +export function correctSyntheticPatch(): SyntheticPatchV1 { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} + +/** + * Authoritative CL-07 route-bound fabric task execution. + * Route identity is derived from routeContext + destination; patch production runs in an isolated child. + */ +export async function runFabricSyntheticPatchTaskForRoute( + options: RunFabricTaskForRouteOptions, +): Promise { + if (!isTrustedFabricPatchExecutor(options.patchExecutor)) { + throw new FabricTaskError("untrusted fabric patch executor", "sandbox_violation", "harness"); + } + const routeSubject = buildRouteSubjectV1(options.routeContext, options.destination, options.configDir); + return runFabricSyntheticPatchTaskInternal({ + routeSubject, + routeContext: options.routeContext, + destination: options.destination, + patchExecutor: options.patchExecutor, + harnessKind: undefined, + executionAuthority: "trusted_route", + configDir: options.configDir, + userRepoRoot: options.userRepoRoot, + sourceRefs: options.sourceRefs, + now: options.now, + }); +} + +/** Harness-only fabric task execution; outcomes are not persistable as production evidence. */ +export async function runFabricSyntheticPatchTaskHarness( + options: RunFabricTaskHarnessOptions, +): Promise { + return runFabricSyntheticPatchTaskInternal({ + routeSubject: options.routeSubject, + harnessKind: options.harnessKind, + executionAuthority: "harness", + configDir: options.configDir, + userRepoRoot: options.userRepoRoot, + sourceRefs: options.sourceRefs, + now: options.now, + }); +} + +/** + * @deprecated Use runFabricSyntheticPatchTaskHarness or runFabricSyntheticPatchTaskForRoute. + */ +export async function runFabricSyntheticPatchTask( + options: Omit & { + producePatch?: never; + harnessKind?: RunFabricTaskHarnessOptions["harnessKind"]; + }, +): Promise { + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: options.routeSubject, + harnessKind: options.harnessKind ?? "deterministic_correct", + configDir: options.configDir, + userRepoRoot: options.userRepoRoot, + sourceRefs: options.sourceRefs, + now: options.now, + }); + return result.outcome; +} + +async function runFabricSyntheticPatchTaskInternal(input: { + routeSubject: RouteSubjectV1; + routeContext?: LabRouteContext; + destination?: LabDestinationV1; + patchExecutor?: TrustedFabricPatchExecutor; + harnessKind?: FabricHarnessProducerKind; + executionAuthority: FabricExecutionAuthority; + configDir?: string; + userRepoRoot?: string; + sourceRefs?: string[]; + now?: () => number; +}): Promise { + rejectProxyEnvironment(); + labSandboxEnvironment(); + + const startedAt = input.now?.() ?? Date.now(); + const fixtureDigest = taskFixtureDigest(); + const verifierDigest = verifierManifestDigest(); + const sandboxDigest = sandboxProfileDigest(); + const taskSubject = buildTaskSubjectV1({ + routeSubject: input.routeSubject, + taskFixtureDigest: fixtureDigest, + verifierManifestDigest: verifierDigest, + sandboxProfileDigest: sandboxDigest, + fabricCompatibilityVersion: FABRIC_COMPATIBILITY_VERSION, + }); + const subjectId = taskSubjectId(taskSubject); + + const usage: FabricUsageV1 = { + inputBytes: Buffer.byteLength(SYNTHETIC_BEFORE_UTF8, "utf8"), + outputBytes: 0, + patchOperations: 0, + filesTouched: 0, + artifactBytes: 0, + elapsedMs: 0, + inactiveMs: 0, + }; + + let scratch: ScratchTree | undefined; + let producerCompletedAt = startedAt; + let lastActivityAt = startedAt; + + try { + scratch = createSyntheticScratch(input.configDir); + if (input.userRepoRoot) { + assertNotUnderUserRepo(scratch.root, input.userRepoRoot); + } + + const isolation = fabricProducerIsolationLimits(); + const inactivityTimeoutMs = + input.harnessKind === "infinite_sync" + ? isolation.totalTimeoutMs + 1_000 + : isolation.inactivityTimeoutMs; + let patchRaw: unknown; + try { + if (input.harnessKind) { + lastActivityAt = input.now?.() ?? Date.now(); + const isolated = await runIsolatedFabricProducer({ + harnessKind: input.harnessKind, + scratchRoot: scratch.root, + totalTimeoutMs: isolation.totalTimeoutMs, + inactivityTimeoutMs, + now: input.now, + }); + patchRaw = isolated.patch; + lastActivityAt = isolated.lastActivityAt; + producerCompletedAt = input.now?.() ?? Date.now(); + } else if (input.patchExecutor && input.routeContext && input.destination) { + lastActivityAt = input.now?.() ?? Date.now(); + const isolated = await runIsolatedFabricProducer({ + executorModulePath: input.patchExecutor.executorModulePath, + scratchRoot: scratch.root, + totalTimeoutMs: isolation.totalTimeoutMs, + inactivityTimeoutMs: isolation.inactivityTimeoutMs, + executorInput: { + routeContext: input.routeContext, + destination: input.destination, + routeSubject: input.routeSubject, + scratchRoot: scratch.root, + reportActivity: () => undefined, + signal: new AbortController().signal, + }, + now: input.now, + }); + patchRaw = isolated.patch; + lastActivityAt = isolated.lastActivityAt; + producerCompletedAt = input.now?.() ?? Date.now(); + } else { + throw new FabricTaskError("fabric task run missing producer", "harness_failure", "harness"); + } + } catch (error) { + const completedAt = input.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: infrastructureVerifier(verifierDigest, error.code), + failure, + sourceRefs: input.sourceRefs, + }), + }; + } + throw error; + } + + let patch; + let applied; + try { + patch = parseSyntheticPatchV1(patchRaw); + applied = applySyntheticPatch(scratch.root, patch); + } catch (error) { + const completedAt = input.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: infrastructureVerifier(verifierDigest, error.code), + failure, + sourceRefs: input.sourceRefs, + }), + }; + } + throw error; + } + usage.outputBytes = applied.bytesWritten; + usage.patchOperations = applied.patchOperations; + usage.filesTouched = applied.filesTouched; + + let verifier; + try { + verifier = verifyExactTreeDiffV1(scratch.root); + } catch (error) { + const completedAt = input.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: infrastructureVerifier(verifierDigest, error.code), + failure, + sourceRefs: input.sourceRefs, + }), + }; + } + throw error; + } + + const completedAt = input.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + + if (verifier.passed) { + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: "pass", + verifier, + sourceRefs: input.sourceRefs, + }), + }; + } + + const failure: FailureRecordV1 = { + class: "behavioral_failure", + code: verifier.reason ?? "verifier_failed", + retryable: false, + attribution: "route", + }; + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: "fail", + verifier, + failure, + sourceRefs: input.sourceRefs, + }), + }; + } catch (error) { + const completedAt = input.now?.() ?? Date.now(); + usage.elapsedMs = completedAt - startedAt; + usage.inactiveMs = Math.max(0, completedAt - lastActivityAt); + if (error instanceof FabricTaskError) { + const failure = failureFromError(error); + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: outcomeFromFailure(failure), + verifier: infrastructureVerifier(verifierDigest, error.code), + failure, + sourceRefs: input.sourceRefs, + }), + }; + } + const failure: FailureRecordV1 = { + class: "harness_failure", + code: "harness_failure", + retryable: false, + attribution: "harness", + }; + return { + executionAuthority: input.executionAuthority, + outcome: sealOutcome({ + taskSubject, + subjectId, + fixtureDigest, + verifierDigest, + sandboxDigest, + startedAt, + completedAt, + usage, + outcome: "inconclusive", + verifier: infrastructureVerifier(verifierDigest, "harness_failure"), + failure, + sourceRefs: input.sourceRefs, + }), + }; + } finally { + scratch?.cleanup(); + } +} + +function infrastructureVerifier(manifestDigest: string, reason: string): FabricTaskOutcomeV1["verifier"] { + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [], + reason, + }; +} + +/** Assemble an immutable FabricTaskOutcomeV1 from executor state. */ +function sealOutcome(input: { + taskSubject: FabricTaskOutcomeV1["taskSubject"]; + subjectId: string; + fixtureDigest: string; + verifierDigest: string; + sandboxDigest: string; + startedAt: number; + completedAt: number; + usage: FabricUsageV1; + outcome: FabricTaskOutcomeV1["outcome"]; + verifier: FabricTaskOutcomeV1["verifier"]; + failure?: FailureRecordV1; + sourceRefs?: string[]; +}): FabricTaskOutcomeV1 { + return { + schemaVersion: 1, + taskClassId: FABRIC_TASK_CLASS_ID, + taskClassVersion: FABRIC_TASK_CLASS_VERSION, + routeSubject: input.taskSubject.routeSubject, + taskSubject: input.taskSubject, + subjectId: input.subjectId, + taskFixtureDigest: input.fixtureDigest, + verifierManifestDigest: input.verifierDigest, + fabricCompatibilityVersion: FABRIC_COMPATIBILITY_VERSION, + sandboxProfileDigest: input.sandboxDigest, + startedAt: input.startedAt, + completedAt: input.completedAt, + limits: { ...FABRIC_LIMITS }, + usage: { ...input.usage }, + outcome: input.outcome, + verifier: { + ...input.verifier, + pathSummaries: [...input.verifier.pathSummaries], + }, + ...(input.failure ? { failure: { ...input.failure } } : {}), + artifactDigests: [], + ...(input.sourceRefs ? { sourceRefs: [...input.sourceRefs] } : {}), + }; +} + +/** Declared sandbox policy for the frozen scratch producer (not runtime enforcement). */ +export function fabricDeclaredSandboxPolicy(): { + network: boolean; + userMcp: boolean; + arbitraryShell: boolean; + userRepository: boolean; +} { + return { + network: SANDBOX_PROFILE_V1.allowNetwork, + userMcp: SANDBOX_PROFILE_V1.allowUserMcp, + arbitraryShell: SANDBOX_PROFILE_V1.allowShell, + userRepository: SANDBOX_PROFILE_V1.allowUserRepository, + }; +} + +/** @deprecated Use fabricDeclaredSandboxPolicy. */ +export const fabricScratchCapabilityProbe = fabricDeclaredSandboxPolicy; diff --git a/src/lab/fabric/index.ts b/src/lab/fabric/index.ts new file mode 100644 index 0000000000..e6f9acca82 --- /dev/null +++ b/src/lab/fabric/index.ts @@ -0,0 +1,80 @@ +/** CL-07 fabric module public surface (explicit allowlist re-exports). */ +export { + FABRIC_SUITE_ID, + FABRIC_SUITE_VERSION, + FABRIC_SCENARIO_ID, + FABRIC_SCENARIO_VERSION, + FABRIC_TASK_CLASS_ID, + FABRIC_TASK_CLASS_VERSION, + FABRIC_COMPATIBILITY_VERSION, + FABRIC_VERIFIER_ID, + FABRIC_EVIDENCE_LAYER, + SYNTHETIC_VALUE_PATH, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_AFTER_UTF8, + FABRIC_LIMITS, + SANDBOX_PROFILE_V1, +} from "./constants"; +export type { + FabricOutcomeKind, + FabricFailureClass, + FabricExecutionAuthority, + FabricLimitsV1, + SyntheticPatchOperationV1, + SyntheticPatchV1, + FabricUsageV1, + ExactTreeDiffPathSummaryV1, + ExactTreeDiffResultV1, + FabricTaskOutcomeV1, + FabricTaskRunResult, + FabricHarnessProducerKind, + FabricPatchExecutorInput, + FabricPatchExecutor, + TrustedFabricPatchExecutor, +} from "./types"; +export { FabricTaskError } from "./types"; +export { + sandboxProfileDigest, + verifierManifestObject, + verifierManifestDigest, + taskFixtureObject, + taskFixtureDigest, + buildTaskSubjectV1, + taskSubjectId, +} from "./subject"; +export { + assertSafeRelativePosixPath, + resolveInsideScratch, + createSyntheticScratch, + assertNotUnderUserRepo, +} from "./scratch"; +export type { ScratchTree, WalkedFile } from "./scratch"; +export { parseSyntheticPatchV1 } from "./patch"; +export { verifyExactTreeDiffV1 } from "./verifier"; +export { + correctSyntheticPatch, + runFabricSyntheticPatchTask, + runFabricSyntheticPatchTaskForRoute, + runFabricSyntheticPatchTaskHarness, + fabricDeclaredSandboxPolicy, + fabricScratchCapabilityProbe, +} from "./executor"; +export type { + RunFabricTaskHarnessOptions, + RunFabricTaskForRouteOptions, +} from "./executor"; +export { + loadFabricCaseAuthority, + discoverFabricScenarios, + expandFabricScenario, + expandFabricSuiteManifest, + fabricScenarioManifestDigest, + fabricSuiteManifestDigest, +} from "./manifest"; +export type { FabricCaseAuthority } from "./manifest"; +export { + assertFabricOutcomeV1, + observationFromFabricOutcome, + persistFabricRunResult, +} from "./observe"; +export type { PersistFabricOptions, PersistedFabricObservation } from "./observe"; diff --git a/src/lab/fabric/manifest.ts b/src/lab/fabric/manifest.ts new file mode 100644 index 0000000000..9e0a1ef806 --- /dev/null +++ b/src/lab/fabric/manifest.ts @@ -0,0 +1,222 @@ +import type { CaseRecord, VerificationRole } from "../conformance/types"; +import { fixtureDigest, scenarioManifestDigest, suiteManifestDigest } from "../digest"; +import { + FABRIC_EVIDENCE_LAYER, + FABRIC_LIMITS, + FABRIC_SCENARIO_ID, + FABRIC_SCENARIO_VERSION, + FABRIC_SUITE_ID, + FABRIC_SUITE_VERSION, + FABRIC_TASK_CLASS_ID, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, +} from "./constants"; +import { verifierManifestDigest } from "./subject"; + +/** In-memory authority for the frozen CL-07 fabric scenario catalogue. */ +export interface FabricCaseAuthority { + schemaVersion: number; + sourceCommit: string; + assertionDslVersion: string; + evidenceSchemaVersion: string; + manifestDefaults: { + version: string; + suiteVersion: string; + evidenceLayer: typeof FABRIC_EVIDENCE_LAYER; + verificationRole: VerificationRole; + freshness: { maxAgeMs: number | null }; + executionLimits: Record; + artifactPolicy: Record; + }; + cases: CaseRecord[]; +} + +const FABRIC_SOURCE_COMMIT = "b66e33ce7207d91014644d99317e456c992a3418"; +const FIXTURE_BYTES_UTF8 = JSON.stringify({ + path: SYNTHETIC_VALUE_PATH, + before: SYNTHETIC_BEFORE_UTF8, + after: SYNTHETIC_AFTER_UTF8, +}); + +/** Frozen CL-07 fabric scenario authority loaded from compiled constants. */ +export function loadFabricCaseAuthority(): FabricCaseAuthority { + const fixtureBytes = new TextEncoder().encode(FIXTURE_BYTES_UTF8); + const digest = fixtureDigest(fixtureBytes); + return { + schemaVersion: 1, + sourceCommit: FABRIC_SOURCE_COMMIT, + assertionDslVersion: "1.0.0", + evidenceSchemaVersion: "1.0.0", + manifestDefaults: { + version: FABRIC_SCENARIO_VERSION, + suiteVersion: FABRIC_SUITE_VERSION, + evidenceLayer: FABRIC_EVIDENCE_LAYER, + verificationRole: "required", + freshness: { maxAgeMs: 2_592_000_000 }, + executionLimits: { + totalTimeoutMs: FABRIC_LIMITS.totalTimeoutMs, + inactivityTimeoutMs: FABRIC_LIMITS.inactivityTimeoutMs, + maxInputBytes: FABRIC_LIMITS.maxAggregateIoBytes, + maxOutputBytes: FABRIC_LIMITS.maxAggregateIoBytes, + maxArtifactBytes: FABRIC_LIMITS.aggregateArtifactBytes, + maxRequests: 0, + maxToolCalls: 0, + maxOutputTokens: null, + connectTimeoutMs: null, + firstByteTimeoutMs: null, + }, + artifactPolicy: { + retain: ["verifier_summary", "normalized_tree_diff", "execution_metadata"], + forbid: ["file_bodies", "prompts", "raw_logs"], + }, + }, + cases: [{ + id: FABRIC_SCENARIO_ID, + suite: FABRIC_SUITE_ID, + capability: FABRIC_TASK_CLASS_ID, + verificationRole: "required", + requirements: { + inboundProtocols: [], + upstreamProtocols: [], + surfaces: [], + requiredClaims: [], + requiredHarnessFeatures: ["fabric-scratch-v1"], + platforms: ["*"], + routePreconditions: ["exact-route-subject"], + }, + fixture: { + id: "fabric-core.task.synthetic-patch.fixture", + role: "adapter_vector", + mediaType: "application/json", + bytesUtf8: FIXTURE_BYTES_UTF8, + digest, + }, + assertions: [{ + id: "exact-tree-diff-pass", + operator: "equals", + selector: "/verifier/passed", + expected: true, + required: true, + }], + }], + }; +} + +/** List fabric scenarios, optionally filtered by suite id. */ +export function discoverFabricScenarios( + authority: FabricCaseAuthority = loadFabricCaseAuthority(), + suites?: readonly string[], +): CaseRecord[] { + return authority.cases.filter((row) => !suites || suites.includes(row.suite)); +} + +/** Expand a fabric case into the canonical scenario manifest object. */ +export function expandFabricScenario( + caseRecord: CaseRecord, + authority: FabricCaseAuthority = loadFabricCaseAuthority(), +): Record { + const defaults = authority.manifestDefaults; + return { + schemaVersion: authority.schemaVersion, + id: caseRecord.id, + version: defaults.version, + suite: { + id: caseRecord.suite, + version: defaults.suiteVersion, + evidenceLayer: defaults.evidenceLayer, + }, + evidenceLayer: defaults.evidenceLayer, + capability: caseRecord.capability, + verificationRole: caseRecord.verificationRole ?? defaults.verificationRole, + requirements: caseRecord.requirements, + fixtures: [{ + id: caseRecord.fixture.id, + role: caseRecord.fixture.role, + mediaType: caseRecord.fixture.mediaType, + digest: caseRecord.fixture.digest, + byteLength: Buffer.byteLength(caseRecord.fixture.bytesUtf8, "utf8"), + syntheticMarker: "ocx-lab-synthetic-v1", + provenance: { + kind: "lab_authored", + authority: "007_cl07_task_effectiveness.md", + sourceCommit: authority.sourceCommit, + }, + }], + executionLimits: defaults.executionLimits, + assertions: caseRecord.assertions, + failureRules: [ + { + id: "verifier-fail", + match: ["behavioral_failure"], + classification: "behavioral_failure", + verdictEffect: "degraded", + retry: "never", + expected: false, + }, + { + id: "sandbox-block", + match: ["sandbox_violation"], + classification: "sandbox_violation", + verdictEffect: "none", + retry: "bounded", + expected: false, + }, + { + id: "timeout-block", + match: ["timeout", "inactivity_timeout", "budget_exhausted"], + classification: "timeout", + verdictEffect: "none", + retry: "bounded", + expected: false, + }, + ], + artifactPolicy: defaults.artifactPolicy, + freshness: defaults.freshness, + verifierManifestDigest: verifierManifestDigest(), + }; +} + +/** Expand a fabric suite id into the canonical suite manifest object. */ +export function expandFabricSuiteManifest( + suiteId: string = FABRIC_SUITE_ID, + authority: FabricCaseAuthority = loadFabricCaseAuthority(), +): Record { + const cases = authority.cases.filter((row) => row.suite === suiteId); + if (cases.length === 0) throw new Error(`unknown fabric suite ${suiteId}`); + const defaults = authority.manifestDefaults; + return { + schemaVersion: 1, + id: suiteId, + version: defaults.suiteVersion, + evidenceLayer: defaults.evidenceLayer, + capability: cases[0]!.capability, + assertionDslVersion: authority.assertionDslVersion, + evidenceSchemaVersion: authority.evidenceSchemaVersion, + freshness: defaults.freshness, + contradictionRule: "newest-required-observation-v1", + scenarios: cases.map((caseRecord) => ({ + id: caseRecord.id, + version: defaults.version, + role: caseRecord.verificationRole ?? defaults.verificationRole, + manifestDigest: scenarioManifestDigest(expandFabricScenario(caseRecord, authority)), + })), + verificationRule: "all-applicable-required-pass-v1", + }; +} + +/** Digest of the expanded fabric scenario manifest for a case. */ +export function fabricScenarioManifestDigest( + caseRecord: CaseRecord = loadFabricCaseAuthority().cases[0]!, + authority: FabricCaseAuthority = loadFabricCaseAuthority(), +): string { + return scenarioManifestDigest(expandFabricScenario(caseRecord, authority)); +} + +/** Digest of the expanded fabric suite manifest. */ +export function fabricSuiteManifestDigest( + suiteId: string = FABRIC_SUITE_ID, + authority: FabricCaseAuthority = loadFabricCaseAuthority(), +): string { + return suiteManifestDigest(expandFabricSuiteManifest(suiteId, authority)); +} diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts new file mode 100644 index 0000000000..7e93d13fd3 --- /dev/null +++ b/src/lab/fabric/observe.ts @@ -0,0 +1,454 @@ +import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; +import { sanitizeDiagnostic, truncateUtf8 } from "../artifacts/sanitize"; +import { + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + LAB_PRODUCER_VERSION, + OBSERVATION_LIMIT_NAMES, + OUTCOMES, +} from "../constants"; +import { FAILURE_CLASSIFICATIONS } from "../conformance/types"; +import { fixtureDigest, isSha256Hex, jcsStringify } from "../digest"; +import type { ObservationEvent, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; +import { LabValidationError } from "../events/errors"; +import { assignEventId, validateSubject } from "../events/validate"; +import { appendLabEventIfAbsent } from "../ledger/store"; +import { ensureLabDirs } from "../paths"; +import { + FABRIC_EVIDENCE_LAYER, + FABRIC_SCENARIO_ID, + FABRIC_SCENARIO_VERSION, + FABRIC_SUITE_ID, + FABRIC_SUITE_VERSION, + FABRIC_LIMITS, + FABRIC_VERIFIER_ID, +} from "./constants"; +import { + expandFabricScenario, + expandFabricSuiteManifest, + fabricScenarioManifestDigest, + fabricSuiteManifestDigest, + loadFabricCaseAuthority, +} from "./manifest"; +import type { FabricLimitsV1, FabricTaskOutcomeV1, FabricTaskRunResult } from "./types"; +import { FabricTaskError } from "./types"; + +/** CL-07 fabric task outcome validation and observation persistence. */ + +/** Options for building or persisting a fabric observation event. */ +export interface PersistFabricOptions { + configDir?: string; + recordedAt?: number; + producerVersion?: string; + artifactStore?: ArtifactStore; + attempt?: number; +} + +/** Result of appending a fabric observation to the compatibility ledger. */ +export interface PersistedFabricObservation { + event: ObservationEvent; + ledgerPath: string; +} + +const OUTCOME_KEYS = new Set([ + "schemaVersion", + "taskClassId", + "taskClassVersion", + "routeSubject", + "taskSubject", + "subjectId", + "taskFixtureDigest", + "verifierManifestDigest", + "fabricCompatibilityVersion", + "sandboxProfileDigest", + "startedAt", + "completedAt", + "limits", + "usage", + "outcome", + "verifier", + "failure", + "artifactDigests", + "sourceRefs", +]); + +const VERIFIER_KEYS = new Set(["verifierId", "manifestDigest", "passed", "pathSummaries", "reason"]); +const PATH_SUMMARY_KEYS = new Set(["path", "kind", "beforeDigest", "afterDigest", "reason"]); +const PATH_SUMMARY_KINDS = new Set(["unchanged", "modified", "added", "deleted", "rejected"]); +const USAGE_KEYS = ["inputBytes", "outputBytes", "patchOperations", "filesTouched", "artifactBytes", "elapsedMs", "inactiveMs"] as const; +const LIMIT_KEYS = Object.keys(FABRIC_LIMITS) as Array; +const FAILURE_KEYS = new Set(["class", "code", "retryable", "attribution"]); +const FAILURE_ATTRIBUTIONS = new Set(["opencodex", "route", "environment", "harness"]); + +/** Require a non-null plain object or throw a harness-class FabricTaskError. */ +function assertPlainObject(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new FabricTaskError(`malformed producer outcome: ${label}`, "malformed_producer_outcome", "harness"); + } + return value as Record; +} + +/** Require a non-empty string field on a producer outcome object. */ +function assertStringField(obj: Record, key: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.length === 0) { + throw new FabricTaskError(`malformed producer outcome: ${key}`, "malformed_producer_outcome", "harness"); + } + return value; +} + +/** Require an integer field on a producer outcome object. */ +function assertIntegerField(obj: Record, key: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value)) { + throw new FabricTaskError(`malformed producer outcome: ${key}`, "malformed_producer_outcome", "harness"); + } + return value; +} + +/** Require a non-negative integer field on a producer outcome object. */ +function assertNonNegativeIntegerField(obj: Record, key: string): number { + const value = assertIntegerField(obj, key); + if (value < 0) { + throw new FabricTaskError(`malformed producer outcome: ${key}`, "malformed_producer_outcome", "harness"); + } + return value; +} + +/** Map LabValidationError into FabricTaskError for producer outcome parsing. */ +function wrapValidationError(error: unknown): never { + if (error instanceof LabValidationError) { + throw new FabricTaskError(error.message, "malformed_producer_outcome", "harness"); + } + throw error; +} + +/** Validate verifier payload shape and reject unknown nested verifier fields. */ +function validateFabricVerifier(raw: Record): void { + for (const key of Object.keys(raw)) { + if (!VERIFIER_KEYS.has(key)) { + throw new FabricTaskError(`unknown verifier field ${key}`, "malformed_producer_outcome", "harness"); + } + } + if (raw.verifierId !== FABRIC_VERIFIER_ID) { + throw new FabricTaskError("malformed producer outcome: verifierId", "malformed_producer_outcome", "harness"); + } + const manifestDigest = assertStringField(raw, "manifestDigest"); + if (!isSha256Hex(manifestDigest)) { + throw new FabricTaskError("malformed producer outcome: manifestDigest", "malformed_producer_outcome", "harness"); + } + if (typeof raw.passed !== "boolean") { + throw new FabricTaskError("malformed producer outcome: passed", "malformed_producer_outcome", "harness"); + } + if (!Array.isArray(raw.pathSummaries)) { + throw new FabricTaskError("malformed producer outcome: pathSummaries", "malformed_producer_outcome", "harness"); + } + for (const row of raw.pathSummaries) { + const summary = assertPlainObject(row, "pathSummaries[]"); + for (const key of Object.keys(summary)) { + if (!PATH_SUMMARY_KEYS.has(key)) { + throw new FabricTaskError(`unknown path summary field ${key}`, "malformed_producer_outcome", "harness"); + } + } + assertStringField(summary, "path"); + const kind = assertStringField(summary, "kind"); + if (!PATH_SUMMARY_KINDS.has(kind)) { + throw new FabricTaskError("malformed producer outcome: path summary kind", "malformed_producer_outcome", "harness"); + } + if (summary.beforeDigest !== undefined && !isSha256Hex(assertStringField(summary, "beforeDigest"))) { + throw new FabricTaskError("malformed producer outcome: beforeDigest", "malformed_producer_outcome", "harness"); + } + if (summary.afterDigest !== undefined && !isSha256Hex(assertStringField(summary, "afterDigest"))) { + throw new FabricTaskError("malformed producer outcome: afterDigest", "malformed_producer_outcome", "harness"); + } + if (summary.reason !== undefined && typeof summary.reason !== "string") { + throw new FabricTaskError("malformed producer outcome: path summary reason", "malformed_producer_outcome", "harness"); + } + } + if (raw.reason !== undefined && typeof raw.reason !== "string") { + throw new FabricTaskError("malformed producer outcome: reason", "malformed_producer_outcome", "harness"); + } +} + +/** Validate non-negative usage counters on a producer outcome. */ +function validateFabricUsage(raw: Record): void { + for (const key of USAGE_KEYS) { + assertNonNegativeIntegerField(raw, key); + } +} + +/** Validate non-negative limit fields on a producer outcome. */ +function validateFabricLimits(raw: Record): void { + for (const key of LIMIT_KEYS) { + assertNonNegativeIntegerField(raw, key); + } +} + +/** Validate failure record shape and known attribution values. */ +function validateFailureRecord(raw: Record): void { + for (const key of Object.keys(raw)) { + if (!FAILURE_KEYS.has(key)) { + throw new FabricTaskError(`unknown failure field ${key}`, "malformed_producer_outcome", "harness"); + } + } + const failureClass = assertStringField(raw, "class"); + if (!FAILURE_CLASSIFICATIONS.includes(failureClass as typeof FAILURE_CLASSIFICATIONS[number])) { + throw new FabricTaskError("malformed producer outcome: failure.class", "malformed_producer_outcome", "harness"); + } + assertStringField(raw, "code"); + if (typeof raw.retryable !== "boolean") { + throw new FabricTaskError("malformed producer outcome: failure.retryable", "malformed_producer_outcome", "harness"); + } + const attribution = assertStringField(raw, "attribution"); + if (!FAILURE_ATTRIBUTIONS.has(attribution)) { + throw new FabricTaskError("malformed producer outcome: failure.attribution", "malformed_producer_outcome", "harness"); + } +} + +/** Compare top-level and nested route subjects by canonical JCS representation. */ +function routeSubjectsMatch(top: RouteSubjectV1, nested: RouteSubjectV1): boolean { + return jcsStringify(top) === jcsStringify(nested); +} + +/** Build an allowlisted verifier_summary artifact with sanitized diagnostics. */ +function sanitizedVerifierSummary(outcome: FabricTaskOutcomeV1): Record { + const verifier = { + verifierId: outcome.verifier.verifierId, + manifestDigest: outcome.verifier.manifestDigest, + passed: outcome.verifier.passed, + pathSummaries: outcome.verifier.pathSummaries.map((row) => ({ + path: row.path, + kind: row.kind, + ...(row.beforeDigest ? { beforeDigest: row.beforeDigest } : {}), + ...(row.afterDigest ? { afterDigest: row.afterDigest } : {}), + ...(row.reason ? { reason: truncateUtf8(sanitizeDiagnostic(row.reason), 512) } : {}), + })), + ...(outcome.verifier.reason + ? { reason: truncateUtf8(sanitizeDiagnostic(outcome.verifier.reason), 512) } + : {}), + }; + return { + scenarioId: FABRIC_SCENARIO_ID, + passed: outcome.verifier.passed, + verifier, + usage: outcome.usage, + outcome: outcome.outcome, + }; +} + +/** Parse and deeply validate a fabric producer outcome before persistence. */ +export function assertFabricOutcomeV1(raw: unknown): FabricTaskOutcomeV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new FabricTaskError("malformed producer outcome", "malformed_producer_outcome", "harness"); + } + const obj = raw as Record; + for (const key of Object.keys(obj)) { + if (!OUTCOME_KEYS.has(key)) { + throw new FabricTaskError(`unknown outcome field ${key}`, "malformed_producer_outcome", "harness"); + } + } + if (obj.schemaVersion !== 1) { + throw new FabricTaskError("schemaVersion must be 1", "malformed_producer_outcome", "harness"); + } + + const taskSubjectObj = assertPlainObject(obj.taskSubject, "taskSubject"); + const routeSubjectObj = assertPlainObject(obj.routeSubject, "routeSubject"); + + let taskSubject!: TaskSubjectV1; + let routeSubject!: RouteSubjectV1; + try { + taskSubject = validateSubject(taskSubjectObj, "task_effectiveness") as TaskSubjectV1; + routeSubject = validateSubject(routeSubjectObj, "live_route_compatibility") as RouteSubjectV1; + } catch (error) { + wrapValidationError(error); + } + if (!routeSubjectsMatch(routeSubject, taskSubject.routeSubject)) { + throw new FabricTaskError("contradictory route subjects", "layer_subject_mismatch", "harness"); + } + + assertStringField(obj, "taskClassId"); + assertStringField(obj, "taskClassVersion"); + const subjectId = assertStringField(obj, "subjectId"); + if (!isSha256Hex(subjectId)) { + throw new FabricTaskError("malformed producer outcome: subjectId", "malformed_producer_outcome", "harness"); + } + const taskFixtureDigest = assertStringField(obj, "taskFixtureDigest"); + const verifierManifestDigest = assertStringField(obj, "verifierManifestDigest"); + if (!isSha256Hex(taskFixtureDigest) || !isSha256Hex(verifierManifestDigest)) { + throw new FabricTaskError("malformed producer outcome: digest field", "malformed_producer_outcome", "harness"); + } + assertStringField(obj, "fabricCompatibilityVersion"); + const sandboxProfileDigest = assertStringField(obj, "sandboxProfileDigest"); + if (!isSha256Hex(sandboxProfileDigest)) { + throw new FabricTaskError("malformed producer outcome: sandboxProfileDigest", "malformed_producer_outcome", "harness"); + } + assertIntegerField(obj, "startedAt"); + assertIntegerField(obj, "completedAt"); + validateFabricLimits(assertPlainObject(obj.limits, "limits")); + validateFabricUsage(assertPlainObject(obj.usage, "usage")); + validateFabricVerifier(assertPlainObject(obj.verifier, "verifier")); + if (!OUTCOMES.includes(obj.outcome as typeof OUTCOMES[number])) { + throw new FabricTaskError("malformed producer outcome: outcome", "malformed_producer_outcome", "harness"); + } + if (!Array.isArray(obj.artifactDigests)) { + throw new FabricTaskError("malformed producer outcome: artifactDigests", "malformed_producer_outcome", "harness"); + } + for (const digest of obj.artifactDigests) { + if (typeof digest !== "string" || !isSha256Hex(digest)) { + throw new FabricTaskError("malformed producer outcome: artifactDigests", "malformed_producer_outcome", "harness"); + } + } + if (obj.failure !== undefined) { + validateFailureRecord(assertPlainObject(obj.failure, "failure")); + } + if (obj.sourceRefs !== undefined) { + if (!Array.isArray(obj.sourceRefs) || !obj.sourceRefs.every((row) => typeof row === "string" && row.length > 0)) { + throw new FabricTaskError("malformed producer outcome: sourceRefs", "malformed_producer_outcome", "harness"); + } + } + + return raw as FabricTaskOutcomeV1; +} + +/** Convert a validated fabric outcome into a ledger-ready observation and artifacts. */ +export function observationFromFabricOutcome( + outcomeRaw: unknown, + opts: PersistFabricOptions = {}, +): { event: ObservationEvent; artifacts: ReturnType[] } { + const outcome = assertFabricOutcomeV1(outcomeRaw); + const taskSubject = outcome.taskSubject as TaskSubjectV1; + const routeSubject = outcome.routeSubject as RouteSubjectV1; + if (taskSubject.subjectKind !== "task") { + throw new FabricTaskError("task layer requires TaskSubjectV1", "layer_subject_mismatch", "harness"); + } + if (routeSubject.subjectKind !== "route") { + throw new FabricTaskError("nested route subject required", "layer_subject_mismatch", "harness"); + } + if (!Number.isInteger(outcome.startedAt) || !Number.isInteger(outcome.completedAt) || outcome.completedAt < outcome.startedAt) { + throw new FabricTaskError("invalid execution timestamps", "malformed_producer_outcome", "harness"); + } + + const paths = ensureLabDirs(opts.configDir); + const ownsStore = !opts.artifactStore; + const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { + const authority = loadFabricCaseAuthority(); + const caseRecord = authority.cases.find((row) => row.id === FABRIC_SCENARIO_ID); + if (!caseRecord) { + throw new FabricTaskError(`missing fabric scenario ${FABRIC_SCENARIO_ID}`, "harness_failure", "harness"); + } + const scenarioDigest = fabricScenarioManifestDigest(caseRecord, authority); + const suiteDigest = fabricSuiteManifestDigest(FABRIC_SUITE_ID, authority); + const expandedScenario = expandFabricScenario(caseRecord, authority); + const expandedSuite = expandFabricSuiteManifest(FABRIC_SUITE_ID, authority); + const fixtureBytes = new TextEncoder().encode(caseRecord.fixture.bytesUtf8); + const fixtureDigests = [fixtureDigest(fixtureBytes)]; + + const artifacts: ReturnType[] = []; + artifacts.push(store.put({ + artifactClass: "fixture", + payload: fixtureBytes, + expectedDigest: fixtureDigests[0], + mediaType: caseRecord.fixture.mediaType, + })); + artifacts.push(store.put({ + artifactClass: "scenario_manifest", + payload: expandedScenario, + expectedDigest: scenarioDigest, + })); + artifacts.push(store.put({ + artifactClass: "suite_manifest", + payload: expandedSuite, + expectedDigest: suiteDigest, + })); + artifacts.push(store.put({ + artifactClass: "verifier_summary", + payload: sanitizedVerifierSummary(outcome), + })); + + const limits: Record = {}; + for (const key of OBSERVATION_LIMIT_NAMES) { + if (key in authority.manifestDefaults.executionLimits) { + limits[key] = authority.manifestDefaults.executionLimits[key] ?? null; + } + } + + const recordedAt = opts.recordedAt ?? outcome.completedAt; + const eventWithoutId = { + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "observation" as const, + recordedAt, + producer: LAB_PRODUCER, + producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, + evidenceLayer: FABRIC_EVIDENCE_LAYER, + scenarioId: FABRIC_SCENARIO_ID, + scenarioVersion: FABRIC_SCENARIO_VERSION, + scenarioManifestDigest: scenarioDigest, + suiteId: FABRIC_SUITE_ID, + suiteVersion: FABRIC_SUITE_VERSION, + suiteManifestDigest: suiteDigest, + fixtureDigests, + subject: outcome.taskSubject, + subjectId: outcome.subjectId, + startedAt: outcome.startedAt, + completedAt: outcome.completedAt, + executionMode: "fabric" as const, + attempt: opts.attempt ?? 1, + limits, + outcome: outcome.outcome, + assertions: [{ + id: "exact-tree-diff-pass", + operator: "equals", + required: true, + passed: outcome.verifier.passed, + expectedSummary: "pass", + observedSummary: truncateUtf8(sanitizeDiagnostic(outcome.verifier.reason ?? (outcome.verifier.passed ? "pass" : "fail")), 512), + }], + environment: { + runtime: { + platform: process.platform, + arch: process.arch, + bunVersion: process.versions.bun ?? Bun.version, + }, + }, + artifactRefs: artifacts, + ...(outcome.failure ? { failure: outcome.failure } : {}), + ...(outcome.sourceRefs ? { sourceRefs: [...outcome.sourceRefs] } : {}), + }; + return { event: assignEventId(eventWithoutId) as ObservationEvent, artifacts }; + } finally { + if (ownsStore) store.close(); + } +} + +/** Module-private ledger append helper; production evidence must use {@link persistFabricRunResult}. */ +function persistFabricOutcome( + outcome: FabricTaskOutcomeV1, + opts: PersistFabricOptions = {}, +): PersistedFabricObservation { + const paths = ensureLabDirs(opts.configDir); + const ownsStore = !opts.artifactStore; + const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); + try { + const { event } = observationFromFabricOutcome(outcome, { ...opts, artifactStore: store }); + appendLabEventIfAbsent(paths.ledgerPath, event); + return { event, ledgerPath: paths.ledgerPath }; + } finally { + if (ownsStore) store.close(); + } +} + +/** Persist production fabric evidence from a trusted-route run result. */ +export function persistFabricRunResult( + result: FabricTaskRunResult, + opts: PersistFabricOptions = {}, +): PersistedFabricObservation { + if (result.executionAuthority !== "trusted_route") { + throw new FabricTaskError( + "fabric evidence requires trusted route execution authority", + "malformed_producer_outcome", + "harness", + ); + } + return persistFabricOutcome(result.outcome, opts); +} diff --git a/src/lab/fabric/patch.ts b/src/lab/fabric/patch.ts new file mode 100644 index 0000000000..9ac196f5b8 --- /dev/null +++ b/src/lab/fabric/patch.ts @@ -0,0 +1,79 @@ +import { FABRIC_LIMITS, SYNTHETIC_VALUE_PATH } from "./constants"; +import { assertSafeRelativePosixPath, writeScratchFileUtf8 } from "./scratch"; +import { FabricTaskError, type SyntheticPatchV1 } from "./types"; + +const ALLOWED_KEYS = new Set(["schemaVersion", "operations"]); +const OP_KEYS = new Set(["op", "path", "contentUtf8"]); + +/** Parse and validate a synthetic-patch producer payload. */ +export function parseSyntheticPatchV1(raw: unknown): SyntheticPatchV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new FabricTaskError("malformed producer outcome: patch must be object", "malformed_producer_outcome", "harness"); + } + const obj = raw as Record; + for (const key of Object.keys(obj)) { + if (!ALLOWED_KEYS.has(key)) { + throw new FabricTaskError(`malformed producer outcome: unknown field ${key}`, "malformed_producer_outcome", "harness"); + } + } + if (obj.schemaVersion !== 1) { + throw new FabricTaskError("malformed producer outcome: schemaVersion", "malformed_producer_outcome", "harness"); + } + if (!Array.isArray(obj.operations)) { + throw new FabricTaskError("malformed producer outcome: operations", "malformed_producer_outcome", "harness"); + } + if (obj.operations.length !== 1) { + throw new FabricTaskError("exactly one patch operation is required", "malformed_producer_outcome", "harness"); + } + const opRaw = obj.operations[0]; + if (!opRaw || typeof opRaw !== "object" || Array.isArray(opRaw)) { + throw new FabricTaskError("malformed producer outcome: operation", "malformed_producer_outcome", "harness"); + } + const op = opRaw as Record; + for (const key of Object.keys(op)) { + if (!OP_KEYS.has(key)) { + throw new FabricTaskError(`malformed producer outcome: unknown op field ${key}`, "malformed_producer_outcome", "harness"); + } + } + if (op.op !== "replace" || typeof op.path !== "string" || typeof op.contentUtf8 !== "string") { + throw new FabricTaskError("malformed producer outcome: replace shape", "malformed_producer_outcome", "harness"); + } + try { + assertSafeRelativePosixPath(op.path); + } catch (error) { + if (error instanceof FabricTaskError) throw error; + throw new FabricTaskError("patch path escapes scratch", "sandbox_violation", "harness"); + } + if (op.path !== SYNTHETIC_VALUE_PATH) { + throw new FabricTaskError("patch path must be src/value.txt", "behavioral_failure", "route"); + } + const contentBytes = Buffer.byteLength(op.contentUtf8, "utf8"); + if (contentBytes > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("patch content exceeds io budget", "budget_exhausted", "environment"); + } + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: op.contentUtf8 }], + }; +} + +/** Apply a validated synthetic patch inside the scratch tree under budget limits. */ +export function applySyntheticPatch(scratchRoot: string, patch: SyntheticPatchV1): { bytesWritten: number; filesTouched: number; patchOperations: number } { + if (patch.operations.length > FABRIC_LIMITS.maxPatchOperations) { + throw new FabricTaskError("too many patch operations", "budget_exhausted", "environment"); + } + let bytesWritten = 0; + for (const operation of patch.operations) { + bytesWritten += writeScratchFileUtf8( + scratchRoot, + operation.path, + operation.contentUtf8, + FABRIC_LIMITS.maxAggregateIoBytes, + ); + } + return { + bytesWritten, + filesTouched: patch.operations.length, + patchOperations: patch.operations.length, + }; +} diff --git a/src/lab/fabric/producer-child.ts b/src/lab/fabric/producer-child.ts new file mode 100644 index 0000000000..6a2f23349f --- /dev/null +++ b/src/lab/fabric/producer-child.ts @@ -0,0 +1,122 @@ +import { + FABRIC_LIMITS, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, +} from "./constants"; +import { + encodeProducerProtocolLine, + FABRIC_PRODUCER_PROTOCOL_MAX_BYTES, + FABRIC_PRODUCER_REQUEST_MAX_BYTES, + type ProducerParentRequest, +} from "./producer-protocol"; +import type { FabricHarnessProducerKind, SyntheticPatchV1 } from "./types"; + +function writeLine(message: Parameters[0]): void { + process.stdout.write(encodeProducerProtocolLine(message)); +} + +function correctPatch(): SyntheticPatchV1 { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} + +function wrongPatch(): SyntheticPatchV1 { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: "wrong\n" }], + }; +} + +async function runHarness(kind: FabricHarnessProducerKind, scratchRoot: string): Promise { + switch (kind) { + case "deterministic_correct": + return correctPatch(); + case "deterministic_wrong": + return wrongPatch(); + case "infinite_sync": + while (true) { + /* terminated by parent */ + } + case "never_resolve": + while (true) { + await Bun.sleep(1_000); + } + case "mutate_after_delay": { + await Bun.sleep(FABRIC_LIMITS.totalTimeoutMs + 5_000); + const { writeFileSync, mkdirSync } = await import("node:fs"); + const { join, dirname } = await import("node:path"); + mkdirSync(join(scratchRoot, dirname(SYNTHETIC_VALUE_PATH)), { recursive: true }); + writeFileSync(join(scratchRoot, SYNTHETIC_VALUE_PATH), "late\n"); + return correctPatch(); + } + case "periodic_activity": + for (let i = 0; i < 20; i++) { + writeLine({ type: "activity" }); + await Bun.sleep(200); + } + return correctPatch(); + case "activity_until_total": + while (true) { + writeLine({ type: "activity" }); + await Bun.sleep(50); + } + case "flood_stdout": + while (true) { + process.stdout.write("x".repeat(4096)); + } + default: + throw new Error(`unsupported harness kind: ${kind as string}`); + } +} + +async function runExecutorModule( + modulePath: string, + executorInput: unknown, + reportActivity: () => void, +): Promise { + const mod = await import(modulePath) as { execute: (input: unknown) => Promise | SyntheticPatchV1 }; + if (typeof mod.execute !== "function") throw new Error("executor module missing execute export"); + const input = { + ...(executorInput as Record), + reportActivity, + }; + return await Promise.resolve(mod.execute(input)); +} + +async function main(): Promise { + const raw = await Bun.stdin.text(); + if (Buffer.byteLength(raw, "utf8") > FABRIC_PRODUCER_REQUEST_MAX_BYTES) { + writeLine({ + type: "error", + code: "budget_exhausted", + message: "request payload exceeds protocol limit", + attribution: "environment", + }); + process.exit(1); + return; + } + const req = JSON.parse(raw) as ProducerParentRequest; + let patch: SyntheticPatchV1; + const reportActivity = () => writeLine({ type: "activity" }); + if (req.harnessKind) { + patch = await runHarness(req.harnessKind as FabricHarnessProducerKind, req.scratchRoot); + } else if (req.executorModulePath) { + patch = await runExecutorModule(req.executorModulePath, req.executorInput, reportActivity); + } else { + throw new Error("child request missing harnessKind or executorModulePath"); + } + writeLine({ type: "result", patch }); +} + +main().catch((error: unknown) => { + writeLine({ + type: "error", + code: "harness_failure", + message: error instanceof Error ? error.message : String(error), + attribution: "harness", + }); + process.exit(1); +}); diff --git a/src/lab/fabric/producer-isolate.ts b/src/lab/fabric/producer-isolate.ts new file mode 100644 index 0000000000..c808de0e12 --- /dev/null +++ b/src/lab/fabric/producer-isolate.ts @@ -0,0 +1,249 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { FABRIC_LIMITS } from "./constants"; +import { + FABRIC_PRODUCER_PROTOCOL_MAX_BYTES, + FABRIC_PRODUCER_REQUEST_MAX_BYTES, + FABRIC_PRODUCER_STDERR_MAX_BYTES, + parseProducerProtocolLine, + type IsolatedProducerResult, +} from "./producer-protocol"; +import type { FabricHarnessProducerKind, FabricPatchExecutorInput, SyntheticPatchV1 } from "./types"; +import { FabricTaskError } from "./types"; + +const CHILD_ENTRY = join(dirname(fileURLToPath(import.meta.url)), "producer-child.ts"); + +interface IsolateRequest { + harnessKind?: FabricHarnessProducerKind; + executorModulePath?: string; + scratchRoot: string; + totalTimeoutMs: number; + inactivityTimeoutMs: number; + executorInput?: FabricPatchExecutorInput; + now?: () => number; +} + +function minimalChildEnv(scratchRoot: string): Record { + return { + TZ: "UTC", + NO_COLOR: "1", + OCX_FABRIC_SCRATCH_ROOT: scratchRoot, + }; +} + +function killChild(child: ChildProcess): void { + try { + child.kill("SIGKILL"); + } catch { + /* ignore */ + } +} + +/** Run a fabric patch producer in an isolated child process with parent-owned timeouts. */ +export async function runIsolatedFabricProducer(request: IsolateRequest): Promise { + const now = request.now ?? (() => Date.now()); + let lastActivityAt = now(); + + return await new Promise((resolve, reject) => { + let child: ChildProcess; + try { + child = spawn(process.execPath, ["run", CHILD_ENTRY], { + env: minimalChildEnv(request.scratchRoot), + stdio: ["pipe", "pipe", "pipe"], + }); + } catch (error) { + reject(new FabricTaskError( + error instanceof Error ? error.message : String(error), + "harness_failure", + "harness", + )); + return; + } + + let stdoutBuffer = ""; + let stderrBytes = 0; + let settled = false; + let receivedResult: SyntheticPatchV1 | undefined; + let killReason: FabricTaskError | undefined; + + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(totalTimer); + clearTimeout(inactivityTimer); + fn(); + }; + + const settleTimeout = (error: FabricTaskError) => { + if (settled) return; + killReason = error; + killChild(child); + }; + + const armInactivity = () => { + clearTimeout(inactivityTimer); + inactivityTimer = setTimeout(() => { + settleTimeout(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); + }, request.inactivityTimeoutMs); + }; + + let inactivityTimer: ReturnType = setTimeout(() => { + settleTimeout(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); + }, request.inactivityTimeoutMs); + + const totalTimer = setTimeout(() => { + settleTimeout(new FabricTaskError("total timeout exceeded", "timeout", "environment")); + }, request.totalTimeoutMs); + + const handleProtocolLine = (line: string) => { + try { + const message = parseProducerProtocolLine(line); + if (message.type === "activity") { + lastActivityAt = now(); + armInactivity(); + return; + } + if (message.type === "result") { + if (settled) return; + receivedResult = message.patch; + finish(() => resolve({ patch: message.patch, lastActivityAt })); + return; + } + if (message.type === "error") { + const fabricCode = message.code === "inactivity_timeout" + ? "inactivity_timeout" + : message.code === "timeout" + ? "timeout" + : message.code === "sandbox_violation" + ? "sandbox_violation" + : message.code === "budget_exhausted" + ? "budget_exhausted" + : "harness_failure"; + const attribution = message.attribution === "environment" ? "environment" : "harness"; + const fabricError = new FabricTaskError(message.message, fabricCode, attribution); + finish(() => reject(fabricError)); + killChild(child); + return; + } + } catch (error) { + settleTimeout(new FabricTaskError( + error instanceof Error ? error.message : String(error), + "harness_failure", + "harness", + )); + } + }; + + const consumeStdout = (chunk: string) => { + stdoutBuffer += chunk; + if (Buffer.byteLength(stdoutBuffer, "utf8") > FABRIC_PRODUCER_PROTOCOL_MAX_BYTES) { + settleTimeout(new FabricTaskError("producer protocol output exceeded limit", "budget_exhausted", "environment")); + stdoutBuffer = ""; + return; + } + let newlineIdx = stdoutBuffer.indexOf("\n"); + while (newlineIdx >= 0) { + const line = stdoutBuffer.slice(0, newlineIdx); + stdoutBuffer = stdoutBuffer.slice(newlineIdx + 1); + handleProtocolLine(line); + newlineIdx = stdoutBuffer.indexOf("\n"); + } + }; + + child.stdout?.on("data", (chunk: Buffer | string) => { + consumeStdout(chunk.toString("utf8")); + }); + + child.stdout?.on("error", (error) => { + if (settled) return; + finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + }); + + child.stderr?.on("data", (chunk: Buffer | string) => { + stderrBytes += Buffer.byteLength(chunk.toString("utf8"), "utf8"); + if (stderrBytes > FABRIC_PRODUCER_STDERR_MAX_BYTES) { + stderrBytes = FABRIC_PRODUCER_STDERR_MAX_BYTES; + } + }); + + child.on("error", (error) => { + finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + }); + + child.stdin?.on("error", (error: NodeJS.ErrnoException) => { + if (settled || error.code === "EPIPE") return; + killChild(child); + finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); + }); + + const payload = JSON.stringify({ + harnessKind: request.harnessKind, + executorModulePath: request.executorModulePath, + scratchRoot: request.scratchRoot, + executorInput: request.executorInput + ? { + routeContext: request.executorInput.routeContext, + destination: request.executorInput.destination, + routeSubject: request.executorInput.routeSubject, + scratchRoot: request.executorInput.scratchRoot, + } + : undefined, + }); + if (Buffer.byteLength(payload, "utf8") > FABRIC_PRODUCER_REQUEST_MAX_BYTES) { + killChild(child); + finish(() => reject(new FabricTaskError("producer request exceeds protocol limit", "budget_exhausted", "environment"))); + return; + } + + try { + child.stdin?.write(payload); + child.stdin?.end(); + } catch (error) { + killChild(child); + finish(() => reject(new FabricTaskError( + error instanceof Error ? error.message : String(error), + "harness_failure", + "harness", + ))); + return; + } + + child.on("close", (code, signal) => { + if (settled) return; + if (killReason) { + finish(() => reject(killReason!)); + return; + } + if (receivedResult) { + finish(() => resolve({ patch: receivedResult!, lastActivityAt })); + return; + } + if (stdoutBuffer.trim()) { + try { + handleProtocolLine(stdoutBuffer.trim()); + if (receivedResult) return; + } catch { + /* fall through */ + } + } + if (signal === "SIGKILL") { + finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); + return; + } + finish(() => reject(new FabricTaskError( + code === 0 ? "isolated producer returned no result" : `isolated producer exited (${code ?? signal ?? "unknown"})`, + "harness_failure", + "harness", + ))); + }); + }); +} + +/** Default isolation limits for fabric producer child processes. */ +export function fabricProducerIsolationLimits(): { totalTimeoutMs: number; inactivityTimeoutMs: number } { + return { + totalTimeoutMs: FABRIC_LIMITS.totalTimeoutMs, + inactivityTimeoutMs: FABRIC_LIMITS.inactivityTimeoutMs, + }; +} \ No newline at end of file diff --git a/src/lab/fabric/producer-protocol.ts b/src/lab/fabric/producer-protocol.ts new file mode 100644 index 0000000000..ab20548467 --- /dev/null +++ b/src/lab/fabric/producer-protocol.ts @@ -0,0 +1,58 @@ +import type { SyntheticPatchV1 } from "./types"; + +/** Maximum bytes accepted from child stdout for the fabric producer protocol. */ +export const FABRIC_PRODUCER_PROTOCOL_MAX_BYTES = 64 * 1024; + +/** Maximum stderr bytes retained for harness diagnostics. */ +export const FABRIC_PRODUCER_STDERR_MAX_BYTES = 4 * 1024; + +/** Maximum request payload size on child stdin. */ +export const FABRIC_PRODUCER_REQUEST_MAX_BYTES = 64 * 1024; + +export type ProducerChildMessage = + | { type: "activity" } + | { type: "result"; patch: SyntheticPatchV1 } + | { type: "error"; code: string; message: string; attribution: "harness" | "environment" }; + +export interface ProducerParentRequest { + harnessKind?: string; + executorModulePath?: string; + scratchRoot: string; + executorInput?: unknown; +} + +export interface IsolatedProducerResult { + patch: SyntheticPatchV1; + lastActivityAt: number; +} + +/** Serialize one protocol line to stdout (newline-delimited JSON). */ +export function encodeProducerProtocolLine(message: ProducerChildMessage): string { + return `${JSON.stringify(message)}\n`; +} + +/** Parse one protocol line from the child. Malformed lines throw. */ +export function parseProducerProtocolLine(line: string): ProducerChildMessage { + const trimmed = line.trim(); + if (!trimmed) { + throw new Error("empty protocol line"); + } + const parsed = JSON.parse(trimmed) as ProducerChildMessage; + if (!parsed || typeof parsed !== "object" || typeof parsed.type !== "string") { + throw new Error("malformed protocol message"); + } + switch (parsed.type) { + case "activity": + return { type: "activity" }; + case "result": + if (!parsed.patch || typeof parsed.patch !== "object") throw new Error("malformed result patch"); + return parsed as ProducerChildMessage; + case "error": + if (typeof parsed.code !== "string" || typeof parsed.message !== "string") { + throw new Error("malformed error message"); + } + return parsed as ProducerChildMessage; + default: + throw new Error(`unknown protocol type: ${(parsed as { type: string }).type}`); + } +} diff --git a/src/lab/fabric/scratch.ts b/src/lab/fabric/scratch.ts new file mode 100644 index 0000000000..6f0c9e8a60 --- /dev/null +++ b/src/lab/fabric/scratch.ts @@ -0,0 +1,439 @@ +import { + closeSync, + constants as fsConstants, + existsSync, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readdirSync, + readSync, + rmSync, + writeSync, + type Stats, +} from "node:fs"; +import { dirname, join, posix, relative, resolve, sep } from "node:path"; +import { randomBytes } from "node:crypto"; +import { ensureLabDirs, ensureRestrictedDir, labRoot, labScratchDir } from "../paths"; +import { FABRIC_LIMITS, SYNTHETIC_BEFORE_UTF8, SYNTHETIC_VALUE_PATH } from "./constants"; +import { FabricTaskError } from "./types"; + +/** CL-07 restricted scratch tree IO for the synthetic-patch fabric task. */ + +const O_DIRECTORY = (fsConstants as { O_DIRECTORY?: number }).O_DIRECTORY; +const O_NOFOLLOW = (fsConstants as { O_NOFOLLOW?: number }).O_NOFOLLOW; +const FILE_MODE = 0o600; + +interface TrustedScratchDir { + path: string; + fd: number; + identity: string; +} + +/** Require stats to describe a regular file, not a symlink or special node. */ +function assertRegularFile(stats: Stats, label: string): void { + if (!stats.isFile() || stats.isSymbolicLink() || stats.isDirectory() || stats.isFIFO() || stats.isSocket() || stats.isCharacterDevice() || stats.isBlockDevice()) { + throw new FabricTaskError(`${label} must be a regular file`, "sandbox_violation", "harness"); + } +} + +/** Require stats to describe a real directory, not a symlink. */ +function assertRealDirectory(stats: Stats, label: string): void { + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new FabricTaskError(`${label} must be a real directory`, "sandbox_violation", "harness"); + } +} + +/** Stable device/inode identity string for a filesystem node. */ +function identityOf(stats: Stats): string { + return `${stats.dev}:${stats.ino}`; +} + +/** Whether the host supports O_NOFOLLOW for scratch open operations. */ +function platformSupportsNoFollow(): boolean { + return typeof O_NOFOLLOW === "number" && O_NOFOLLOW !== 0; +} + +/** Compose open flags, optionally adding O_NOFOLLOW when supported. */ +function openFlags(base: number, noFollow = true): number { + if (noFollow && platformSupportsNoFollow()) return base | O_NOFOLLOW!; + return base; +} + +/** Reject scratch relative names that traverse or embed forbidden characters. */ +function assertScratchName(name: string): void { + if (name.includes("/") || name.includes("\\") || name.includes("..") || name.includes("\0")) { + throw new FabricTaskError("invalid scratch relative name", "sandbox_violation", "harness"); + } +} + +/** Confirm the scratch root path still refers to the same directory inode. */ +function revalidateScratchDir(dir: TrustedScratchDir): void { + const stats = lstatSync(dir.path); + assertRealDirectory(stats, "scratch dir"); + if (identityOf(stats) !== dir.identity) { + throw new FabricTaskError("scratch directory identity changed", "sandbox_violation", "harness"); + } +} + +/** Resolve a single path segment under a trusted scratch directory. */ +function childScratchPath(dir: TrustedScratchDir, name: string): string { + revalidateScratchDir(dir); + assertScratchName(name); + return join(dir.path, name); +} + +/** Open a single name relative to a trusted scratch directory using pinned paths. */ +function openAtScratch(dir: TrustedScratchDir, name: string, flags: number, mode = 0, expectDirectory = false): number { + revalidateScratchDir(dir); + assertScratchName(name); + const path = childScratchPath(dir, name); + const fd = openSync(path, openFlags(flags), mode); + const opened = fstatSync(fd); + if (expectDirectory) { + assertRealDirectory(opened, name); + } else { + assertRegularFile(opened, name); + const pathEntry = lstatSync(path); + if (identityOf(opened) !== identityOf(pathEntry)) { + throw new FabricTaskError("scratch path redirection detected", "sandbox_violation", "harness"); + } + } + return fd; +} + +/** Pin an absolute scratch root path into a trusted directory handle. */ +function openTrustedScratchRoot(scratchRoot: string): TrustedScratchDir { + const abs = resolve(scratchRoot); + let fd: number; + if (typeof O_DIRECTORY === "number") { + fd = openSync(abs, openFlags(fsConstants.O_RDONLY | O_DIRECTORY, true)); + } else { + fd = openSync(abs, fsConstants.O_RDONLY); + } + const stats = fstatSync(fd); + assertRealDirectory(stats, "scratch root"); + return { path: abs, fd, identity: identityOf(stats) }; +} + +/** Close a trusted scratch directory handle. */ +function closeTrustedScratchRoot(dir: TrustedScratchDir): void { + try { + closeSync(dir.fd); + } catch { + /* ignore */ + } +} + +/** Open a relative scratch path by walking trusted directory handles. */ +function openScratchRelativePath( + root: TrustedScratchDir, + relativePath: string, + flags: number, + mode: number, +): number { + const parts = assertSafeRelativePosixPath(relativePath).split("/"); + let current = root; + const intermediateFds: number[] = []; + try { + for (let i = 0; i < parts.length; i++) { + const isFinal = i === parts.length - 1; + const part = parts[i]!; + if (!isFinal) { + const dirFlags = typeof O_DIRECTORY === "number" + ? openFlags(fsConstants.O_RDONLY | O_DIRECTORY, true) + : openFlags(fsConstants.O_RDONLY, true); + const subFd = openAtScratch(current, part, dirFlags, 0, true); + intermediateFds.push(subFd); + const subStats = fstatSync(subFd); + assertRealDirectory(subStats, part); + current = { path: join(current.path, part), fd: subFd, identity: identityOf(subStats) }; + continue; + } + return openAtScratch(current, part, flags, mode); + } + throw new FabricTaskError("empty scratch path", "sandbox_violation", "harness"); + } finally { + for (const fd of intermediateFds) { + try { + closeSync(fd); + } catch { + /* ignore */ + } + } + } +} + +/** Read exactly size bytes from an open scratch file descriptor. */ +function readAllFromFd(fd: number, size: number): string { + const buf = Buffer.alloc(size); + let offset = 0; + while (offset < buf.length) { + const n = readSync(fd, buf, offset, buf.length - offset, offset); + if (n <= 0) break; + offset += n; + } + if (offset !== size) { + throw new FabricTaskError("short read from scratch file", "sandbox_violation", "harness"); + } + return buf.toString("utf8"); +} + +/** Normalize and reject traversal / absolute / Windows drive / NUL paths. */ +export function assertSafeRelativePosixPath(raw: string): string { + if (typeof raw !== "string" || raw.length === 0) { + throw new FabricTaskError("path required", "sandbox_violation", "harness"); + } + if (raw.includes("\0") || raw.includes("\\")) { + throw new FabricTaskError("path contains forbidden characters", "sandbox_violation", "harness"); + } + if (raw.startsWith("/") || /^[A-Za-z]:/.test(raw)) { + throw new FabricTaskError("absolute paths are forbidden", "sandbox_violation", "harness"); + } + const normalized = posix.normalize(raw); + if (normalized === "." || normalized.startsWith("../") || normalized === ".." || normalized.includes("/../")) { + throw new FabricTaskError("path traversal is forbidden", "sandbox_violation", "harness"); + } + if (normalized.split("/").some((part) => part === ".." || part === "")) { + throw new FabricTaskError("invalid path segments", "sandbox_violation", "harness"); + } + return normalized; +} + +/** Prove a resolved path remains under the scratch root prefix. */ +function assertUnderScratchRoot(root: string, target: string): void { + if (target !== root && !target.startsWith(root + sep)) { + throw new FabricTaskError("path escapes scratch root", "sandbox_violation", "harness"); + } +} + +/** + * Resolve a relative path under scratch without following intermediate symlinks. + * Missing final component is allowed (for create); intermediate components must exist. + */ +export function resolveInsideScratch(scratchRoot: string, relativePath: string, opts: { allowMissingFinal?: boolean } = {}): string { + const safe = assertSafeRelativePosixPath(relativePath); + const root = resolve(scratchRoot); + const rootStats = lstatSync(root); + assertRealDirectory(rootStats, "scratch root"); + + let current = root; + const parts = safe.split("/"); + for (let i = 0; i < parts.length; i++) { + const next = join(current, parts[i]!); + assertUnderScratchRoot(root, next); + const isFinal = i === parts.length - 1; + if (!existsSync(next)) { + if (isFinal && opts.allowMissingFinal) return next; + throw new FabricTaskError(isFinal ? `missing file ${relativePath}` : "missing intermediate directory", "sandbox_violation", "harness"); + } + const stats = lstatSync(next); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + if (!isFinal) { + assertRealDirectory(stats, next); + } + current = next; + } + return current; +} + +/** Create a relative directory tree under scratch without following symlink components. */ +function ensureScratchRelativeDir(scratchRoot: string, relativeDir: string): void { + const safe = assertSafeRelativePosixPath(relativeDir); + const root = resolve(scratchRoot); + const rootStats = lstatSync(root); + assertRealDirectory(rootStats, "scratch root"); + let current = root; + for (const part of safe.split("/")) { + const next = join(current, part); + assertUnderScratchRoot(root, next); + if (!existsSync(next)) { + try { + mkdirSync(next, { mode: 0o700 }); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "EEXIST") throw error; + } + } + const stats = lstatSync(next); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + assertRealDirectory(stats, next); + current = next; + } +} + +/** Handle for a fabric scratch tree and its best-effort cleanup callback. */ +export interface ScratchTree { + root: string; + cleanup: () => void; +} + +/** Create an isolated scratch tree with the frozen synthetic-patch fixture file. */ +export function createSyntheticScratch(configDir?: string): ScratchTree { + ensureLabDirs(configDir); + const labBoundary = labRoot(configDir); + const base = labScratchDir(configDir); + ensureRestrictedDir(base, labBoundary); + const root = join(base, `fabric-${Date.now().toString(36)}-${randomBytes(6).toString("hex")}`); + let trusted: TrustedScratchDir | undefined; + try { + ensureRestrictedDir(root, labBoundary); + ensureRestrictedDir(join(root, dirname(SYNTHETIC_VALUE_PATH)), labBoundary); + trusted = openTrustedScratchRoot(root); + const bytes = Buffer.from(SYNTHETIC_BEFORE_UTF8, "utf8"); + if (bytes.byteLength > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("fixture exceeds io budget", "budget_exhausted", "harness"); + } + const fd = openScratchRelativePath( + trusted, + SYNTHETIC_VALUE_PATH, + openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL, true), + FILE_MODE, + ); + try { + writeSync(fd, bytes); + } finally { + closeSync(fd); + } + const trustedForCleanup = trusted; + return { + root, + cleanup: () => { + closeTrustedScratchRoot(trustedForCleanup); + try { + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + } catch { + // best-effort cleanup + } + }, + }; + } catch (error) { + if (trusted) closeTrustedScratchRoot(trusted); + try { + rmSync(root, { recursive: true, force: true, maxRetries: 3 }); + } catch { + /* best-effort */ + } + throw error; + } +} + +/** One regular file discovered during a bounded scratch tree walk. */ +export interface WalkedFile { + relativePosix: string; + absolute: string; + byteLength: number; +} + +/** No-follow walk; rejects symlinks and special files; enforces file/byte budgets. */ +export function walkScratchFiles(scratchRoot: string): WalkedFile[] { + const root = resolve(scratchRoot); + const rootStats = lstatSync(root); + assertRealDirectory(rootStats, "scratch root"); + const out: WalkedFile[] = []; + const stack: string[] = [root]; + let aggregateBytes = 0; + while (stack.length > 0) { + const current = stack.pop()!; + const stats = lstatSync(current); + if (stats.isSymbolicLink()) { + throw new FabricTaskError("symlink rejected", "sandbox_violation", "harness"); + } + if (stats.isDirectory()) { + const entries = readdirSync(current, { withFileTypes: true }); + for (const entry of entries) { + stack.push(join(current, entry.name)); + } + continue; + } + assertRegularFile(stats, current); + if (out.length >= FABRIC_LIMITS.maxScratchFiles) { + throw new FabricTaskError("scratch file count exceeds budget", "budget_exhausted", "harness"); + } + aggregateBytes += stats.size; + if (aggregateBytes > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("scratch aggregate bytes exceed budget", "budget_exhausted", "harness"); + } + const rel = relative(root, current).split(sep).join("/"); + const safe = assertSafeRelativePosixPath(rel); + out.push({ relativePosix: safe, absolute: current, byteLength: stats.size }); + } + out.sort((a, b) => { + const left = Buffer.from(a.relativePosix, "utf8"); + const right = Buffer.from(b.relativePosix, "utf8"); + const n = Math.min(left.length, right.length); + for (let i = 0; i < n; i++) { + const d = left[i]! - right[i]!; + if (d !== 0) return d; + } + return left.length - right.length; + }); + return out; +} + +/** Read a scratch file as UTF-8 text under a byte budget. */ +export function readScratchFileUtf8(scratchRoot: string, relativePath: string, maxBytes: number): string { + resolveInsideScratch(scratchRoot, relativePath); + const trusted = openTrustedScratchRoot(scratchRoot); + try { + const fd = openScratchRelativePath(trusted, relativePath, openFlags(fsConstants.O_RDONLY, true), 0); + try { + const stats = fstatSync(fd); + assertRegularFile(stats, relativePath); + if (stats.size > maxBytes) { + throw new FabricTaskError("file exceeds io budget", "budget_exhausted", "environment"); + } + return readAllFromFd(fd, stats.size); + } finally { + closeSync(fd); + } + } finally { + closeTrustedScratchRoot(trusted); + } +} + +/** Write UTF-8 content to a scratch file, replacing any existing bytes. */ +export function writeScratchFileUtf8(scratchRoot: string, relativePath: string, contentUtf8: string, maxBytes: number): number { + const bytes = Buffer.from(contentUtf8, "utf8"); + if (bytes.byteLength > maxBytes) { + throw new FabricTaskError("write exceeds io budget", "budget_exhausted", "environment"); + } + const safe = assertSafeRelativePosixPath(relativePath); + const parentPath = safe.includes("/") ? safe.slice(0, safe.lastIndexOf("/")) : ""; + resolveInsideScratch(scratchRoot, relativePath, { allowMissingFinal: true }); + if (parentPath) { + ensureScratchRelativeDir(scratchRoot, parentPath); + } + const trusted = openTrustedScratchRoot(scratchRoot); + try { + const fd = openScratchRelativePath( + trusted, + relativePath, + openFlags(fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_TRUNC, true), + FILE_MODE, + ); + try { + const stats = fstatSync(fd); + assertRegularFile(stats, relativePath); + writeSync(fd, bytes); + } finally { + closeSync(fd); + } + } finally { + closeTrustedScratchRoot(trusted); + } + return bytes.byteLength; +} + +/** Prove the resolved path cannot escape into a caller-provided user repository root. */ +export function assertNotUnderUserRepo(scratchRoot: string, userRepoRoot: string): void { + const scratch = resolve(scratchRoot); + const repo = resolve(userRepoRoot); + if (scratch === repo || scratch.startsWith(repo + sep)) { + throw new FabricTaskError("scratch must not live inside user repository", "sandbox_violation", "harness"); + } +} diff --git a/src/lab/fabric/subject.ts b/src/lab/fabric/subject.ts new file mode 100644 index 0000000000..d4b2657c5b --- /dev/null +++ b/src/lab/fabric/subject.ts @@ -0,0 +1,106 @@ +import { domainHash, jcsStringify, subjectIdForSubject } from "../digest"; +import type { RouteSubjectV1, TaskSubjectV1 } from "../events/types"; +import { freezeRouteSubject } from "../subject/route-subject"; +import { + FABRIC_COMPATIBILITY_VERSION, + FABRIC_TASK_CLASS_ID, + FABRIC_TASK_CLASS_VERSION, + FABRIC_VERIFIER_ID, + SANDBOX_PROFILE_V1, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, +} from "./constants"; + +const SANDBOX_DOMAIN = "ocx-lab:fabric-sandbox-profile:v1"; +const VERIFIER_DOMAIN = "ocx-lab:fabric-verifier-manifest:v1"; +const FIXTURE_DOMAIN = "ocx-lab:fabric-task-fixture:v1"; + +/** Domain hash of the frozen fabric sandbox profile object. */ +export function sandboxProfileDigest(profile: unknown = SANDBOX_PROFILE_V1): string { + return domainHash(SANDBOX_DOMAIN, jcsStringify(profile)); +} + +/** Canonical exact-tree-diff verifier manifest object for CL-07. */ +export function verifierManifestObject(): Record { + return { + schemaVersion: 1, + verifierId: FABRIC_VERIFIER_ID, + sort: "utf8-bytes-posix-path", + rejectSymlinks: true, + rejectSpecialFiles: true, + rejectPathTraversal: true, + allowAdds: false, + allowDeletes: false, + allowRenames: false, + requiredChange: { + path: SYNTHETIC_VALUE_PATH, + beforeUtf8: SYNTHETIC_BEFORE_UTF8, + afterUtf8: SYNTHETIC_AFTER_UTF8, + }, + }; +} + +/** Domain hash of the fabric verifier manifest object. */ +export function verifierManifestDigest(manifest: unknown = verifierManifestObject()): string { + return domainHash(VERIFIER_DOMAIN, jcsStringify(manifest)); +} + +/** Canonical synthetic-patch task fixture object for CL-07. */ +export function taskFixtureObject(): Record { + return { + schemaVersion: 1, + taskClassId: FABRIC_TASK_CLASS_ID, + taskClassVersion: FABRIC_TASK_CLASS_VERSION, + files: [ + { + path: SYNTHETIC_VALUE_PATH, + contentUtf8: SYNTHETIC_BEFORE_UTF8, + }, + ], + requestedFinal: [ + { + path: SYNTHETIC_VALUE_PATH, + contentUtf8: SYNTHETIC_AFTER_UTF8, + }, + ], + }; +} + +/** Domain hash of the fabric task fixture object. */ +export function taskFixtureDigest(fixture: unknown = taskFixtureObject()): string { + return domainHash(FIXTURE_DOMAIN, jcsStringify(fixture)); +} + +/** Build a frozen TaskSubjectV1 for the synthetic-patch fabric task. */ +export function buildTaskSubjectV1(input: { + routeSubject: RouteSubjectV1; + taskClassId?: string; + taskClassVersion?: string; + taskFixtureDigest?: string; + verifierManifestDigest?: string; + fabricCompatibilityVersion?: string; + sandboxProfileDigest?: string; +}): TaskSubjectV1 { + const routeSubject = freezeRouteSubject(input.routeSubject); + const subject: TaskSubjectV1 = { + subjectSchemaVersion: 1, + subjectKind: "task", + routeSubject, + taskClassId: input.taskClassId ?? FABRIC_TASK_CLASS_ID, + taskClassVersion: input.taskClassVersion ?? FABRIC_TASK_CLASS_VERSION, + taskFixtureDigest: input.taskFixtureDigest ?? taskFixtureDigest(), + verifierManifestDigest: input.verifierManifestDigest ?? verifierManifestDigest(), + fabricCompatibilityVersion: input.fabricCompatibilityVersion ?? FABRIC_COMPATIBILITY_VERSION, + sandboxProfileDigest: input.sandboxProfileDigest ?? sandboxProfileDigest(), + }; + return Object.freeze({ + ...subject, + routeSubject: freezeRouteSubject(subject.routeSubject), + }); +} + +/** Stable subject id for a fabric TaskSubjectV1. */ +export function taskSubjectId(subject: TaskSubjectV1): string { + return subjectIdForSubject(subject); +} diff --git a/src/lab/fabric/types.ts b/src/lab/fabric/types.ts new file mode 100644 index 0000000000..de2780a6f2 --- /dev/null +++ b/src/lab/fabric/types.ts @@ -0,0 +1,134 @@ +import type { FailureClassification } from "../conformance/types"; +import type { ObservationOutcome } from "../constants"; +import type { FailureRecordV1, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; +import type { LabDestinationV1, LabRouteContext } from "../live/types"; + +export type FabricOutcomeKind = ObservationOutcome; + +export type FabricFailureClass = FailureClassification; + +export type FabricExecutionAuthority = "trusted_route" | "harness"; + +export interface FabricLimitsV1 { + maxFiles: number; + maxScratchFiles: number; + maxAggregateIoBytes: number; + maxPatchOperations: number; + totalTimeoutMs: number; + inactivityTimeoutMs: number; + aggregateArtifactBytes: number; + maxArtifacts: number; +} + +export interface SyntheticPatchOperationV1 { + op: "replace"; + path: string; + contentUtf8: string; +} + +export interface SyntheticPatchV1 { + schemaVersion: 1; + operations: SyntheticPatchOperationV1[]; +} + +export interface FabricUsageV1 { + inputBytes: number; + outputBytes: number; + patchOperations: number; + filesTouched: number; + artifactBytes: number; + elapsedMs: number; + inactiveMs: number; +} + +export interface ExactTreeDiffPathSummaryV1 { + path: string; + kind: "unchanged" | "modified" | "added" | "deleted" | "rejected"; + beforeDigest?: string; + afterDigest?: string; + reason?: string; +} + +export interface ExactTreeDiffResultV1 { + verifierId: string; + manifestDigest: string; + passed: boolean; + pathSummaries: ExactTreeDiffPathSummaryV1[]; + reason?: string; +} + +export interface FabricTaskOutcomeV1 { + schemaVersion: 1; + taskClassId: string; + taskClassVersion: string; + routeSubject: RouteSubjectV1; + taskSubject: TaskSubjectV1; + subjectId: string; + taskFixtureDigest: string; + verifierManifestDigest: string; + fabricCompatibilityVersion: string; + sandboxProfileDigest: string; + startedAt: number; + completedAt: number; + limits: FabricLimitsV1; + usage: FabricUsageV1; + outcome: FabricOutcomeKind; + verifier: ExactTreeDiffResultV1; + failure?: FailureRecordV1; + artifactDigests: string[]; + sourceRefs?: string[]; +} + +/** Authoritative route execution input for trusted fabric patch producers. */ +export interface FabricPatchExecutorInput { + routeContext: LabRouteContext; + destination: LabDestinationV1; + routeSubject: RouteSubjectV1; + scratchRoot: string; + reportActivity: () => void; + signal: AbortSignal; +} + +export type FabricPatchExecutor = (input: FabricPatchExecutorInput) => SyntheticPatchV1 | Promise; + +/** Opaque host-issued capability for exact-route fabric patch production. */ +export interface TrustedFabricPatchExecutor { + execute(input: FabricPatchExecutorInput): Promise; + readonly executorModulePath: string; +} + +/** Sealed fabric run result including evidence authority (not part of outcome JSON). */ +export interface FabricTaskRunResult { + outcome: FabricTaskOutcomeV1; + executionAuthority: FabricExecutionAuthority; +} + +export type FabricHarnessProducerKind = + | "deterministic_correct" + | "deterministic_wrong" + | "infinite_sync" + | "never_resolve" + | "mutate_after_delay" + | "periodic_activity" + | "activity_until_total" + | "flood_stdout"; + +/** @deprecated Test-only harness producer; use FabricHarnessProducerKind via runFabricSyntheticPatchTaskHarness. */ +export type SyntheticPatchProducer = (ctx: { + taskClassId: string; + taskClassVersion: string; + scratchRoot: string; + reportActivity: () => void; +}) => SyntheticPatchV1 | Promise; + +/** Typed error for fabric task producer, verifier, and sandbox failures. */ +export class FabricTaskError extends Error { + override readonly name = "FabricTaskError"; + constructor( + message: string, + readonly code: FabricFailureClass, + readonly attribution: FailureRecordV1["attribution"] = "harness", + ) { + super(message); + } +} diff --git a/src/lab/fabric/verifier.ts b/src/lab/fabric/verifier.ts new file mode 100644 index 0000000000..88c58f7b89 --- /dev/null +++ b/src/lab/fabric/verifier.ts @@ -0,0 +1,98 @@ +import { createHash } from "node:crypto"; +import { + FABRIC_LIMITS, + FABRIC_VERIFIER_ID, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, +} from "./constants"; +import { readScratchFileUtf8, walkScratchFiles } from "./scratch"; +import { verifierManifestDigest } from "./subject"; +import type { ExactTreeDiffResultV1 } from "./types"; +import { FabricTaskError } from "./types"; + +/** SHA-256 hex digest of a UTF-8 string for verifier tree comparisons. */ +function sha256Utf8(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +/** + * Deterministic exact-tree-diff-v1 verifier. No LLM. Walks only the scratch tree. + */ +export function verifyExactTreeDiffV1(scratchRoot: string): ExactTreeDiffResultV1 { + const manifestDigest = verifierManifestDigest(); + const files = walkScratchFiles(scratchRoot); + let aggregate = 0; + for (const file of files) { + aggregate += file.byteLength; + if (aggregate > FABRIC_LIMITS.maxAggregateIoBytes) { + throw new FabricTaskError("tree exceeds io budget", "budget_exhausted", "environment"); + } + } + if (files.length !== 1 || files[0]!.relativePosix !== SYNTHETIC_VALUE_PATH) { + const unexpected = files + .filter((row) => row.relativePosix !== SYNTHETIC_VALUE_PATH) + .map((row) => row.relativePosix); + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [ + ...unexpected.map((path) => ({ + path, + kind: "added" as const, + reason: "unexpected_file", + })), + ...(files.some((row) => row.relativePosix === SYNTHETIC_VALUE_PATH) + ? [] + : [{ path: SYNTHETIC_VALUE_PATH, kind: "deleted" as const, reason: "missing_required_file" }]), + ], + reason: files.length === 0 ? "deleted_required_file" : "unexpected_tree_shape", + }; + } + const after = readScratchFileUtf8(scratchRoot, SYNTHETIC_VALUE_PATH, FABRIC_LIMITS.maxAggregateIoBytes); + const beforeDigest = sha256Utf8(SYNTHETIC_BEFORE_UTF8); + const afterDigest = sha256Utf8(after); + const expectedDigest = sha256Utf8(SYNTHETIC_AFTER_UTF8); + if (after === SYNTHETIC_BEFORE_UTF8) { + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "unchanged", + beforeDigest, + afterDigest, + reason: "unchanged", + }], + reason: "unchanged_file", + }; + } + if (after !== SYNTHETIC_AFTER_UTF8) { + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: false, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "modified", + beforeDigest, + afterDigest, + reason: "wrong_final_bytes", + }], + reason: `expected_digest_${expectedDigest}`, + }; + } + return { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest, + passed: true, + pathSummaries: [{ + path: SYNTHETIC_VALUE_PATH, + kind: "modified", + beforeDigest, + afterDigest, + }], + }; +} diff --git a/src/lab/index.ts b/src/lab/index.ts index b410b147ff..3020e556f4 100644 --- a/src/lab/index.ts +++ b/src/lab/index.ts @@ -18,6 +18,7 @@ export * from "./ledger/artifact-refs"; export * from "./projection/verification"; export * from "./observe/from-conformance"; export * from "./observe/from-live"; +export * from "./fabric"; export * from "./live/manifest"; export * from "./live/runner"; export * from "./live/executor"; diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 204d96dc58..ff0ac451ec 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -1,13 +1,17 @@ import { closeSync, + constants as fsConstants, existsSync, fsyncSync, mkdirSync, openSync, + readFileSync, readSync, statSync, + unlinkSync, writeSync, } from "node:fs"; +import { randomBytes } from "node:crypto"; import { dirname } from "node:path"; import { jcsStringify } from "../digest"; import { MAX_SERIALIZED_EVENT_BYTES } from "../constants"; @@ -21,6 +25,153 @@ export interface LedgerStore { replay(): ReplayResult; } +const eventIdIndexByLedger = new Map>(); +const LEDGER_LOCK_STALE_MS = 60_000; +const LEDGER_LOCK_WAIT_MS = 5_000; + +interface LedgerLockMeta { + pid: number; + createdAt: number; + token: string; +} + +/** Block synchronously for the given duration (ledger lock retry only). */ +function sleepSyncMs(ms: number): void { + const end = Date.now() + ms; + while (Date.now() < end) { + /* spin */ + } +} + +/** Read pid, createdAt, and token metadata from a ledger lock file, if well-formed. */ +function readLedgerLockMeta(lockPath: string): LedgerLockMeta | null { + try { + const parsed = JSON.parse(readFileSync(lockPath, "utf8")) as LedgerLockMeta; + if ( + typeof parsed.pid === "number" + && typeof parsed.createdAt === "number" + && typeof parsed.token === "string" + && parsed.token.length > 0 + ) { + return parsed; + } + } catch { + /* ignore */ + } + return null; +} + +/** Return true when the lock holder process is still running. */ +function isLockHolderAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ESRCH") return false; + if (code === "EPERM") return true; + return false; + } +} + +/** Return true when a ledger lock file has dead metadata and can be recovered. */ +function isLedgerLockStale(lockPath: string): boolean { + const meta = readLedgerLockMeta(lockPath); + if (!meta) { + try { + return Date.now() - statSync(lockPath).mtimeMs > LEDGER_LOCK_STALE_MS; + } catch { + return false; + } + } + if (isLockHolderAlive(meta.pid)) return false; + return Date.now() - meta.createdAt > LEDGER_LOCK_STALE_MS; +} + +/** Create a ledger lock file exclusively, recovering stale locks when needed. */ +function tryAcquireLedgerLock(lockPath: string, deadline: number): { fd: number; token: string } { + while (Date.now() < deadline) { + const token = randomBytes(16).toString("hex"); + let fd: number; + try { + fd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600); + } catch (error) { + if (existsSync(lockPath) && isLedgerLockStale(lockPath)) { + try { + unlinkSync(lockPath); + } catch (unlinkError) { + if (Date.now() >= deadline) throw unlinkError; + sleepSyncMs(10); + } + continue; + } + if (Date.now() >= deadline) throw error; + sleepSyncMs(10); + continue; + } + try { + const metadataBytes = Buffer.from(JSON.stringify({ pid: process.pid, createdAt: Date.now(), token }), "utf8"); + const written = writeSync(fd, metadataBytes); + if (written !== metadataBytes.byteLength) { + throw new LabValidationError("short_write", "ledger lock metadata write incomplete"); + } + } catch (error) { + closeSync(fd); + try { + unlinkSync(lockPath); + } catch { + /* best-effort */ + } + throw error; + } + return { fd, token }; + } + throw new Error("ledger lock acquisition timed out"); +} + +/** Release a ledger lock only when the token still matches the lock file. */ +function releaseLedgerLock(lockPath: string, lockFd: number, token: string): void { + try { + closeSync(lockFd); + } catch { + /* ignore */ + } + try { + const meta = readLedgerLockMeta(lockPath); + if (meta?.token === token) unlinkSync(lockPath); + } catch { + /* best-effort */ + } +} + +/** Run a ledger mutation while holding the compatibility ledger lock file. */ +function withLedgerLock(ledgerPath: string, fn: () => T): T { + const lockPath = `${ledgerPath}.lock`; + mkdirSync(dirname(ledgerPath), { recursive: true, mode: 0o700 }); + const deadline = Date.now() + LEDGER_LOCK_WAIT_MS; + const { fd, token } = tryAcquireLedgerLock(lockPath, deadline); + try { + return fn(); + } finally { + releaseLedgerLock(lockPath, fd, token); + } +} + +/** Load or build the in-memory event-id index for a ledger path. */ +function loadEventIdIndex(ledgerPath: string): Set { + let index = eventIdIndexByLedger.get(ledgerPath); + if (!index) { + index = new Set(); + if (existsSync(ledgerPath)) { + for (const event of replayLabLedger(ledgerPath).events) { + index.add(event.eventId); + } + } + eventIdIndexByLedger.set(ledgerPath, index); + } + return index; +} + /** Durable append of one validated event as a single JSONL line + fsync. */ export function appendLabEvent(ledgerPath: string, event: LabEvent): void { const validated = validateLabEvent(event); @@ -41,6 +192,28 @@ export function appendLabEvent(ledgerPath: string, event: LabEvent): void { } finally { closeSync(fd); } + loadEventIdIndex(ledgerPath).add(validated.eventId); +} + +/** + * Append only when eventId is absent. Uses an exclusive lock file plus a + * process-local event-id index refreshed under the lock. + */ +export function appendLabEventIfAbsent(ledgerPath: string, event: LabEvent): boolean { + const validated = validateLabEvent(event); + return withLedgerLock(ledgerPath, () => { + // Refresh from disk under the lock so concurrent writers are visible. + const fresh = new Set(); + if (existsSync(ledgerPath)) { + for (const row of replayLabLedger(ledgerPath).events) { + fresh.add(row.eventId); + } + } + eventIdIndexByLedger.set(ledgerPath, fresh); + if (fresh.has(validated.eventId)) return false; + appendLabEvent(ledgerPath, validated); + return true; + }); } function processLine( diff --git a/src/lab/live/transport.ts b/src/lab/live/transport.ts index 90b0b8bea2..f1dac8e211 100644 --- a/src/lab/live/transport.ts +++ b/src/lab/live/transport.ts @@ -104,8 +104,10 @@ export function classifyTransportError(error: unknown): { classification: Failur case "network_blocked": return { classification: "network_failure", secondaryCode: code }; case "region_blocked": return { classification: "region_blocked", secondaryCode: code }; case "provider_transient": return { classification: "provider_transient", secondaryCode: code }; - case "connect_timeout": case "first_byte_timeout": case "inactivity_timeout": case "total_timeout": + case "connect_timeout": case "first_byte_timeout": case "total_timeout": return { classification: "timeout", secondaryCode: code }; + case "inactivity_timeout": + return { classification: "inactivity_timeout", secondaryCode: code }; case "request_limit": case "input_byte_limit": case "output_byte_limit": case "output_token_limit": case "tool_call_limit": case "artifact_byte_limit": case "memory_limit": case "child_process_limit": return { classification: "budget_exhausted", secondaryCode: code }; diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 35dbd06001..ba353f3050 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -90,7 +90,9 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { if (result.passed) return "pass"; switch (result.classification) { case "timeout": + case "inactivity_timeout": case "budget_exhausted": + case "sandbox_violation": case "authentication_blocked": case "quota_blocked": case "region_blocked": @@ -99,6 +101,8 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { return "blocked"; case "inconclusive": case "harness_failure": + case "malformed_producer_outcome": + case "layer_subject_mismatch": return "inconclusive"; case "protocol_failure": case "capability_failure": diff --git a/src/lab/observe/from-live.ts b/src/lab/observe/from-live.ts index 2169daae1e..857874ed49 100644 --- a/src/lab/observe/from-live.ts +++ b/src/lab/observe/from-live.ts @@ -20,7 +20,9 @@ function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcom if (result.passed) return "pass"; switch (result.classification) { case "timeout": + case "inactivity_timeout": case "budget_exhausted": + case "sandbox_violation": case "authentication_blocked": case "quota_blocked": case "region_blocked": @@ -33,7 +35,13 @@ function outcomeFromLiveResult(result: LiveScenarioRunResult): ObservationOutcom return "fail"; case "inconclusive": case "harness_failure": + case "malformed_producer_outcome": + case "layer_subject_mismatch": return "inconclusive"; + default: { + const _never: never = result.classification; + throw new Error(`unmapped failure classification: ${String(_never)}`); + } } } @@ -41,7 +49,7 @@ function failureFromLiveResult(result: LiveScenarioRunResult, retryable: boolean if (result.passed || result.classification === "inconclusive") return undefined; const attribution: FailureRecordV1["attribution"] = result.classification === "harness_failure" ? "harness" - : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "budget_exhausted"].includes(result.classification) + : ["authentication_blocked", "quota_blocked", "region_blocked", "network_failure", "provider_transient", "timeout", "inactivity_timeout", "budget_exhausted"].includes(result.classification) ? "environment" : "route"; return { diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 068db32033..754b78bb4d 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -1,12 +1,63 @@ -import { chmodSync, mkdirSync, statSync } from "node:fs"; -import { join } from "node:path"; +import { chmodSync, existsSync, mkdirSync, lstatSync, realpathSync } from "node:fs"; +import { basename, dirname, join, resolve, sep } from "node:path"; import { getConfigDir } from "../config"; -function ensureRestrictedDir(dir: string): void { - mkdirSync(dir, { recursive: true, mode: 0o700 }); - if (process.platform === "win32") return; - const mode = statSync(dir).mode & 0o777; - if (mode !== 0o700) chmodSync(dir, 0o700); +/** Return true when path is the Lab root or a descendant path component. */ +function isAtOrBelowLabBoundary(path: string, labRootBoundary: string): boolean { + const normalizedPath = resolve(path); + const normalizedBoundary = resolve(labRootBoundary); + if (normalizedPath === normalizedBoundary) return true; + const prefix = normalizedBoundary + sep; + return normalizedPath.startsWith(prefix) && normalizedPath.length > normalizedBoundary.length; +} + +/** Reject symlink, non-directory, and reparse-point substitutions for a Lab-owned component. */ +function assertRestrictedDirectoryComponent(path: string): void { + const componentStats = lstatSync(path); + if (componentStats.isSymbolicLink()) { + throw new Error(`restricted directory component is a symbolic link: ${path}`); + } + if (!componentStats.isDirectory()) { + throw new Error(`restricted path component is not a directory: ${path}`); + } + const canonical = resolve(realpathSync.native(path)); + const canonicalParent = resolve(realpathSync.native(dirname(path))); + const expectedCanonical = resolve(join(canonicalParent, basename(path))); + if (canonical !== expectedCanonical) { + throw new Error(`restricted directory component contains a link or reparse-point substitution: ${path}`); + } +} + +/** + * Create (or harden) a directory to mode 0o700 without following symlinks on Lab-owned + * components. Ancestors above labRootBoundary are treated as infrastructure and are not + * inspected for symlinks (for example macOS /var → /private/var). + */ +export function ensureRestrictedDir(dir: string, labRootBoundary: string): void { + const abs = resolve(dir); + const boundary = resolve(labRootBoundary); + const parts = abs.split(sep); + let current = parts[0] === "" ? sep : parts[0]!; + for (const part of parts.slice(1)) { + if (part === "") continue; + current = join(current, part); + try { + mkdirSync(current, { mode: 0o700 }); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "EEXIST") throw error; + } + if (isAtOrBelowLabBoundary(current, boundary)) { + assertRestrictedDirectoryComponent(current); + } + } + if (!existsSync(abs)) return; + if (!isAtOrBelowLabBoundary(abs, boundary)) return; + assertRestrictedDirectoryComponent(abs); + if (process.platform !== "win32") { + const mode = lstatSync(abs).mode & 0o777; + if (mode !== 0o700) chmodSync(abs, 0o700); + } } /** Canonical Compatibility Lab state root under the OpenCodex config dir. */ @@ -52,8 +103,10 @@ export function ensureLabDirs(configDir = getConfigDir()): { const artifactsDir = labArtifactsDir(configDir); const scratchDir = labScratchDir(configDir); const exportDir = labExportDir(configDir); - ensureRestrictedDir(root); - ensureRestrictedDir(artifactsDir); + ensureRestrictedDir(root, root); + ensureRestrictedDir(artifactsDir, root); + ensureRestrictedDir(scratchDir, root); + ensureRestrictedDir(exportDir, root); return { root, ledgerPath: labLedgerPath(configDir), diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index db0bc683e3..cbc0a2acc9 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -301,6 +301,50 @@ function isMatchedCapabilityAbsenceControl(obs: ObservationEvent): boolean { ); } +function evaluateRequiredPassVerdict(input: { + suiteManifest: SuiteManifestV1 | null; + ordered: ObservationEvent[]; + executionMode: ObservationEvent["executionMode"]; + subject?: ObservationEvent["subject"]; + routeSupportedClaims?: readonly string[]; + fabricCapability?: { + harnessFeatures: readonly string[]; + platforms: readonly string[]; + routePreconditions: readonly string[]; + }; + loadScenarioManifest?: (digest: string) => Record | null; + loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"]; + asOf: number; +}): { verdict: CompatibilityVerdict; notes: string[] } { + const notes: string[] = []; + if (!input.suiteManifest) { + return { verdict: "PROBED", notes: ["suite_manifest_unavailable"] }; + } + const evaluation = evaluateAllApplicableRequiredPassV1( + input.suiteManifest, + input.ordered, + input.executionMode, + { + subject: input.subject, + routeSupportedClaims: input.routeSupportedClaims, + fabricCapability: input.fabricCapability, + loadScenarioManifest: input.loadScenarioManifest, + loadScenarioRequirements: input.loadScenarioRequirements, + asOf: input.asOf, + }, + ); + notes.push(...evaluation.notes); + if (evaluation.applicableRequiredScenarioIds.length === 0) { + return { verdict: "UNKNOWN", notes }; + } + if (evaluation.canVerify) { + notes.push("all-applicable-required-pass-v1"); + return { verdict: "VERIFIED", notes }; + } + notes.push("incomplete_required_coverage"); + return { verdict: "PROBED", notes }; +} + function projectObservationGroup( key: ProjectionKey, ordered: ObservationEvent[], @@ -310,6 +354,11 @@ function projectObservationGroup( loadScenarioManifest?: (digest: string) => Record | null; loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"]; routeSupportedClaims?: readonly string[]; + fabricCapability?: { + harnessFeatures: readonly string[]; + platforms: readonly string[]; + routePreconditions: readonly string[]; + }; } = {}, ): DerivedVerdict { const contributing: string[] = []; @@ -353,60 +402,47 @@ function projectObservationGroup( verdict = "PROBED"; notes.push("mixed_execution_modes"); } else if (key.evidenceLayer === "protocol_conformance" && newestCurrent?.executionMode === "fixture") { - if (!suiteManifest) { - verdict = "PROBED"; - notes.push("suite_manifest_unavailable"); - } else { - const evaluation = evaluateAllApplicableRequiredPassV1( - suiteManifest, - ordered, - newestCurrent.executionMode, - { - subject: newestCurrent.subject.subjectKind === "protocol" ? newestCurrent.subject : undefined, - loadScenarioManifest: opts.loadScenarioManifest, - loadScenarioRequirements: opts.loadScenarioRequirements, - asOf, - }, - ); - notes.push(...evaluation.notes); - if (evaluation.applicableRequiredScenarioIds.length === 0) { - verdict = "UNKNOWN"; - } else if (evaluation.canVerify) { - verdict = "VERIFIED"; - notes.push("all-applicable-required-pass-v1"); - } else { - verdict = "PROBED"; - notes.push("incomplete_required_coverage"); - } - } + const evaluated = evaluateRequiredPassVerdict({ + suiteManifest, + ordered, + executionMode: newestCurrent.executionMode, + subject: newestCurrent.subject.subjectKind === "protocol" ? newestCurrent.subject : undefined, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }); + verdict = evaluated.verdict; + notes.push(...evaluated.notes); } else if (key.evidenceLayer === "live_route_compatibility" && newestCurrent?.executionMode === "live") { - if (!suiteManifest) { - verdict = "PROBED"; - notes.push("suite_manifest_unavailable"); - } else { - const evaluation = evaluateAllApplicableRequiredPassV1( - suiteManifest, - ordered, - newestCurrent.executionMode, - { - subject: newestCurrent.subject.subjectKind === "route" ? newestCurrent.subject : undefined, - routeSupportedClaims: opts.routeSupportedClaims, - loadScenarioManifest: opts.loadScenarioManifest, - loadScenarioRequirements: opts.loadScenarioRequirements, - asOf, - }, - ); - notes.push(...evaluation.notes); - if (evaluation.applicableRequiredScenarioIds.length === 0) { - verdict = "UNKNOWN"; - } else if (evaluation.canVerify) { - verdict = "VERIFIED"; - notes.push("all-applicable-required-pass-v1"); - } else { - verdict = "PROBED"; - notes.push("incomplete_required_coverage"); - } - } + const evaluated = evaluateRequiredPassVerdict({ + suiteManifest, + ordered, + executionMode: newestCurrent.executionMode, + subject: newestCurrent.subject.subjectKind === "route" ? newestCurrent.subject : undefined, + routeSupportedClaims: opts.routeSupportedClaims, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }); + verdict = evaluated.verdict; + notes.push(...evaluated.notes); + } else if (key.evidenceLayer === "task_effectiveness" && newestCurrent?.executionMode === "fabric") { + const evaluated = evaluateRequiredPassVerdict({ + suiteManifest, + ordered, + executionMode: newestCurrent.executionMode, + subject: newestCurrent.subject.subjectKind === "task" ? newestCurrent.subject : undefined, + fabricCapability: opts.fabricCapability ?? { + harnessFeatures: ["fabric-scratch-v1"], + platforms: [process.platform, "*"], + routePreconditions: ["exact-route-subject"], + }, + loadScenarioManifest: opts.loadScenarioManifest, + loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, + }); + verdict = evaluated.verdict; + notes.push(...evaluated.notes); } else { verdict = "PROBED"; } diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index ac48e342d1..86ef9d6f08 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -1,4 +1,4 @@ -import type { ObservationEvent, ProtocolSubjectV1, RouteSubjectV1 } from "../events/types"; +import type { ObservationEvent, ProtocolSubjectV1, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; import { EVIDENCE_LAYERS, type ExecutionMode } from "../constants"; import type { SuiteManifestV1 } from "../conformance/suite-manifest"; import type { VerificationRole } from "../conformance/types"; @@ -19,6 +19,9 @@ export interface ScenarioRequirements { upstreamProtocols?: string[]; surfaces?: string[]; requiredClaims?: string[]; + requiredHarnessFeatures?: string[]; + platforms?: string[]; + routePreconditions?: string[]; freshness?: { maxAgeMs: number | null }; } @@ -38,6 +41,7 @@ export function isScenarioApplicable( return true; } +/** Match protocol subject fields against scenario requirement lists. */ function scenarioApplicableToRequirements( requirements: ScenarioRequirements, subject: ProtocolSubjectV1, @@ -71,10 +75,32 @@ export function routeSubjectApplicableToRequirements( ); } +/** Task-layer applicability against harness/platform/precondition state. */ +export function taskSubjectApplicableToRequirements( + requirements: ScenarioRequirements, + capability: { + harnessFeatures: readonly string[]; + platforms: readonly string[]; + routePreconditions: readonly string[]; + }, +): boolean { + const platforms = requirements.platforms ?? ["*"]; + const features = requirements.requiredHarnessFeatures ?? []; + const preconditions = requirements.routePreconditions ?? []; + const platformOk = platforms.includes("*") + || capability.platforms.includes("*") + || platforms.some((platform) => capability.platforms.includes(platform)); + const featuresOk = features.every((feature) => capability.harnessFeatures.includes(feature)); + const preconditionsOk = preconditions.every((item) => capability.routePreconditions.includes(item)); + return platformOk && featuresOk && preconditionsOk; +} + +/** Return true for non-negative integer values used in freshness parsing. */ function isNonNegativeInteger(value: unknown): value is number { return typeof value === "number" && Number.isInteger(value) && value >= 0; } +/** Parse a freshness object from a scenario manifest fragment. */ function parseFreshness(value: unknown): { maxAgeMs: number | null } | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; const maxAgeMs = (value as { maxAgeMs?: unknown }).maxAgeMs; @@ -83,6 +109,7 @@ function parseFreshness(value: unknown): { maxAgeMs: number | null } | null { return { maxAgeMs }; } +/** Parse a string array requirement field from a manifest fragment. */ function parseStringArray(value: unknown): string[] | null { if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) return null; return value; @@ -101,9 +128,28 @@ function scenarioContractFromManifest( if (!inboundProtocols || !upstreamProtocols || !surfaces) return null; const requiredClaims = row.requiredClaims === undefined ? [] : parseStringArray(row.requiredClaims); if (!requiredClaims) return null; + const requiredHarnessFeatures = row.requiredHarnessFeatures === undefined + ? [] + : parseStringArray(row.requiredHarnessFeatures); + if (!requiredHarnessFeatures) return null; + const platforms = row.platforms === undefined ? ["*"] : parseStringArray(row.platforms); + if (!platforms) return null; + const routePreconditions = row.routePreconditions === undefined + ? [] + : parseStringArray(row.routePreconditions); + if (!routePreconditions) return null; const freshness = parseFreshness(scenarioManifest.freshness); if (!freshness) return null; - return { inboundProtocols, upstreamProtocols, surfaces, requiredClaims, freshness }; + return { + inboundProtocols, + upstreamProtocols, + surfaces, + requiredClaims, + requiredHarnessFeatures, + platforms, + routePreconditions, + freshness, + }; } function effectiveMaxAgeMs( @@ -139,9 +185,15 @@ export function evaluateAllApplicableRequiredPassV1( observations: ObservationEvent[], executionMode: ExecutionMode, opts: { - subject?: ProtocolSubjectV1 | RouteSubjectV1; + subject?: ProtocolSubjectV1 | RouteSubjectV1 | TaskSubjectV1; /** For live projection this must come from validated current claim snapshots for subjectId. */ routeSupportedClaims?: readonly string[]; + /** Fabric harness/platform/precondition state for task_effectiveness applicability. */ + fabricCapability?: { + harnessFeatures: readonly string[]; + platforms: readonly string[]; + routePreconditions: readonly string[]; + }; loadScenarioManifest?: LoadScenarioManifest; loadScenarioRequirements?: LoadScenarioRequirements; asOf?: number; @@ -165,6 +217,26 @@ export function evaluateAllApplicableRequiredPassV1( return { applicableRequiredScenarioIds: [], passingRequiredScenarioIds: [], missingRequiredScenarioIds: [], canVerify: false, notes: ["route_claim_state_required"] }; } } + if (suiteManifest.evidenceLayer === "task_effectiveness") { + if (opts.subject?.subjectKind !== "task") { + return { + applicableRequiredScenarioIds: [], + passingRequiredScenarioIds: [], + missingRequiredScenarioIds: [], + canVerify: false, + notes: ["task_subject_required"], + }; + } + if (executionMode !== "fabric") { + return { + applicableRequiredScenarioIds: [], + passingRequiredScenarioIds: [], + missingRequiredScenarioIds: [], + canVerify: false, + notes: ["fabric_execution_mode_required"], + }; + } + } const requiredScenarios = suiteManifest.scenarios.filter( (s) => s.role === "required" || s.role === "negative_control", @@ -198,6 +270,14 @@ export function evaluateAllApplicableRequiredPassV1( if (suiteManifest.evidenceLayer === "live_route_compatibility" && opts.subject?.subjectKind === "route") { if (!routeSubjectApplicableToRequirements(requirements, opts.subject, opts.routeSupportedClaims!)) continue; } + if (suiteManifest.evidenceLayer === "task_effectiveness" && opts.subject?.subjectKind === "task") { + const capability = opts.fabricCapability ?? { + harnessFeatures: ["fabric-scratch-v1"], + platforms: [process.platform, "*"], + routePreconditions: ["exact-route-subject"], + }; + if (!taskSubjectApplicableToRequirements(requirements, capability)) continue; + } applicableRequired.push(s.id); scenarioMaxAgeById.set(s.id, requirements.freshness?.maxAgeMs ?? null); } diff --git a/src/lab/query/catalog.ts b/src/lab/query/catalog.ts index 876f9c4199..d3f8159333 100644 --- a/src/lab/query/catalog.ts +++ b/src/lab/query/catalog.ts @@ -14,15 +14,31 @@ import { import { suiteManifestDigestForCase, } from "../conformance/suite-manifest"; +import { + discoverFabricScenarios, + expandFabricScenario, + fabricSuiteManifestDigest, + loadFabricCaseAuthority, +} from "../fabric/manifest"; import { scenarioManifestDigest } from "../digest"; -import type { CaseRecord } from "../conformance/types"; +import type { CaseRecord, EvidenceLayer, VerificationRole } from "../conformance/types"; import type { CatalogFilters, CatalogScenarioDto } from "./types"; -function mapCaseToCatalogScenario( +type CatalogAuthority = { + manifestDefaults: { + version: string; + suiteVersion: string; + evidenceLayer: EvidenceLayer; + verificationRole: VerificationRole; + freshness: { maxAgeMs: number | null }; + }; +}; + +function mapCaseToCatalogScenario( caseRecord: CaseRecord, - authority: ReturnType, - suiteDigestFn: (caseRecord: CaseRecord, authority: ReturnType) => string, - expandFn: (caseRecord: CaseRecord, authority: ReturnType) => Record, + authority: TAuthority, + suiteDigestFn: (caseRecord: CaseRecord, authority: TAuthority) => string, + expandFn: (caseRecord: CaseRecord, authority: TAuthority) => Record, ): CatalogScenarioDto { const expanded = expandFn(caseRecord, authority); return { @@ -62,6 +78,19 @@ export function queryLabCatalog(filters: CatalogFilters = {}): CatalogScenarioDt items.push(mapCaseToCatalogScenario(caseRecord, authority, liveSuiteManifestDigestForCase, expandLiveScenario)); } } + if (!filters.layer || filters.layer === "task_effectiveness") { + const authority = loadFabricCaseAuthority(); + const suites = filters.suiteId ? [filters.suiteId] : undefined; + const scenarios = discoverFabricScenarios(authority, suites ?? undefined); + for (const caseRecord of scenarios) { + items.push(mapCaseToCatalogScenario( + caseRecord, + authority, + (row, auth) => fabricSuiteManifestDigest(row.suite, auth), + expandFabricScenario, + )); + } + } return items.sort((a, b) => { const layerCmp = a.evidenceLayer.localeCompare(b.evidenceLayer); if (layerCmp !== 0) return layerCmp; diff --git a/src/lib/fabric-task-execution-authority.ts b/src/lib/fabric-task-execution-authority.ts new file mode 100644 index 0000000000..0de6d5bacf --- /dev/null +++ b/src/lib/fabric-task-execution-authority.ts @@ -0,0 +1,7 @@ +import type { TrustedFabricPatchExecutor } from "../lab/fabric/types"; +import { isHostIssuedFabricPatchExecutor } from "./fabric-task-host"; + +/** Trusted exact-route fabric patch executor recognition facade. */ +export function isTrustedFabricPatchExecutor(value: unknown): value is TrustedFabricPatchExecutor { + return isHostIssuedFabricPatchExecutor(value); +} diff --git a/src/lib/fabric-task-host.ts b/src/lib/fabric-task-host.ts new file mode 100644 index 0000000000..15ff341fae --- /dev/null +++ b/src/lib/fabric-task-host.ts @@ -0,0 +1,29 @@ +import type { FabricPatchExecutor, FabricPatchExecutorInput, TrustedFabricPatchExecutor } from "../lab/fabric/types"; + +const HOST_ISSUED_EXECUTORS = new WeakSet(); + +/** + * Trusted host integration boundary for CL-07 exact-route fabric patch production. + * + * @internal host integration only + */ +export function createHostIssuedFabricPatchExecutor( + executorModulePath: string, + execute: FabricPatchExecutor, +): TrustedFabricPatchExecutor { + const capability: TrustedFabricPatchExecutor = Object.freeze({ + executorModulePath, + execute: (input: FabricPatchExecutorInput) => Promise.resolve(execute(input)), + }); + HOST_ISSUED_EXECUTORS.add(capability); + return capability; +} + +/** Internal recognition check consumed by the read-only authority facade. */ +export function isHostIssuedFabricPatchExecutor(value: unknown): value is TrustedFabricPatchExecutor { + if (typeof value !== "object" || value === null || !HOST_ISSUED_EXECUTORS.has(value as object)) return false; + const capability = value as TrustedFabricPatchExecutor; + return typeof capability.executorModulePath === "string" + && capability.executorModulePath.length > 0 + && typeof capability.execute === "function"; +} diff --git a/tests/fixtures/fabric-executors/correct-patch.ts b/tests/fixtures/fabric-executors/correct-patch.ts new file mode 100644 index 0000000000..5d5be44407 --- /dev/null +++ b/tests/fixtures/fabric-executors/correct-patch.ts @@ -0,0 +1,9 @@ +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "../../../src/lab/fabric/types"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "../../../src/lab/fabric/constants"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} diff --git a/tests/helpers/fabric-task-test.ts b/tests/helpers/fabric-task-test.ts new file mode 100644 index 0000000000..264d69a83c --- /dev/null +++ b/tests/helpers/fabric-task-test.ts @@ -0,0 +1,145 @@ +import { writeFileSync, mkdirSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createLabDestination } from "../../src/lab"; +import { runFabricSyntheticPatchTaskForRoute } from "../../src/lab/fabric/executor"; +import type { FabricTaskRunResult } from "../../src/lab/fabric/types"; +import { createHostIssuedFabricPatchExecutor } from "../../src/lib/fabric-task-host"; +import type { LabBehaviorValues, LabRouteContext } from "../../src/lab/live/types"; +import type { TrustedFabricPatchExecutor } from "../../src/lab/fabric/types"; +import { correctSyntheticPatch } from "../../src/lab/fabric/executor"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const FIXTURE_DIR = join(REPO_ROOT, "tests", "fixtures", "fabric-executors"); + +function repoImport(subpath: string): string { + return join(REPO_ROOT, subpath).replace(/\\/g, "/"); +} + +export function fabricBehavior(adapter: string, upstreamProtocol: string): LabBehaviorValues { + return { + "wire.adapter": { source: "lab_forced", value: adapter }, + "wire.upstreamProtocol": { source: "lab_forced", value: upstreamProtocol }, + "auth.mode": { source: "provider_config", value: "api_key" }, + "auth.transport": { source: "provider_config", value: "authorization_bearer" }, + "mcp.nativeLocalExec": { source: "lab_forced", value: false }, + "runtime.bunVersion": { source: "lab_forced", value: Bun.version }, + "runtime.platform": { source: "lab_forced", value: process.platform }, + "runtime.arch": { source: "lab_forced", value: process.arch }, + "runtime.streamMode": { source: "lab_forced", value: "auto" }, + "runtime.fastMode": { source: "lab_forced", value: false }, + "runtime.effortCap": { source: "lab_forced", value: null }, + "headers.nonCredentialBehaviorDigest": { source: "provider_config", value: "0".repeat(64) }, + }; +} + +export function fabricMockRoute(overrides: Partial = {}): LabRouteContext { + const base = { + providerId: "provider-a", + providerInstanceKey: "fixture-provider-instance", + clientModelId: "model-a", + upstreamModelId: "model-a", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + baseUrl: "https://api.example.com/v1", + opencodexCompatibilityVersion: "a".repeat(64), + labRunApproval: true, + allowPrivateNetwork: false, + requiredClaims: [], + availableHarnessFeatures: ["fabric-scratch-v1"], + }; + const merged = { ...base, ...overrides }; + return { + ...merged, + behaviorValues: overrides.behaviorValues ?? fabricBehavior(merged.effectiveAdapter, merged.upstreamProtocol), + }; +} + +export function fabricCorrectPatchExecutor(): TrustedFabricPatchExecutor { + const modulePath = join(FIXTURE_DIR, "correct-patch.ts"); + return createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()); +} + +export function fabricRouteBoundPatchExecutor(home: string, expectedProviderId: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, `route-bound-${expectedProviderId}.ts`); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +const expectedProviderId = "${expectedProviderId}"; + +export async function execute(input: FabricPatchExecutorInput): Promise { + if (input.routeSubject.providerId !== expectedProviderId) { + throw new Error("route subject mismatch"); + } + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async (input) => { + if (input.routeSubject.providerId !== expectedProviderId) { + throw new Error("route subject mismatch"); + } + return correctSyntheticPatch(); + }); +} + +export function fabricWrongPatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "wrong-patch.ts"); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + return { schemaVersion: 1, operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: "wrong\\n" }] }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => ({ + schemaVersion: 1, + operations: [{ op: "replace", path: "src/value.txt", contentUtf8: "wrong\n" }], + })); +} + +export function fabricOversizedPatchExecutor(home: string, huge: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "oversized-patch.ts"); + writeFileSync(modulePath, `import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; +const huge = ${JSON.stringify(huge)}; +export async function execute(_input: FabricPatchExecutorInput): Promise { + return { schemaVersion: 1, operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: huge }] }; +}`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => ({ + schemaVersion: 1, + operations: [{ op: "replace", path: "src/value.txt", contentUtf8: huge }], + })); +} + +export async function runTrustedFabricTask( + home: string, + overrides: Partial = {}, + now?: () => number, +): Promise { + const destination = await createLabDestination({ + baseUrl: overrides.baseUrl ?? "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); + return runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(overrides), + destination, + patchExecutor: fabricCorrectPatchExecutor(), + configDir: home, + now, + }); +} diff --git a/tests/lab-evidence-ledger.test.ts b/tests/lab-evidence-ledger.test.ts index 9678c9dd4d..ed633c7b00 100644 --- a/tests/lab-evidence-ledger.test.ts +++ b/tests/lab-evidence-ledger.test.ts @@ -1,4 +1,5 @@ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import * as nodeFs from "node:fs"; import { chmodSync, existsSync, mkdirSync, writeFileSync, symlinkSync, linkSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -30,6 +31,7 @@ import { LAB_PROJECTION_SPEC_VERSION, behaviorFingerprintForCase, } from "../src/lab"; +import { appendLabEventIfAbsent } from "../src/lab/ledger/store"; import { ArtifactFsError, closeTrustedArtifactDir, putArtifactBytes, putNamedDigestBytes, readArtifactBytes, digestFileName } from "../src/lab/artifacts/secure-fs"; import { expandSuiteManifest } from "../src/lab/conformance/suite-manifest"; import { evaluateAllApplicableRequiredPassV1 } from "../src/lab/projection/verification"; @@ -279,6 +281,63 @@ describe("CL-02 ledger append/replay", () => { expect(replay.corruptions.some((c) => c.kind === "duplicate_event")).toBe(true); }); }); + + test("ledger lock metadata write failure propagates without lock-timeout wait", () => { + withHome((home) => { + const event = baseObservation(); + const ledger = join(home, "lab", "compatibility.jsonl"); + mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); + const metadataWriteError = new Error("simulated ledger lock metadata write failure"); + const originalWriteSync = nodeFs.writeSync; + const writeSpy = spyOn(nodeFs, "writeSync").mockImplementation((...args: Parameters) => { + const buffer = args[1]; + if ( + Buffer.isBuffer(buffer) + && buffer.includes('"pid"') + && buffer.includes('"createdAt"') + && buffer.includes('"token"') + ) { + throw metadataWriteError; + } + return originalWriteSync(...args); + }); + try { + const start = Date.now(); + expect(() => appendLabEventIfAbsent(ledger, event)).toThrow(metadataWriteError); + expect(Date.now() - start).toBeLessThan(500); + } finally { + writeSpy.mockRestore(); + } + }); + }); + + test("incomplete ledger lock metadata write removes lock file", () => { + withHome((home) => { + const event = baseObservation(); + const ledger = join(home, "lab", "compatibility.jsonl"); + const lockPath = join(home, "lab", "compatibility.jsonl.lock"); + mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); + const originalWriteSync = nodeFs.writeSync; + const writeSpy = spyOn(nodeFs, "writeSync").mockImplementation((...args: Parameters) => { + const buffer = args[1]; + if ( + Buffer.isBuffer(buffer) + && buffer.includes('"pid"') + && buffer.includes('"createdAt"') + && buffer.includes('"token"') + ) { + return 0; + } + return originalWriteSync(...args); + }); + try { + expect(() => appendLabEventIfAbsent(ledger, event)).toThrow(LabValidationError); + expect(existsSync(lockPath)).toBe(false); + } finally { + writeSpy.mockRestore(); + } + }); + }); }); describe("CL-02 invalidation validation", () => { diff --git a/tests/lab-fabric-persistence-boundary.test.ts b/tests/lab-fabric-persistence-boundary.test.ts new file mode 100644 index 0000000000..572c7ed6cf --- /dev/null +++ b/tests/lab-fabric-persistence-boundary.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, test } from "bun:test"; + +describe("CL-07 fabric persistence export boundary", () => { + test("authority-free persistence helper is module-private", async () => { + const observeExports = await import("../src/lab/fabric/observe"); + + expect("persistFabricOutcome" in observeExports).toBe(false); + expect(typeof observeExports.persistFabricRunResult).toBe("function"); + }); +}); diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts new file mode 100644 index 0000000000..e63c19fa9d --- /dev/null +++ b/tests/lab-fabric-task.test.ts @@ -0,0 +1,1074 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import * as nodeFs from "node:fs"; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + appendLabEvent, + assertNotUnderUserRepo, + assertSafeRelativePosixPath, + assignEventId, + buildTaskSubjectV1, + correctSyntheticPatch, + createLabDestination, + createSyntheticScratch, + fabricDeclaredSandboxPolicy, + FABRIC_VERIFIER_ID, + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + observationFromFabricOutcome, + persistFabricRunResult, + queryLabCatalog, + readVerdictSnapshot, + rebuildLabProjection, + replayLabLedger, + runFabricSyntheticPatchTaskForRoute, + runFabricSyntheticPatchTaskHarness, + sandboxProfileDigest, + subjectIdForSubject, + SYNTHETIC_AFTER_UTF8, + SYNTHETIC_BEFORE_UTF8, + SYNTHETIC_VALUE_PATH, + taskFixtureDigest, + taskFixtureObject, + taskSubjectId, + verifierManifestDigest, + verifierManifestObject, + type RouteSubjectV1, +} from "../src/lab"; +import { FabricTaskError } from "../src/lab/fabric/types"; +import { writeScratchFileUtf8, readScratchFileUtf8 } from "../src/lab/fabric/scratch"; +import { routingProfileIssues } from "../src/routing/profile"; +import { buildInvalidationIndex } from "../src/lab/ledger/invalidation"; +import { createArtifactStore } from "../src/lab/artifacts/store"; +import { ensureLabDirs, ensureRestrictedDir } from "../src/lab/paths"; +import { verifyExactTreeDiffV1 } from "../src/lab/fabric/verifier"; +import { parseSyntheticPatchV1 } from "../src/lab/fabric/patch"; +import { FABRIC_LIMITS } from "../src/lab/fabric/constants"; +import { taskSubjectApplicableToRequirements } from "../src/lab/projection/verification"; +import { createHostIssuedFabricPatchExecutor } from "../src/lib/fabric-task-host"; +import type { TrustedFabricPatchExecutor } from "../src/lab/fabric/types"; +import { + fabricCorrectPatchExecutor, + fabricMockRoute, + fabricOversizedPatchExecutor, + fabricRouteBoundPatchExecutor, + runTrustedFabricTask, +} from "./helpers/fabric-task-test"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); +const CREDENTIAL_CANARY = "credential-canary-abcdefghijklmnopqrstuvwxyz1234567890"; + +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-cl07-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +function repoImport(subpath: string): string { + return join(REPO_ROOT, subpath).replace(/\\/g, "/"); +} + +async function fabricDestination(home: string) { + return createLabDestination({ + baseUrl: "https://api.example.com/v1", + labRunApproval: true, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + configDir: home, + }); +} + +function fabricActivityPatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "activity-patch.ts"); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +export async function execute(input: FabricPatchExecutorInput): Promise { + for (let i = 0; i < 3; i++) { + input.reportActivity(); + await Bun.sleep(Math.floor(${FABRIC_LIMITS.inactivityTimeoutMs} * 0.6)); + } + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async (input) => { + for (let i = 0; i < 3; i++) { + input.reportActivity(); + await Bun.sleep(Math.floor(FABRIC_LIMITS.inactivityTimeoutMs * 0.6)); + } + return correctSyntheticPatch(); + }); +} + +function fabricInactivePatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "inactive-patch.ts"); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + await Bun.sleep(${FABRIC_LIMITS.inactivityTimeoutMs} + 50); + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => { + await Bun.sleep(FABRIC_LIMITS.inactivityTimeoutMs + 50); + return correctSyntheticPatch(); + }); +} + +function fabricTraversalPatchExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "traversal-patch.ts"); + writeFileSync(modulePath, ` +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + return { + schemaVersion: 1, + operations: [{ op: "replace", path: "../outside.txt", contentUtf8: "evil\\n" }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => ({ + schemaVersion: 1, + operations: [{ op: "replace", path: "../outside.txt", contentUtf8: "evil\n" }], + })); +} + +function fabricOutsideScratchWriteExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const outside = join(home, "outside-scratch"); + mkdirSync(outside, { recursive: true }); + const modulePath = join(dir, "outside-write.ts"); + writeFileSync(modulePath, ` +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +const outside = "${outside.replace(/\\/g, "/")}"; + +export async function execute(_input: FabricPatchExecutorInput): Promise { + writeFileSync(join(outside, "evil.txt"), "evil\\n"); + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async () => correctSyntheticPatch()); +} + +function fabricSymlinkSandboxExecutor(home: string): TrustedFabricPatchExecutor { + const dir = join(home, "fabric-executors"); + mkdirSync(dir, { recursive: true }); + const modulePath = join(dir, "symlink-sandbox.ts"); + writeFileSync(modulePath, ` +import { symlinkSync } from "node:fs"; +import { join } from "node:path"; +import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("src/lab/fabric/types")}"; +import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; + +export async function execute(input: FabricPatchExecutorInput): Promise { + const target = join(input.scratchRoot, "src", "value.txt"); + const link = join(input.scratchRoot, "src", "link.txt"); + try { + symlinkSync(target, link); + } catch { + /* platform may reject symlink creation */ + } + return { + schemaVersion: 1, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], + }; +} +`); + return createHostIssuedFabricPatchExecutor(modulePath, async (input) => { + const target = join(input.scratchRoot, "src", "value.txt"); + const link = join(input.scratchRoot, "src", "link.txt"); + try { + symlinkSync(target, link); + } catch { + /* platform may reject symlink creation */ + } + return correctSyntheticPatch(); + }); +} + +function linkDirectory(target: string, linkPath: string): boolean { + try { + symlinkSync(target, linkPath, process.platform === "win32" ? "junction" : "dir"); + return true; + } catch { + return false; + } +} + +const RESTRICTED_LINK_ERROR = /symbolic link|reparse-point substitution/i; + +afterEach(() => { + for (const dir of HOMES.splice(0)) { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + /* ignore */ + } + } + delete process.env.OPENCODEX_HOME; +}); + +function routeSubject(overrides: Partial = {}): RouteSubjectV1 { + return { + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: "provider-a", + providerInstanceFingerprint: "a".repeat(64), + clientModelId: "model-a", + upstreamModelId: "model-a", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + opencodexCompatibilityVersion: "b".repeat(64), + behaviorFingerprint: "c".repeat(64), + endpointFingerprint: "d".repeat(64), + dependencies: [], + ...overrides, + }; +} + +describe("CL-07 task effectiveness producer", () => { + test("canonical TaskSubjectV1 identity is stable", () => { + const subject = buildTaskSubjectV1({ routeSubject: routeSubject() }); + expect(subject.subjectKind).toBe("task"); + expect(taskSubjectId(subject)).toBe(subjectIdForSubject(subject)); + expect(taskSubjectId(buildTaskSubjectV1({ routeSubject: routeSubject() }))).toBe(taskSubjectId(subject)); + }); + + test("route change changes task subject", () => { + const a = taskSubjectId(buildTaskSubjectV1({ routeSubject: routeSubject() })); + const b = taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject({ providerId: "provider-b", endpointFingerprint: "e".repeat(64) }), + })); + expect(a).not.toBe(b); + }); + + test("fixture/verifier/sandbox/fabric version changes change task subject", () => { + const base = taskSubjectId(buildTaskSubjectV1({ routeSubject: routeSubject() })); + expect(taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject(), + taskFixtureDigest: "1".repeat(64), + }))).not.toBe(base); + expect(taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject(), + verifierManifestDigest: "2".repeat(64), + }))).not.toBe(base); + expect(taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject(), + sandboxProfileDigest: "3".repeat(64), + }))).not.toBe(base); + expect(taskSubjectId(buildTaskSubjectV1({ + routeSubject: routeSubject(), + fabricCompatibilityVersion: "other-fabric-v1", + }))).not.toBe(base); + expect(taskFixtureDigest()).toMatch(/^[0-9a-f]{64}$/); + expect(verifierManifestDigest()).toMatch(/^[0-9a-f]{64}$/); + expect(sandboxProfileDigest()).toMatch(/^[0-9a-f]{64}$/); + }); + + test("exact synthetic patch passes via trusted route", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runTrustedFabricTask(home); + expect(result.executionAuthority).toBe("trusted_route"); + expect(result.outcome.outcome).toBe("pass"); + expect(result.outcome.verifier.passed).toBe(true); + }); + + test("deterministic_correct harness passes without trusted authority", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "deterministic_correct", + configDir: home, + }); + expect(result.executionAuthority).toBe("harness"); + expect(result.outcome.outcome).toBe("pass"); + }); + + test("harness execution cannot be persisted as production evidence", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "deterministic_correct", + configDir: home, + }); + expect(() => persistFabricRunResult(result, { configDir: home })).toThrow(FabricTaskError); + try { + persistFabricRunResult(result, { configDir: home }); + } catch (error) { + expect((error as FabricTaskError).code).toBe("malformed_producer_outcome"); + } + const labExports = await import("../src/lab"); + expect("persistFabricOutcome" in labExports).toBe(false); + }); + + test("trusted route execution can be persisted as production evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runTrustedFabricTask(home); + const persisted = persistFabricRunResult(result, { configDir: home }); + expect(existsSync(join(home, "lab", "compatibility.jsonl"))).toBe(true); + expect(persisted.event.evidenceLayer).toBe("task_effectiveness"); + }); + + test("unchanged / wrong / added / deleted trees fail verifier", () => { + const home = tempHome(); + const scratch = createSyntheticScratch(home); + try { + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(verifyExactTreeDiffV1(scratch.root).reason).toBe("unchanged_file"); + + writeScratchFileUtf8(scratch.root, SYNTHETIC_VALUE_PATH, "nope\n", FABRIC_LIMITS.maxAggregateIoBytes); + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(verifyExactTreeDiffV1(scratch.root).reason?.startsWith("expected_digest_")).toBe(true); + + writeScratchFileUtf8(scratch.root, SYNTHETIC_VALUE_PATH, SYNTHETIC_AFTER_UTF8, FABRIC_LIMITS.maxAggregateIoBytes); + writeScratchFileUtf8(scratch.root, "src/extra.txt", "x\n", FABRIC_LIMITS.maxAggregateIoBytes); + expect(verifyExactTreeDiffV1(scratch.root).passed).toBe(false); + expect(verifyExactTreeDiffV1(scratch.root).reason).toBe("unexpected_tree_shape"); + } finally { + scratch.cleanup(); + } + + const deleted = createSyntheticScratch(home); + try { + rmSync(join(deleted.root, "src", "value.txt"), { force: true }); + expect(verifyExactTreeDiffV1(deleted.root).passed).toBe(false); + } finally { + deleted.cleanup(); + } + }); + + test("renamed file fails verifier", () => { + const home = tempHome(); + const scratch = createSyntheticScratch(home); + try { + writeScratchFileUtf8(scratch.root, "src/renamed.txt", SYNTHETIC_AFTER_UTF8, FABRIC_LIMITS.maxAggregateIoBytes); + rmSync(join(scratch.root, "src", "value.txt"), { force: true }); + const result = verifyExactTreeDiffV1(scratch.root); + expect(result.passed).toBe(false); + expect(result.pathSummaries.some((row) => row.kind === "added" || row.kind === "deleted")).toBe(true); + } finally { + scratch.cleanup(); + } + }); + + test("symlink and path traversal are rejected", () => { + expect(() => assertSafeRelativePosixPath("../etc/passwd")).toThrow(); + expect(() => assertSafeRelativePosixPath("/abs")).toThrow(); + expect(() => assertSafeRelativePosixPath("C:\\windows")).toThrow(); + + const home = tempHome(); + const scratch = createSyntheticScratch(home); + try { + const target = join(scratch.root, "src", "value.txt"); + const link = join(scratch.root, "src", "link.txt"); + try { + symlinkSync(target, link); + } catch { + return; + } + expect(() => verifyExactTreeDiffV1(scratch.root)).toThrow(FabricTaskError); + } finally { + scratch.cleanup(); + } + }); + + test("special file is rejected where supported", () => { + if (process.platform === "win32") return; + const home = tempHome(); + const scratch = createSyntheticScratch(home); + try { + const fifo = join(scratch.root, "src", "fifo"); + const created = Bun.spawnSync(["mkfifo", fifo], { stdout: "pipe", stderr: "pipe" }); + if (created.exitCode !== 0) return; + expect(() => verifyExactTreeDiffV1(scratch.root)).toThrow(FabricTaskError); + } finally { + scratch.cleanup(); + } + }); + + test("intermediate symlink cannot redirect scratch IO", () => { + if (process.platform === "win32") return; + const home = tempHome(); + const scratch = createSyntheticScratch(home); + const outside = join(home, "outside.txt"); + writeFileSync(outside, "secret\n"); + try { + const srcDir = join(scratch.root, "src"); + rmSync(srcDir, { recursive: true, force: true }); + try { + symlinkSync(home, srcDir); + } catch { + return; + } + expect(() => readScratchFileUtf8(scratch.root, SYNTHETIC_VALUE_PATH, FABRIC_LIMITS.maxAggregateIoBytes)).toThrow(); + expect(() => writeScratchFileUtf8(scratch.root, SYNTHETIC_VALUE_PATH, "x\n", FABRIC_LIMITS.maxAggregateIoBytes)).toThrow(); + } finally { + scratch.cleanup(); + } + }); + + test("oversized patch fails safely", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const huge = "x".repeat(FABRIC_LIMITS.maxAggregateIoBytes + 8); + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricOversizedPatchExecutor(home, huge), + configDir: home, + }); + expect(result.outcome.outcome).not.toBe("pass"); + expect(result.outcome.failure?.code).toBe("budget_exhausted"); + }); + + test("never_resolve harness terminates on inactivity timeout", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "never_resolve", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(result.outcome.failure?.code).toBe("inactivity_timeout"); + }, 20_000); + + test("infinite_sync harness terminates on total timeout", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "infinite_sync", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(result.outcome.failure?.code).toBe("timeout"); + }, 40_000); + + test("periodic_activity harness survives inactivity within total budget", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "periodic_activity", + configDir: home, + }); + expect(result.outcome.outcome).toBe("pass"); + }, 20_000); + + test("activity_until_total harness terminates on total timeout", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "activity_until_total", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(result.outcome.failure?.code).toBe("timeout"); + }, 40_000); + + test("flood_stdout harness terminates on protocol byte limit", async () => { + const home = tempHome(); + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "flood_stdout", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(result.outcome.failure?.code).toBe("budget_exhausted"); + }, 20_000); + + test("mutate_after_delay harness kills producer and cleans scratch", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "mutate_after_delay", + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); + const scratchBase = join(home, "lab", "scratch"); + await Bun.sleep(6_000); + if (existsSync(scratchBase)) { + expect(readdirSync(scratchBase).some((name) => name.startsWith("fabric-"))).toBe(false); + } + for (const name of existsSync(scratchBase) ? readdirSync(scratchBase) : []) { + if (!name.startsWith("fabric-")) continue; + const latePath = join(scratchBase, name, SYNTHETIC_VALUE_PATH); + if (existsSync(latePath)) { + expect(readFileSync(latePath, "utf8")).not.toBe("late\n"); + } + } + }, 45_000); + + test("inactivity timeout is bounded for trusted route executors", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricInactivePatchExecutor(home), + configDir: home, + }); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + expect(result.outcome.failure?.code).toBe("inactivity_timeout"); + }, 20_000); + + test("activity resets inactivity deadline within total budget", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricActivityPatchExecutor(home), + configDir: home, + }); + expect(result.outcome.outcome).toBe("pass"); + }, 30_000); + + test("inactiveMs on success is meaningful with controlled clock", async () => { + const home = tempHome(); + let tick = 10_000; + const now = () => tick; + const result = await runFabricSyntheticPatchTaskHarness({ + routeSubject: routeSubject(), + harnessKind: "deterministic_correct", + configDir: home, + now, + }); + expect(result.outcome.outcome).toBe("pass"); + expect(result.outcome.usage.inactiveMs).toBeGreaterThanOrEqual(0); + expect(result.outcome.usage.inactiveMs).toBeLessThanOrEqual(result.outcome.usage.elapsedMs); + tick += 2_500; + const trusted = await runTrustedFabricTask(home, {}, now); + expect(trusted.outcome.outcome).toBe("pass"); + expect(trusted.outcome.usage.inactiveMs).toBeGreaterThanOrEqual(0); + expect(trusted.outcome.usage.inactiveMs).toBeLessThanOrEqual(trusted.outcome.usage.elapsedMs); + }); + + test("sandbox violations do not become behavioral_failure", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricSymlinkSandboxExecutor(home), + configDir: home, + }); + if (result.outcome.failure?.code === "sandbox_violation") { + expect(result.outcome.failure.class).not.toBe("behavioral_failure"); + expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); + return; + } + const scratch = createSyntheticScratch(home); + try { + symlinkSync(join(scratch.root, "src", "value.txt"), join(scratch.root, "src", "link.txt")); + expect(() => verifyExactTreeDiffV1(scratch.root)).toThrow(FabricTaskError); + try { + verifyExactTreeDiffV1(scratch.root); + } catch (error) { + expect((error as FabricTaskError).code).toBe("sandbox_violation"); + } + } catch { + /* symlink unsupported on this platform */ + } finally { + scratch.cleanup(); + } + }); + + test("declared sandbox policy denies network MCP shell and user repo", () => { + expect(fabricDeclaredSandboxPolicy()).toEqual({ + network: false, + userMcp: false, + arbitraryShell: false, + userRepository: false, + }); + }); + + test("untrusted patch executor is rejected before isolation", async () => { + const home = tempHome(); + const fakeExecutor = { + executorModulePath: join(home, "evil.ts"), + execute: async () => correctSyntheticPatch(), + }; + await expect(runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fakeExecutor as TrustedFabricPatchExecutor, + configDir: home, + })).rejects.toThrow(FabricTaskError); + }); + + test("authoritative route subject is bound into trusted outcomes", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const routeContext = fabricMockRoute({ providerId: "provider-bound" }); + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext, + destination: await fabricDestination(home), + patchExecutor: fabricRouteBoundPatchExecutor(home, "provider-bound"), + configDir: home, + }); + expect(result.outcome.routeSubject.providerId).toBe("provider-bound"); + expect(result.outcome.taskSubject.routeSubject.providerId).toBe("provider-bound"); + expect(result.outcome.routeSubject).toEqual(result.outcome.taskSubject.routeSubject); + }); + + test("patch path traversal is rejected at scratch apply boundary", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricTraversalPatchExecutor(home), + configDir: home, + }); + expect(result.outcome.outcome).not.toBe("pass"); + expect(result.outcome.failure?.code).toBe("sandbox_violation"); + expect(existsSync(join(home, "outside.txt"))).toBe(false); + }); + + test("adversarial executor direct write outside scratch is not production evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outside = join(home, "outside-scratch"); + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricOutsideScratchWriteExecutor(home), + configDir: home, + }); + expect(result.outcome.outcome).toBe("pass"); + expect(existsSync(join(outside, "evil.txt"))).toBe(true); + expect(() => persistFabricRunResult({ + ...result, + executionAuthority: "harness", + }, { configDir: home })).toThrow(FabricTaskError); + }); + + test("user repository cannot host the scratch root", () => { + const home = tempHome(); + const repo = join(home, "user-repo"); + mkdirSync(join(repo, "src"), { recursive: true }); + writeFileSync(join(repo, "src", "value.txt"), "before\n"); + expect(() => assertNotUnderUserRepo(repo, repo)).toThrow(); + const scratch = createSyntheticScratch(home); + try { + expect(existsSync(join(scratch.root, "src", "value.txt"))).toBe(true); + expect(scratch.root.startsWith(join(home, "lab", "scratch"))).toBe(true); + } finally { + scratch.cleanup(); + } + }); + + test("malformed producer result is rejected", () => { + expect(() => parseSyntheticPatchV1({ schemaVersion: 1, operations: [], extra: true })).toThrow(); + expect(() => parseSyntheticPatchV1({ + schemaVersion: 2, + operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: "after\n" }], + })).toThrow(); + expect(() => observationFromFabricOutcome({ + schemaVersion: 1, + taskSubject: { subjectKind: "route" }, + })).toThrow(); + let incompleteError: unknown; + try { + observationFromFabricOutcome({ schemaVersion: 1 }); + } catch (error) { + incompleteError = error; + } + expect((incompleteError as FabricTaskError | undefined)?.name).toBe("FabricTaskError"); + }); + + test("fixture and verifier manifests derive from frozen constants", () => { + const verifier = verifierManifestObject() as { + verifierId: string; + requiredChange: { path: string; beforeUtf8: string; afterUtf8: string }; + }; + expect(verifier.verifierId).toBe(FABRIC_VERIFIER_ID); + expect(verifier.requiredChange).toEqual({ + path: SYNTHETIC_VALUE_PATH, + beforeUtf8: SYNTHETIC_BEFORE_UTF8, + afterUtf8: SYNTHETIC_AFTER_UTF8, + }); + const fixture = taskFixtureObject() as { + files: Array<{ path: string; contentUtf8: string }>; + requestedFinal: Array<{ path: string; contentUtf8: string }>; + }; + expect(fixture.files[0]).toEqual({ path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_BEFORE_UTF8 }); + expect(fixture.requestedFinal[0]).toEqual({ path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }); + }); + + test("task applicability excludes mismatched platform and harness features", () => { + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: ["fabric-scratch-v1"], + platforms: ["linux"], + routePreconditions: ["exact-route-subject"], + }, { + harnessFeatures: ["fabric-scratch-v1"], + platforms: ["win32"], + routePreconditions: ["exact-route-subject"], + })).toBe(false); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: ["fabric-scratch-v1", "missing-feature"], + platforms: ["*"], + routePreconditions: ["exact-route-subject"], + }, { + harnessFeatures: ["fabric-scratch-v1"], + platforms: ["win32"], + routePreconditions: ["exact-route-subject"], + })).toBe(false); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: ["fabric-scratch-v1"], + platforms: ["*"], + routePreconditions: ["exact-route-subject"], + }, { + harnessFeatures: ["fabric-scratch-v1"], + platforms: ["win32"], + routePreconditions: ["exact-route-subject"], + })).toBe(true); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: ["fabric-scratch-v1"], + routePreconditions: ["exact-route-subject"], + }, { + harnessFeatures: ["fabric-scratch-v1"], + platforms: ["win32"], + routePreconditions: ["exact-route-subject"], + })).toBe(true); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: [], + platforms: ["linux"], + routePreconditions: [], + }, { + harnessFeatures: [], + platforms: ["win32", "*"], + routePreconditions: [], + })).toBe(true); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: [], + platforms: ["linux", "darwin"], + routePreconditions: [], + }, { + harnessFeatures: [], + platforms: ["linux"], + routePreconditions: [], + })).toBe(true); + expect(taskSubjectApplicableToRequirements({ + requiredHarnessFeatures: [], + platforms: ["linux", "darwin"], + routePreconditions: [], + }, { + harnessFeatures: [], + platforms: ["win32"], + routePreconditions: [], + })).toBe(false); + }); + + test("malformed nested outcome fields throw FabricTaskError", async () => { + const home = tempHome(); + const base = (await runTrustedFabricTask(home)).outcome; + expect(() => observationFromFabricOutcome({ ...base, verifier: {} })).toThrow(FabricTaskError); + try { + observationFromFabricOutcome({ + ...base, + usage: { ...base.usage, inputBytes: -1 }, + }); + throw new Error("expected malformed usage to throw"); + } catch (error) { + expect(error).toBeInstanceOf(FabricTaskError); + expect((error as FabricTaskError).code).toBe("malformed_producer_outcome"); + } + try { + observationFromFabricOutcome({ + ...base, + routeSubject: routeSubject({ providerId: "other-provider" }), + }); + throw new Error("expected layer mismatch to throw"); + } catch (error) { + expect(error).toBeInstanceOf(FabricTaskError); + expect((error as FabricTaskError).code).toBe("layer_subject_mismatch"); + } + }); + + test("verifier summary artifact omits raw credential diagnostics", async () => { + const home = tempHome(); + const paths = ensureLabDirs(home); + const store = createArtifactStore(paths.artifactsDir); + const base = (await runTrustedFabricTask(home)).outcome; + try { + const { artifacts } = observationFromFabricOutcome({ + ...base, + verifier: { + ...base.verifier, + reason: `failed ${CREDENTIAL_CANARY}`, + }, + }, { configDir: home, artifactStore: store }); + const summaryRef = artifacts.find((row) => row.artifactClass === "verifier_summary"); + expect(summaryRef).toBeDefined(); + const text = new TextDecoder().decode(store.get(summaryRef!.digest)); + expect(text.includes(CREDENTIAL_CANARY)).toBe(false); + const trustedRun = await runTrustedFabricTask(home); + persistFabricRunResult({ + ...trustedRun, + outcome: { + ...trustedRun.outcome, + verifier: { + ...trustedRun.outcome.verifier, + reason: `failed ${CREDENTIAL_CANARY}`, + }, + }, + }, { configDir: home }); + const ledger = readFileSync(join(home, "lab", "compatibility.jsonl"), "utf8"); + expect(ledger.includes(CREDENTIAL_CANARY)).toBe(false); + } finally { + store.close(); + } + }); + + test("expired ledger lock is recovered before append", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); + const lockPath = join(home, "lab", "compatibility.jsonl.lock"); + writeFileSync(lockPath, JSON.stringify({ + pid: 4_000_000_000, + createdAt: Date.now() - 120_000, + token: "stale-lock-token", + }), { mode: 0o600 }); + const result = await runTrustedFabricTask(home); + persistFabricRunResult(result, { configDir: home }); + expect(existsSync(join(home, "lab", "compatibility.jsonl"))).toBe(true); + }); + + test("symlinked lab scratch dir is rejected", () => { + const home = tempHome(); + const outside = join(home, "outside-scratch"); + mkdirSync(outside, { recursive: true, mode: 0o700 }); + mkdirSync(join(home, "lab"), { recursive: true, mode: 0o700 }); + const scratchLink = join(home, "lab", "scratch"); + if (!linkDirectory(outside, scratchLink)) return; + expect(() => createSyntheticScratch(home)).toThrow(); + expect(readdirSync(outside).some((name) => name.startsWith("fabric-"))).toBe(false); + }); + + test("failed scratch construction removes partial fabric directories", () => { + const home = tempHome(); + const originalWriteSync = nodeFs.writeSync; + const writeSpy = spyOn(nodeFs, "writeSync").mockImplementation((...args: Parameters) => { + const buffer = args[1]; + if (Buffer.isBuffer(buffer) && buffer.toString("utf8") === SYNTHETIC_BEFORE_UTF8) { + throw new Error("simulated fixture write failure"); + } + return originalWriteSync(...args); + }); + try { + expect(() => createSyntheticScratch(home)).toThrow("simulated fixture write failure"); + const scratchBase = join(home, "lab", "scratch"); + expect(readdirSync(scratchBase).some((name) => name.startsWith("fabric-"))).toBe(false); + } finally { + writeSpy.mockRestore(); + } + }); + + test("symlinked lab root rejects directory escape via ensureLabDirs", () => { + const home = tempHome(); + const outside = join(home, "outside-lab"); + mkdirSync(outside, { recursive: true, mode: 0o700 }); + const labLink = join(home, "lab"); + if (!linkDirectory(outside, labLink)) return; + expect(() => ensureLabDirs(home)).toThrow(RESTRICTED_LINK_ERROR); + expect(existsSync(join(outside, "scratch"))).toBe(false); + expect(existsSync(join(outside, "artifacts"))).toBe(false); + }); + + test("symlink under lab boundary is rejected by ensureRestrictedDir", () => { + const home = tempHome(); + const labDir = join(home, "lab"); + const outside = join(home, "outside-scratch"); + mkdirSync(labDir, { recursive: true, mode: 0o700 }); + mkdirSync(outside, { recursive: true, mode: 0o700 }); + const scratchLink = join(labDir, "scratch"); + if (!linkDirectory(outside, scratchLink)) return; + expect(() => ensureRestrictedDir(join(labDir, "scratch", "nested"), labDir)).toThrow(RESTRICTED_LINK_ERROR); + }); + + test("duplicate outcome delivery is idempotent; distinct attempts remain distinct", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const firstRun = await runTrustedFabricTask(home, {}, () => 1_000); + const first = persistFabricRunResult(firstRun, { configDir: home, recordedAt: 1_000, attempt: 1 }); + const second = persistFabricRunResult(firstRun, { configDir: home, recordedAt: 1_000, attempt: 1 }); + expect(second.event.eventId).toBe(first.event.eventId); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + const ids = replay.events.filter((row) => row.eventKind === "observation").map((row) => row.eventId); + expect(ids).toEqual([first.event.eventId]); + expect(replay.corruptions.filter((row) => row.kind === "duplicate_event")).toEqual([]); + + const secondAttempt = await runTrustedFabricTask(home, {}, () => 2_000); + const third = persistFabricRunResult(secondAttempt, { configDir: home, recordedAt: 2_000, attempt: 2 }); + expect(third.event.eventId).not.toBe(first.event.eventId); + }); + + test("structured result converts to fabric Lab observation", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = (await runTrustedFabricTask(home)).outcome; + const { event } = observationFromFabricOutcome(outcome, { configDir: home }); + expect(event.evidenceLayer).toBe("task_effectiveness"); + expect(event.executionMode).toBe("fabric"); + expect(event.subject.subjectKind).toBe("task"); + expect(event.suiteId).toBe("fabric-core"); + const serialized = JSON.stringify(event); + expect(serialized.includes(CREDENTIAL_CANARY)).toBe(false); + expect(serialized.includes("OPENAI_API_KEY")).toBe(false); + }); + + test("task layer rejects RouteSubjectV1 mismatches", () => { + expect(() => observationFromFabricOutcome({ + schemaVersion: 1, + taskClassId: "x", + taskClassVersion: "1", + routeSubject: routeSubject(), + taskSubject: { subjectSchemaVersion: 1, subjectKind: "route" }, + subjectId: "a".repeat(64), + taskFixtureDigest: "b".repeat(64), + verifierManifestDigest: "c".repeat(64), + fabricCompatibilityVersion: "v", + sandboxProfileDigest: "d".repeat(64), + startedAt: 1, + completedAt: 2, + limits: FABRIC_LIMITS, + usage: { + inputBytes: 0, + outputBytes: 0, + patchOperations: 0, + filesTouched: 0, + artifactBytes: 0, + elapsedMs: 1, + inactiveMs: 0, + }, + outcome: "pass", + verifier: { + verifierId: "exact-tree-diff-v1", + manifestDigest: "c".repeat(64), + passed: true, + pathSummaries: [], + }, + artifactDigests: [], + })).toThrow(); + }); + + test("projection rebuild and invalidation work for task evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runTrustedFabricTask(home, {}, () => 5_000); + const persisted = persistFabricRunResult(result, { configDir: home, recordedAt: 5_000 }); + const rebuilt = rebuildLabProjection(home); + expect(rebuilt.events).toBeGreaterThan(0); + expect(rebuilt.corruptions).toEqual([]); + expect(rebuilt.verdicts).toBeGreaterThan(0); + const snap = readVerdictSnapshot(rebuilt.sqlitePath) as Array<{ + evidence_layer?: string; + verdict?: string; + }>; + const taskRow = snap.find((row) => row.evidence_layer === "task_effectiveness"); + expect(taskRow?.verdict).toBe("VERIFIED"); + + const inv = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "invalidation" as const, + recordedAt: 6_000, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + targetEventIds: [persisted.event.eventId], + reason: "manual_correction" as const, + }); + appendLabEvent(join(home, "lab", "compatibility.jsonl"), inv as never); + const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); + const index = buildInvalidationIndex(replay.events); + expect(index.invalidatedBy.has(persisted.event.eventId)).toBe(true); + }); + + test("artifacts are bounded and catalog exposes task evidence", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const outcome = (await runTrustedFabricTask(home)).outcome; + const { event, artifacts } = observationFromFabricOutcome(outcome, { configDir: home }); + expect(artifacts.length).toBeGreaterThan(0); + expect(artifacts.every((row) => row.byteCount <= 256 * 1024)).toBe(true); + expect(event.artifactRefs.every((row) => /^[0-9a-f]{64}$/.test(row.digest))).toBe(true); + + const catalog = queryLabCatalog({ layer: "task_effectiveness" }); + expect(catalog.some((row) => row.suiteId === "fabric-core" && row.scenarioId === "fabric-core.task.synthetic-patch")).toBe(true); + }); + + test("CL-06 routing still rejects task_effectiveness required suites", () => { + const issues = routingProfileIssues("p", { + candidates: [{ provider: "a", model: "m" }], + compatibility: { + requiredSuites: [{ + suiteId: "fabric-core", + evidenceLayer: "task_effectiveness" as "live_route_compatibility", + }], + }, + }, { + providers: { + a: { type: "openai", baseUrl: "http://127.0.0.1:9", models: { m: {} } }, + }, + } as never); + expect(issues.some((row) => row.message.includes("protocol_conformance or live_route_compatibility"))).toBe(true); + }); + + test("ledger lines omit prompts and credentials", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const result = await runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), + destination: await fabricDestination(home), + patchExecutor: fabricCorrectPatchExecutor(), + configDir: home, + sourceRefs: ["routeDecision:abcd1234"], + }); + persistFabricRunResult(result, { configDir: home }); + const text = readFileSync(join(home, "lab", "compatibility.jsonl"), "utf8"); + expect(text.includes("system prompt")).toBe(false); + expect(text.includes(CREDENTIAL_CANARY)).toBe(false); + }); +}); diff --git a/tests/lab-live-probe.test.ts b/tests/lab-live-probe.test.ts index 95d40e7f0c..d869f227ea 100644 --- a/tests/lab-live-probe.test.ts +++ b/tests/lab-live-probe.test.ts @@ -12,6 +12,7 @@ import { import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; import { replayLabLedger } from "../src/lab/ledger/store"; import { clearMcpStub } from "../src/lab/live/mcp-loopback"; +import { TransportError } from "../src/lab/live/transport"; import { expandLiveSuiteManifest } from "../src/lab/live/suite-manifest"; import type { LabBehaviorValues, LabRouteContext, LiveScenarioRunResult } from "../src/lab/live/types"; import type { ProtocolSubjectV1, RouteSubjectV1 } from "../src/lab/events/types"; @@ -182,4 +183,22 @@ describe("CL-03 live probe harness", () => { const result = await runLiveScenario(scenario, mockRoute({ requiredClaims: ["reasoning"] }), { configDir: home, resolve: async () => [{ address: "93.184.216.34", family: 4 }], routeExecutor: trustedObservation(reasoningObservation()) }); expect(result.passed).toBe(true); const { event } = observationFromLiveResult(result, scenario, authority, { configDir: home }); expect(JSON.stringify(event)).not.toContain("PLAN"); }); + + test("inactivity_timeout failures attribute environment on observation", async () => { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases[0]!; + const result = await runLiveScenario(caseRecord, mockRoute(), { + configDir: home, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + routeExecutor: createHostIssuedLabRouteExecutor(async () => { + throw new TransportError("inactivity_timeout", "inactivity timeout exceeded"); + }), + }); + expect(result.classification).toBe("inactivity_timeout"); + const { event } = observationFromLiveResult(result, caseRecord, authority, { configDir: home }); + expect(event.failure?.class).toBe("inactivity_timeout"); + expect(event.failure?.attribution).toBe("environment"); + }); }); diff --git a/tests/lab-paths-security.test.ts b/tests/lab-paths-security.test.ts new file mode 100644 index 0000000000..d93580b585 --- /dev/null +++ b/tests/lab-paths-security.test.ts @@ -0,0 +1,51 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ensureRestrictedDir } from "../src/lab/paths"; + +const ROOTS: string[] = []; + +function tempRoot(): string { + const root = mkdtempSync(join(tmpdir(), "ocx-lab-paths-")); + ROOTS.push(root); + return root; +} + +afterEach(() => { + for (const root of ROOTS.splice(0)) { + try { rmSync(root, { recursive: true, force: true }); } catch { /* ignore */ } + } +}); + +test("restricted Lab paths allow a symlinked infrastructure ancestor above the Lab boundary", () => { + const root = tempRoot(); + const actualInfrastructure = join(root, "actual-infrastructure"); + const infrastructureAlias = join(root, "infrastructure-alias"); + mkdirSync(actualInfrastructure, { recursive: true }); + + try { + symlinkSync(actualInfrastructure, infrastructureAlias, process.platform === "win32" ? "junction" : "dir"); + } catch { + return; + } + + const lab = join(infrastructureAlias, "opencodex", "lab"); + expect(() => ensureRestrictedDir(lab, lab)).not.toThrow(); + expect(realpathSync.native(lab)).toBe(realpathSync.native(join(actualInfrastructure, "opencodex", "lab"))); +}); + +test("restricted Lab paths still reject a symlink at the Lab boundary itself", () => { + const root = tempRoot(); + const actualLab = join(root, "actual-lab"); + const substitutedLab = join(root, "lab"); + mkdirSync(actualLab, { recursive: true }); + + try { + symlinkSync(actualLab, substitutedLab, process.platform === "win32" ? "junction" : "dir"); + } catch { + return; + } + + expect(() => ensureRestrictedDir(substitutedLab, substitutedLab)).toThrow(/symbolic link|link or reparse-point substitution/); +});