From f4f69aa8d47da68322d64b694dfd7873dca6202b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:42:20 +0200 Subject: [PATCH 01/67] docs(lab): freeze CL-08 automation contract Record bounded orchestration scope, default-off policy, and implementation traceability for feat/cl-08-lab-automation. --- .../008_cl08_automation.md | 1019 +++++++++++++++++ 1 file changed, 1019 insertions(+) create mode 100644 devlog/_plan/260807_compatibility_lab/008_cl08_automation.md diff --git a/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md b/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md new file mode 100644 index 0000000000..971b941be9 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md @@ -0,0 +1,1019 @@ +# CL-08 — Compatibility Lab Automation / Evidence Refresh + +## Programme position + +**Repository:** `lidge-jun/opencodex` +**Integration target:** `dev` +**Branch:** `feat/cl-08-lab-automation` +**Starting SHA:** `da8ebd3135553c1d4dd85c1f258e998a5de14f28` +**CL-07 merge prerequisite:** `02e62fc8c7354c544ef71f8bb3db5ebba42cb600` + +CL-07 is accepted and closed. + +CL-08 adds bounded, explicit orchestration around the existing Compatibility Lab producers. It does **not** redesign evidence semantics, production routing, or Agent Fabric. + +--- + +# 1. Goal + +CL-08 answers: + +> Which Compatibility Lab evidence is due for collection or refresh, when may it run, and how can OpenCodex execute that work safely without duplicate probes, uncontrolled retries, unexpected provider traffic, or production-request coupling? + +The architecture is: + +```text + CL-08 Orchestrator + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ + CL-01 protocol CL-03 live CL-07 Fabric + runner probe producer + │ │ │ + └──────────────┴──────────────┘ + │ + ▼ + CL-02 evidence + ledger + │ + ▼ + projection +``` + +The orchestrator schedules work. + +It does **not** create compatibility evidence directly. + +--- + +# 2. Core invariants + +## 2.1 No production-path execution + +Compatibility Lab automation must never be initiated by: + +- production request routing; +- Router Intelligence evaluation; +- Routing Profile evaluation; +- provider discovery; +- request-history reads; +- compatibility projection reads; +- dashboard rendering; +- model selection. + +Those components may read existing Lab projections only. + +They must never synchronously or asynchronously trigger evidence collection. + +--- + +## 2.2 Default off + +Automatic Lab execution is opt-in. + +The default configuration must produce: + +```text +0 automatic provider requests +0 automatic Fabric executions +``` + +Manual Lab execution remains independent of automation. + +--- + +## 2.3 Existing evidence authority remains unchanged + +CL-08 must use the existing: + +- scenarios; +- suites; +- subject identities; +- CL-01 protocol runner; +- CL-03 live-route runner; +- CL-07 task-effectiveness producer; +- CL-02 canonical JSONL ledger; +- SQLite projection; +- invalidation/purge semantics. + +CL-08 must not introduce: + +- a second evidence ledger; +- mutable compatibility flags; +- an alternate subject identity; +- an alternate verdict system. + +--- + +# 3. V1 automation scope + +## 3.1 `protocol_conformance` + +Automatic execution allowed. + +Protocol scenarios are deterministic and synthetic. + +No external provider request is permitted. + +--- + +## 3.2 `live_route_compatibility` + +Automatic execution allowed only when: + +- automation is explicitly enabled; +- the exact route is eligible; +- the suite/scenario is approved for automatic execution; +- evidence is missing or approaching expiry; +- route cooldown allows execution; +- global/provider/request budgets allow execution. + +Every request must still pass the existing CL-03 destination, credential, SSRF, environment, and scenario authorization boundaries. + +--- + +## 3.3 `task_effectiveness` + +CL-08 V1 may understand and plan task-effectiveness work, but **automatic unattended task execution remains disabled by default and should not be enabled merely because general Lab automation is enabled**. + +Required policy: + +```text +task_effectiveness background execution = explicit separate opt-in +``` + +If the existing CL-07 execution boundary is not strong enough to safely guarantee unattended operation, CL-08 must leave task-effectiveness automation unavailable and expose that state honestly. + +Manual CL-07 execution remains valid. + +Do not weaken CL-07 security boundaries to make background execution easier. + +--- + +# 4. Automation policy + +Create a versioned closed policy model, approximately: + +```ts +type LabAutomationLayer = + | "protocol_conformance" + | "live_route_compatibility" + | "task_effectiveness"; + +interface LabAutomationPolicyV1 { + schemaVersion: 1; + + enabled: boolean; + + layers: { + protocolConformance: boolean; + liveRouteCompatibility: boolean; + taskEffectiveness: boolean; + }; + + refreshBeforeStaleMs: number; + + maxConcurrentRuns: number; + maxConcurrentLiveRuns: number; + maxConcurrentRunsPerRoute: number; + + maxRunsPerHour: number; + maxLiveRequestsPerHour: number; + + failureCooldownMs: number; + blockedCooldownMs: number; + + taskEffectivenessBackgroundEnabled: boolean; +} +``` + +Exact shape may change after repository audit. + +Requirements: + +- closed schema; +- strict validation; +- deterministic normalization; +- versioned; +- hard upper bounds; +- callers cannot increase CL-00/CL-03/CL-07 execution ceilings; +- no secrets or destinations in persisted policy; +- no arbitrary cron syntax in V1. + +Automation policy is operational Lab configuration. + +It must **not** be embedded in Routing Profiles. + +--- + +# 5. Planning model + +Implement a deterministic planner. + +Input: + +- automation policy; +- current time; +- scenario/suite catalogue; +- current Lab projection/evidence; +- eligible exact subjects/routes; +- current queue/running state; +- cooldown/budget state. + +Output: + +```ts +PlannedLabRunV1[] +``` + +A planned run should identify only safe canonical information such as: + +```text +runKey +evidenceLayer +suiteId +suiteVersion +scenarioId +scenarioVersion +subjectId +reason +priority +eligibleAt +``` + +Never persist credentials, raw URLs, prompts, repository paths, request bodies, model responses, or account identity. + +--- + +# 6. Run identity and deduplication + +Define a deterministic `runKey`. + +It must bind at least: + +```text +evidence layer +subject ID +suite ID/version/manifest digest +scenario ID/version/manifest digest +relevant execution contract identity +``` + +Properties: + +- same effective work → same `runKey`; +- materially different scenario/subject/contracts → different `runKey`; +- only one queued/running run may exist for one key; +- duplicate scheduler ticks must not duplicate execution; +- process restart must not cause duplicate accepted evidence. + +Use existing event idempotency after execution as defense in depth, not as scheduler deduplication. + +--- + +# 7. Freshness planning + +Planner states should distinguish: + +```text +fresh +refresh_due +missing +cooldown +budget_blocked +already_queued +already_running +automation_disabled +layer_disabled +inapplicable +``` + +A refresh should be scheduled before evidence becomes stale according to: + +```text +effectiveFreshnessDeadline - refreshBeforeStaleMs +``` + +Do not invent new verdict semantics. + +Freshness planning controls execution timing only. + +--- + +# 8. Queue / lifecycle + +Use a bounded state machine: + +```text +queued + ↓ +running + ├── completed + ├── blocked + ├── failed + └── cancelled +``` + +Optional explicit: + +```text +abandoned +``` + +if useful for crash recovery. + +Each state transition must be deterministic and validated. + +Queue state is orchestration state, **not compatibility evidence**. + +--- + +# 9. Queue persistence + +Audit existing application scheduling/background-state facilities first. + +Prefer reuse. + +If durable orchestration state is required, persist only the minimum bounded scheduler metadata required for: + +- deduplication; +- restart recovery; +- cooldown; +- budgets; +- operator visibility. + +Do not create another canonical compatibility evidence database. + +Queue persistence must be disposable or reconcilable against canonical Lab evidence where possible. + +Never persist: + +- API credentials; +- prompts; +- provider request/response bodies; +- task repository content; +- task patches; +- arbitrary stdout/stderr; +- arbitrary filesystem paths. + +--- + +# 10. Scheduler + +Implement a bounded scheduler tick. + +Conceptually: + +```text +load policy + ↓ +read current Lab state + ↓ +build deterministic plan + ↓ +apply queue dedup/cooldowns/budgets + ↓ +enqueue eligible work + ↓ +dispatch within concurrency limits +``` + +Requirements: + +- one scheduler tick cannot enqueue unlimited work; +- scheduler cannot recursively trigger itself; +- one failed run cannot create an infinite retry loop; +- scheduling decisions must be inspectable; +- disabled automation must exit without execution; +- shutdown must stop new dispatch. + +Do not implement an arbitrary cron engine. + +--- + +# 11. Concurrency + +Enforce: + +- global concurrent Lab runs; +- global concurrent live-route probes; +- per-route concurrent runs; +- task-effectiveness concurrency if ever enabled. + +Initial conservative defaults are preferred. + +Example: + +```text +global Lab 2 +live probes 1 +same route 1 +task effectiveness 1 +``` + +Exact defaults require repository review. + +--- + +# 12. Budgets + +Implement bounded rolling-window budgets. + +At minimum: + +```text +max runs / hour +max live requests / hour +``` + +Prefer monotonic deterministic accounting. + +Provider-specific/request-specific limits may be added if justified. + +Budget exhaustion causes scheduling deferral/blocking. + +It must never produce a route compatibility failure. + +--- + +# 13. Cooldowns + +Differentiate at minimum: + +### Attributable compatibility result + +Ordinary freshness rules apply. + +### Authentication blocked + +Do not repeatedly retry missing/invalid credentials. + +Use blocked cooldown. + +### Provider/environment transient failure + +Use bounded retry/cooldown. + +### Harness defect + +Do not hammer the route. + +Prefer strong cooldown / operator visibility. + +### Budget exhausted + +Wait for budget eligibility. + +No exponential retry loop without a hard cap. + +--- + +# 14. Retry policy + +V1 should use minimal automatic retries. + +Recommended: + +```text +one execution attempt per scheduled run +``` + +A future scheduler tick may reconsider it after cooldown. + +Avoid internal retry loops unless required by an existing runner contract. + +A compatibility failure must not immediately generate repeated provider traffic. + +--- + +# 15. Crash recovery + +On process startup: + +- inspect durable scheduler state; +- identify runs left `running`; +- do not assume they completed; +- reconcile against canonical evidence where possible; +- otherwise mark them abandoned/recoverable; +- apply a bounded recovery cooldown; +- never immediately duplicate an uncertain live request. + +Crash recovery must be deterministic and testable. + +--- + +# 16. Cancellation + +Support cancellation of queued work. + +For running work: + +- use the runner's existing cancellation/termination mechanism; +- do not introduce unsafe process killing outside owned execution boundaries; +- cancellation is orchestration state; +- it must not create false route incompatibility evidence. + +Application shutdown should: + +1. stop new scheduling; +2. stop new dispatch; +3. request bounded cancellation of owned runners; +4. flush safe orchestration state. + +--- + +# 17. Execution dispatch + +Use a closed dispatcher: + +```ts +switch (plannedRun.evidenceLayer) { + case "protocol_conformance": + // invoke existing CL-01 runner + break; + + case "live_route_compatibility": + // invoke existing CL-03 route probe + break; + + case "task_effectiveness": + // invoke CL-07 only if explicitly authorized + break; +} +``` + +No: + +- dynamic modules; +- callbacks loaded from config; +- arbitrary executables; +- arbitrary shell commands; +- arbitrary scenario-provided functions. + +--- + +# 18. Task-effectiveness unattended gate + +Before enabling CL-07 automatic execution, explicitly review: + +- host-issued executor authority; +- child-process isolation; +- filesystem access; +- network access; +- shell/process capability; +- secret exposure; +- cancellation; +- post-timeout cleanup; +- operator consent. + +If required unattended guarantees cannot be enforced: + +```text +taskEffectivenessBackgroundEnabled = false +``` + +must remain mandatory. + +Do not falsely claim an OS sandbox exists. + +--- + +# 19. Management API + +Add bounded management endpoints, approximately: + +```text +GET /api/lab/automation +PUT /api/lab/automation +GET /api/lab/automation/runs +POST /api/lab/automation/run +POST /api/lab/automation/runs/:id/cancel +``` + +Exact routing should follow existing management API conventions. + +Requirements: + +- explicit mutation; +- validation; +- bounded pagination; +- no secret material; +- manual run clearly distinguished from scheduled run; +- read endpoints never trigger execution. + +--- + +# 20. CLI + +Expose equivalent operator functionality through the existing Lab CLI structure. + +Possible UX: + +```text +ocx lab automation status +ocx lab automation enable +ocx lab automation disable +ocx lab automation runs +ocx lab run ... +``` + +Do not invent CLI names before auditing current Lab command structure. + +--- + +# 21. UI + +UI is optional for CL-08 core unless the existing Compatibility Lab management surface clearly requires configuration exposure. + +If added: + +- automation toggle; +- enabled layers; +- next scheduled refresh; +- current queue; +- recent runs; +- cooldown/budget reason; +- manual run action. + +No huge scheduler dashboard. + +No auto-refresh behavior that itself starts runs. + +--- + +# 22. Security requirements + +CL-00 security/privacy rules remain authoritative. + +The scheduler must never receive or persist: + +- raw provider credentials; +- auth headers; +- user prompts; +- conversation content; +- user repositories; +- task worktrees; +- MCP configuration; +- arbitrary filesystem content; +- arbitrary provider responses; +- hidden reasoning. + +CL-03 remains responsible for live-route network/credential enforcement. + +CL-07 remains responsible for task producer execution containment. + +CL-08 must not weaken either boundary. + +--- + +# 23. Observability + +Every planning decision should be explainable without sensitive data. + +Useful reason codes: + +```text +automation_disabled +layer_disabled +fresh +refresh_due +missing +already_queued +already_running +cooldown +budget_exhausted +route_ineligible +scenario_inapplicable +task_background_disabled +``` + +Expose bounded counters such as: + +```text +queued +running +completed_last_hour +blocked_last_hour +remaining_run_budget +remaining_live_request_budget +``` + +Do not create a numerical compatibility score. + +--- + +# 24. Tests + +Required adversarial and deterministic coverage: + +## Planner + +- missing evidence schedules once; +- stale evidence schedules once; +- fresh evidence schedules nothing; +- refresh window works correctly; +- inapplicable scenario schedules nothing; +- changed subject creates new run key. + +## Dedup + +- two scheduler ticks cannot enqueue duplicate work; +- concurrent scheduler ticks cannot execute same key twice; +- restart cannot duplicate a known running/completed run. + +## Disabled state + +- default configuration schedules nothing; +- disabled automation executes zero runners; +- disabling during queued state prevents further dispatch. + +## Production boundary + +Prove that: + +- production request handling never invokes scheduler; +- Routing Profile evaluation never invokes scheduler; +- Router Intelligence never invokes scheduler; +- provider discovery never invokes scheduler; +- Lab query/read API never invokes scheduler; +- dashboard rendering never invokes scheduler. + +## Budgets + +- hourly run budget enforced; +- live request budget enforced; +- budget cannot be raised above hard maximum; +- budget exhaustion does not create compatibility failure. + +## Cooldown + +- auth blocked does not hammer route; +- route/environment failure applies cooldown; +- harness failure applies cooldown; +- successful run clears/replaces appropriate scheduling state. + +## Concurrency + +- global max respected; +- live max respected; +- per-route max respected; +- queue continues after completion. + +## Recovery + +- abandoned running entry recovered safely; +- uncertain live run not immediately duplicated; +- corrupt scheduler state fails closed. + +## Cancellation + +- queued cancellation; +- running cancellation; +- process shutdown; +- cancellation does not become route failure. + +## Privacy + +Canary secrets must never appear in: + +- queue; +- scheduler persistence; +- API; +- CLI; +- logs; +- errors; +- tests/artifacts. + +## Task-effectiveness + +- task background execution remains disabled unless separately enabled; +- general automation enablement does not enable Fabric background execution; +- untrusted Fabric executor cannot be scheduled; +- manual CL-07 still works. + +--- + +# 25. Validation + +At minimum: + +```text +bun x tsc --noEmit +bun run privacy:scan +bun test tests/lab-automation.test.ts +bun test relevant existing Lab suites +bun test routing/profile regression suites +bun test repo hygiene +``` + +Then full CI. + +Cross-platform behavior required on: + +```text +Linux +macOS +Windows +``` + +No platform may silently weaken scheduler locking/dedup/security semantics. + +--- + +# 26. Documentation + +Create: + +```text +devlog/_plan/260807_compatibility_lab/008_cl08_automation.md +``` + +Update: + +```text +001_pr_stack_status.md +000_master_plan.md +``` + +only where required. + +Record: + +- starting SHA; +- branch; +- scope; +- frozen automation contract; +- security boundary; +- validation; +- acceptance status; +- final accepted head; +- merge commit. + +--- + +# 27. Explicit non-goals + +CL-08 must not implement: + +- CL-09 work; +- general Agent Fabric; +- arbitrary jobs; +- arbitrary cron; +- user-worktree execution; +- autonomous repository modification; +- Routing Profile mutation; +- automatic production routing changes; +- provider metadata mutation; +- universal compatibility score; +- public evidence publishing; +- community leaderboard; +- prompt capture; +- hidden reasoning capture; +- second evidence ledger. + +--- + +# 28. Delivery strategy + +Recommended implementation sequence: + +### CL-08.0 — Audit and contract + +- inspect current `dev`; +- audit existing timers/background services/config persistence/API lifecycle; +- create `008_cl08_automation.md`; +- freeze policy/run/dedup/security contracts. + +### CL-08.1 — Pure planner + +Implement: + +- policy validation; +- run key; +- freshness planner; +- reason codes. + +No execution yet. + +### CL-08.2 — Queue and recovery + +Implement: + +- state machine; +- dedup; +- concurrency; +- cooldown; +- budgets; +- crash reconciliation. + +### CL-08.3 — Existing runner dispatch + +Wire: + +1. protocol conformance; +2. live route compatibility. + +Task effectiveness remains background-disabled. + +### CL-08.4 — Management surfaces + +Add API/CLI controls and status. + +### CL-08.5 — Task-effectiveness gate + +Audit unattended CL-07 execution. + +Either: + +```text +ENABLE with explicit separate opt-in + proven boundary +``` + +or: + +```text +DEFER with documented security reason +``` + +Both outcomes are acceptable if technically justified. + +### CL-08.6 — Adversarial validation + +Run: + +- concurrency races; +- restart; +- duplicate scheduling; +- cooldown; +- budget; +- privacy; +- production-path isolation; +- cross-platform CI. + +--- + +# 29. Acceptance criteria + +CL-08 is accepted only when: + +1. automation defaults off; +2. no production request path can trigger Lab execution; +3. deterministic planner exists; +4. duplicate work cannot run concurrently; +5. retries/cooldowns are bounded; +6. concurrency is bounded; +7. provider request budgets are bounded; +8. crash recovery is deterministic; +9. cancellation works; +10. no sensitive material enters orchestration state; +11. existing evidence authority remains unchanged; +12. protocol automation works; +13. live-route automation works under explicit opt-in; +14. task background execution is either proven safe and separately enabled or explicitly deferred; +15. routing semantics remain unchanged; +16. relevant focused tests pass; +17. privacy scan passes; +18. full CI passes; +19. all valid CodeRabbit Critical/High/Medium findings are resolved; +20. independent final review reports `MERGE`. + +Do not start CL-09 until CL-08 is accepted and merged. + +--- + +# 30. Implementation traceability (feat/cl-08-lab-automation) + +Audit against repository state after CL-08 implementation on `feat/cl-08-lab-automation`. + +| Section | Status | Notes | +| --- | --- | --- | +| §1 Goal | IMPLEMENTED | Orchestrator in `src/lab/automation/` plans and dispatches to CL-01/CL-03 producers. | +| §2 Core invariants | IMPLEMENTED | Default-off policy; read paths do not call scheduler (tests + static checks). | +| §3 V1 scope protocol | IMPLEMENTED | Protocol auto-planning when layer enabled. | +| §3 V1 scope live | IMPLEMENTED | Live planning + dispatch via host `TrustedLabRouteExecutor`; `buildAutomationLiveRouteContext` + production factory in `src/lib/lab-live-route-production.ts`. | +| §3 V1 scope task | DEFERRED | `taskEffectivenessBackgroundEnabled` remains false; dispatch rejects background task runs. | +| §4–§7 Policy / run key / planner | IMPLEMENTED | `policy.ts`, `run-key.ts`, `planner.ts`, `persistence.ts`. | +| §8–§14 Queue, budgets, cooldown, recovery | IMPLEMENTED | `queue.ts`, `budgets.ts`, `cooldown.ts`, `recovery.ts`, `orchestrator.ts`. | +| §15 Crash recovery | IMPLEMENTED | `recovery.ts` + startup reconcile in `startLabAutomationScheduler`. | +| §16 Cancellation | IMPLEMENTED | Queued cancel; running cancel via `AbortSignal` / `cancelSignal` on live executor. | +| §17 Dispatch | IMPLEMENTED | `dispatch.ts` closed to CL-01/CL-03; live without executor → `route_ineligible` blocked, not throw. | +| §18 Task gate | DEFERRED | Unattended CL-07 guarantees not proven; separate opt-in enforced; manual CL-07 unchanged. | +| §19 Management API | IMPLEMENTED | `lab-automation-routes.ts`. | +| §20 CLI | IMPLEMENTED | `ocx lab automation …`, `ocx lab run …`. | +| §21 UI | OUT OF SCOPE | Optional per plan. | +| §22–§23 Security / observability | IMPLEMENTED | Policy/routes hold refs only; reason codes + counters in status. | +| §27 Non-goals | VALIDATED | CL-09 not started. | +| §24 Tests | IMPLEMENTED | `tests/lab-automation.test.ts` (trusted dispatch, authority forging, budgets, shutdown, auth block) + existing Lab suites. | + +**Production trusted live dispatch:** `src/server/index.ts` wires `createProductionLabRouteExecutor({ loadConfig, configDir })` into `setLabAutomationDispatchDeps` at process startup. CL-08 consumes the host-issued executor only; it does not mint credentials, construct provider transport, or bypass CL-03 destination/SSRF/credential binding. Missing executor → `route_ineligible` with no provider traffic. Protocol automation remains independent of live authority. + +| Area | Status | Notes | +| --- | --- | --- | +| Protocol automation | IMPLEMENTED | CL-01 harness dispatch when layer enabled. | +| Live automation planning | IMPLEMENTED | Planner + routes + freshness + dedup. | +| Production trusted live dispatch | IMPLEMENTED | Server startup wires production CL-03 executor. | +| Live-route authority/security reuse | VALIDATED | Credential lease, pinned transport, receipt sealing, persistence gate on `trusted_route`. | +| Task-effectiveness background | DEFERRED | Separate opt-in remains disabled; unattended CL-07 not proven safe. | +| UI | OUT OF SCOPE | Per plan §21. | + +**Known limitation:** Automation trusted-route contract advertises `live_transport` only (`CL08_TRUSTED_LIVE_HARNESS_FEATURES`). Scenarios requiring additional harness features (inert tools, MCP stubs, etc.) are not planned or dispatched until a broader trusted contract is explicitly defined. From bb2c1c047ddac18675f710802abebd8de902d3a9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:42:34 +0200 Subject: [PATCH 02/67] feat(lab): add bounded automation orchestration Add default-off CL-08 policy, planner, queue, budgets, dispatch seam, scheduler lifecycle, and persistence without provider authority. --- src/lab/automation/budgets.ts | 36 +++ src/lab/automation/constants.ts | 38 ++++ src/lab/automation/cooldown.ts | 41 ++++ src/lab/automation/dispatch.ts | 103 +++++++++ src/lab/automation/index.ts | 13 ++ src/lab/automation/orchestrator.ts | 209 +++++++++++++++++ src/lab/automation/persistence.ts | 217 ++++++++++++++++++ src/lab/automation/planner.ts | 339 ++++++++++++++++++++++++++++ src/lab/automation/policy.ts | 98 ++++++++ src/lab/automation/queue.ts | 140 ++++++++++++ src/lab/automation/recovery.ts | 23 ++ src/lab/automation/route-context.ts | 21 ++ src/lab/automation/run-key.ts | 44 ++++ src/lab/automation/runs-query.ts | 32 +++ src/lab/automation/types.ts | 158 +++++++++++++ src/lab/index.ts | 1 + src/lab/paths.ts | 12 + 17 files changed, 1525 insertions(+) create mode 100644 src/lab/automation/budgets.ts create mode 100644 src/lab/automation/constants.ts create mode 100644 src/lab/automation/cooldown.ts create mode 100644 src/lab/automation/dispatch.ts create mode 100644 src/lab/automation/index.ts create mode 100644 src/lab/automation/orchestrator.ts create mode 100644 src/lab/automation/persistence.ts create mode 100644 src/lab/automation/planner.ts create mode 100644 src/lab/automation/policy.ts create mode 100644 src/lab/automation/queue.ts create mode 100644 src/lab/automation/recovery.ts create mode 100644 src/lab/automation/route-context.ts create mode 100644 src/lab/automation/run-key.ts create mode 100644 src/lab/automation/runs-query.ts create mode 100644 src/lab/automation/types.ts diff --git a/src/lab/automation/budgets.ts b/src/lab/automation/budgets.ts new file mode 100644 index 0000000000..545ab3e149 --- /dev/null +++ b/src/lab/automation/budgets.ts @@ -0,0 +1,36 @@ +import { LAB_AUTOMATION_HARD_MAX } from "./constants"; +import type { LabAutomationPolicyV1, LabAutomationStateV1 } from "./types"; + +export function rollBudgetWindow(state: LabAutomationStateV1, now: number): LabAutomationStateV1 { + if (now - state.budgetWindowStartedAt < LAB_AUTOMATION_HARD_MAX.budgetWindowMs) return state; + return { + ...state, + budgetWindowStartedAt: now, + runsThisHour: 0, + liveRequestsThisHour: 0, + }; +} + +export function runBudgetRemaining(policy: LabAutomationPolicyV1, state: LabAutomationStateV1): number { + return Math.max(0, policy.maxRunsPerHour - state.runsThisHour); +} + +export function liveRequestBudgetRemaining(policy: LabAutomationPolicyV1, state: LabAutomationStateV1): number { + return Math.max(0, policy.maxLiveRequestsPerHour - state.liveRequestsThisHour); +} + +export function isRunBudgetExhausted(policy: LabAutomationPolicyV1, state: LabAutomationStateV1): boolean { + return runBudgetRemaining(policy, state) <= 0; +} + +export function isLiveRequestBudgetExhausted(policy: LabAutomationPolicyV1, state: LabAutomationStateV1): boolean { + return liveRequestBudgetRemaining(policy, state) <= 0; +} + +export function recordRunBudgetUse(state: LabAutomationStateV1, liveRequest: boolean): LabAutomationStateV1 { + return { + ...state, + runsThisHour: state.runsThisHour + 1, + liveRequestsThisHour: liveRequest ? state.liveRequestsThisHour + 1 : state.liveRequestsThisHour, + }; +} diff --git a/src/lab/automation/constants.ts b/src/lab/automation/constants.ts new file mode 100644 index 0000000000..e9d1540403 --- /dev/null +++ b/src/lab/automation/constants.ts @@ -0,0 +1,38 @@ +/** CL-08 automation hard ceilings — callers cannot raise policy above these values. */ + +export const LAB_AUTOMATION_POLICY_SCHEMA_VERSION = 1 as const; + +export const LAB_AUTOMATION_HARD_MAX = Object.freeze({ + refreshBeforeStaleMs: 30 * 24 * 60 * 60 * 1000, + maxConcurrentRuns: 4, + maxConcurrentLiveRuns: 2, + maxConcurrentRunsPerRoute: 2, + maxRunsPerHour: 60, + maxLiveRequestsPerHour: 30, + failureCooldownMs: 24 * 60 * 60 * 1000, + blockedCooldownMs: 7 * 24 * 60 * 60 * 1000, + maxQueuedRuns: 256, + maxPersistedRuns: 512, + maxAutomationRoutes: 64, + budgetWindowMs: 60 * 60 * 1000, + schedulerTickMs: 60 * 1000, +}); + +export const LAB_AUTOMATION_DEFAULT_POLICY = Object.freeze({ + schemaVersion: LAB_AUTOMATION_POLICY_SCHEMA_VERSION, + enabled: false, + layers: Object.freeze({ + protocolConformance: false, + liveRouteCompatibility: false, + taskEffectiveness: false, + }), + refreshBeforeStaleMs: 24 * 60 * 60 * 1000, + maxConcurrentRuns: 2, + maxConcurrentLiveRuns: 1, + maxConcurrentRunsPerRoute: 1, + maxRunsPerHour: 12, + maxLiveRequestsPerHour: 6, + failureCooldownMs: 5 * 60 * 1000, + blockedCooldownMs: 60 * 60 * 1000, + taskEffectivenessBackgroundEnabled: false, +}); diff --git a/src/lab/automation/cooldown.ts b/src/lab/automation/cooldown.ts new file mode 100644 index 0000000000..78bc1745a0 --- /dev/null +++ b/src/lab/automation/cooldown.ts @@ -0,0 +1,41 @@ +import type { LabAutomationPolicyV1, LabAutomationStateV1 } from "./types"; + +export function cooldownActive(state: LabAutomationStateV1, key: string, now: number): boolean { + const until = state.cooldownUntilByKey[key]; + return typeof until === "number" && until > now; +} + +export function setCooldown( + state: LabAutomationStateV1, + key: string, + untilMs: number, +): LabAutomationStateV1 { + return { + ...state, + cooldownUntilByKey: { + ...state.cooldownUntilByKey, + [key]: untilMs, + }, + }; +} + +export function clearCooldown(state: LabAutomationStateV1, key: string): LabAutomationStateV1 { + if (!state.cooldownUntilByKey[key]) return state; + const next = { ...state.cooldownUntilByKey }; + delete next[key]; + return { ...state, cooldownUntilByKey: next }; +} + +export function cooldownForFailure( + policy: LabAutomationPolicyV1, + classification: string, + now: number, +): number { + if (classification === "authentication_blocked" || classification === "auth_blocked" || classification === "quota_blocked" || classification === "region_blocked") { + return now + policy.blockedCooldownMs; + } + if (classification === "harness_failure") { + return now + policy.blockedCooldownMs; + } + return now + policy.failureCooldownMs; +} diff --git a/src/lab/automation/dispatch.ts b/src/lab/automation/dispatch.ts new file mode 100644 index 0000000000..7773ed0157 --- /dev/null +++ b/src/lab/automation/dispatch.ts @@ -0,0 +1,103 @@ +import { loadCaseAuthority } from "../conformance/manifest"; +import { runScenario } from "../conformance/executor"; +import { persistConformanceResult } from "../observe/from-conformance"; +import { loadLiveCaseAuthority } from "../live/manifest"; +import { runLiveScenario } from "../live/executor"; +import { persistLiveResult } from "../observe/from-live"; +import { rebuildLabProjection } from "../projection/rebuild"; +import { resolvePolicyCompatibilitySubjects } from "../../routing/compatibility/subject"; +import type { OcxConfig } from "../../types"; +import type { AutomationDispatchDeps, LabAutomationRunRecordV1 } from "./types"; +import { LabAutomationError } from "./types"; +import { buildAutomationLiveRouteContext } from "./route-context"; + +export interface DispatchResult { + terminalState: "completed" | "blocked" | "failed"; + terminalCode: string; + liveRequest: boolean; +} + +/** Closed dispatcher to existing CL-01 / CL-03 / CL-07 producers. */ +export async function dispatchLabAutomationRun( + run: LabAutomationRunRecordV1, + deps: AutomationDispatchDeps, +): Promise { + const configDir = deps.configDir; + switch (run.evidenceLayer) { + case "protocol_conformance": { + if (deps.abortSignal?.aborted) { + throw new LabAutomationError("cancelled", "cancelled"); + } + const authority = loadCaseAuthority(); + const caseRecord = authority.cases.find((row) => row.id === run.scenarioId); + if (!caseRecord) { + throw new LabAutomationError("missing protocol scenario", "dispatch_failure"); + } + const result = await runScenario(caseRecord); + if (deps.abortSignal?.aborted) { + throw new LabAutomationError("cancelled", "cancelled"); + } + persistConformanceResult(result, caseRecord, authority, { configDir }); + rebuildLabProjection(configDir); + return { + terminalState: result.passed ? "completed" : "failed", + terminalCode: result.passed ? "pass" : "protocol_fail", + liveRequest: false, + }; + } + case "live_route_compatibility": { + if (!deps.routeExecutor) { + return { terminalState: "blocked", terminalCode: "route_ineligible", liveRequest: false }; + } + if (!deps.loadConfig || !run.providerName || !run.modelId) { + throw new LabAutomationError("live dispatch requires route configuration", "dispatch_failure"); + } + const config = deps.loadConfig(); + const routed = config.providers?.[run.providerName]; + if (!routed) { + return { terminalState: "blocked", terminalCode: "route_ineligible", liveRequest: false }; + } + const resolved = resolvePolicyCompatibilitySubjects( + config, + run.providerName, + run.modelId, + routed, + configDir, + ); + if (!resolved.route) { + return { terminalState: "blocked", terminalCode: "route_ineligible", liveRequest: false }; + } + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases.find((row) => row.id === run.scenarioId); + if (!caseRecord) { + throw new LabAutomationError("missing live scenario", "dispatch_failure"); + } + const routeContext = buildAutomationLiveRouteContext( + resolved.route, + routed.allowPrivateNetwork === true, + ); + const result = await runLiveScenario(caseRecord, routeContext, { + configDir, + routeExecutor: deps.routeExecutor, + cancelSignal: deps.abortSignal, + resolve: deps.resolve, + }); + if (result.executionAuthority === "trusted_route") { + persistLiveResult(result, caseRecord, authority, { configDir }); + rebuildLabProjection(configDir); + } + const blocked = result.classification === "authentication_blocked" + || result.classification === "quota_blocked" + || result.classification === "region_blocked"; + return { + terminalState: result.passed ? "completed" : blocked ? "blocked" : "failed", + terminalCode: result.secondaryCode ?? result.classification, + liveRequest: true, + }; + } + case "task_effectiveness": + throw new LabAutomationError("task_effectiveness background execution is not enabled", "task_background_disabled"); + default: + throw new LabAutomationError(`unsupported evidence layer ${run.evidenceLayer as string}`, "dispatch_failure"); + } +} diff --git a/src/lab/automation/index.ts b/src/lab/automation/index.ts new file mode 100644 index 0000000000..31062247dc --- /dev/null +++ b/src/lab/automation/index.ts @@ -0,0 +1,13 @@ +export * from "./types"; +export * from "./constants"; +export * from "./policy"; +export * from "./run-key"; +export * from "./persistence"; +export * from "./planner"; +export * from "./queue"; +export * from "./budgets"; +export * from "./cooldown"; +export * from "./dispatch"; +export * from "./recovery"; +export * from "./runs-query"; +export * from "./orchestrator"; diff --git a/src/lab/automation/orchestrator.ts b/src/lab/automation/orchestrator.ts new file mode 100644 index 0000000000..3d9752609c --- /dev/null +++ b/src/lab/automation/orchestrator.ts @@ -0,0 +1,209 @@ +import { readConfigDiagnostics } from "../../config"; +import { planLabAutomationRuns } from "./planner"; +import { + loadLabAutomationPolicy, + loadLabAutomationRoutes, + loadLabAutomationState, + saveLabAutomationState, +} from "./persistence"; +import { rollBudgetWindow, recordRunBudgetUse, runBudgetRemaining, liveRequestBudgetRemaining, isRunBudgetExhausted, isLiveRequestBudgetExhausted } from "./budgets"; +import { cooldownForFailure, setCooldown } from "./cooldown"; +import { + enqueuePlannedRuns, + selectDispatchableRuns, + transitionRun, + trimTerminalRuns, + countRunsByState, +} from "./queue"; +import { dispatchLabAutomationRun, type DispatchResult } from "./dispatch"; +import { recoverLabAutomationState } from "./recovery"; +import type { AutomationDispatchDeps, LabAutomationRunRecordV1, LabAutomationStatusV1 } from "./types"; +import { LabAutomationError } from "./types"; +import { LAB_AUTOMATION_HARD_MAX } from "./constants"; +import { cancelQueuedRun } from "./queue"; + +let schedulerTimer: ReturnType | null = null; +let tickInProgress = false; +let shutdownRequested = false; +let dispatchDeps: AutomationDispatchDeps = {}; +const inFlightControllers = new Map(); +const cancellingRunIds = new Set(); + +export function setLabAutomationDispatchDeps(deps: AutomationDispatchDeps): void { + dispatchDeps = deps; +} + +export function isLabAutomationSchedulerRunning(): boolean { + return schedulerTimer !== null; +} + +export function requestLabAutomationShutdown(): void { + shutdownRequested = true; + for (const controller of inFlightControllers.values()) { + controller.abort(new Error("cancelled")); + } +} + +export function buildLabAutomationStatus(configDir?: string): LabAutomationStatusV1 { + const policy = loadLabAutomationPolicy(configDir); + const routes = loadLabAutomationRoutes(configDir); + const state = loadLabAutomationState(configDir); + const hourAgo = Date.now() - LAB_AUTOMATION_HARD_MAX.budgetWindowMs; + const completedLastHour = state.runs.filter((row) => row.state === "completed" && (row.completedAt ?? 0) >= hourAgo).length; + const blockedLastHour = state.runs.filter((row) => row.state === "blocked" && (row.completedAt ?? 0) >= hourAgo).length; + return { + policy, + routes, + counters: { + queued: countRunsByState(state, "queued"), + running: countRunsByState(state, "running"), + completedLastHour, + blockedLastHour, + remainingRunBudget: runBudgetRemaining(policy, state), + remainingLiveRequestBudget: liveRequestBudgetRemaining(policy, state), + }, + schedulerRunning: isLabAutomationSchedulerRunning(), + }; +} + +async function runDispatchBatch(configDir?: string, now = Date.now()): Promise { + if (shutdownRequested) return; + const policy = loadLabAutomationPolicy(configDir); + if (!policy.enabled) return; + let state = rollBudgetWindow(loadLabAutomationState(configDir), now); + const routes = loadLabAutomationRoutes(configDir); + let config: import("../../types").OcxConfig | undefined; + try { + config = readConfigDiagnostics().config; + } catch { + config = undefined; + } + const planned = planLabAutomationRuns({ + policy, + routes, + state, + now, + config, + configDir, + }); + state = enqueuePlannedRuns(state, planned, "scheduled", now); + const dispatchable = selectDispatchableRuns(policy, state, now); + for (const run of dispatchable) { + if (shutdownRequested) break; + if (isRunBudgetExhausted(policy, state)) break; + if (run.evidenceLayer === "live_route_compatibility" && isLiveRequestBudgetExhausted(policy, state)) { + continue; + } + state = transitionRun(state, run.runId, "running", now); + saveLabAutomationState(state, configDir); + const controller = new AbortController(); + inFlightControllers.set(run.runId, controller); + try { + const deps: AutomationDispatchDeps = { + ...dispatchDeps, + configDir, + loadConfig: dispatchDeps.loadConfig ?? (() => readConfigDiagnostics().config), + abortSignal: controller.signal, + }; + const result: DispatchResult = await dispatchLabAutomationRun(run, deps); + if (cancellingRunIds.has(run.runId) || controller.signal.aborted) { + state = transitionRun(state, run.runId, "cancelled", Date.now(), "cancelled"); + } else { + state = recordRunBudgetUse(state, result.liveRequest); + state = transitionRun(state, run.runId, result.terminalState, Date.now(), result.terminalCode); + if (result.terminalState !== "completed") { + state = setCooldown( + state, + run.runKey, + cooldownForFailure(policy, result.terminalCode, Date.now()), + ); + } + } + } catch (error) { + if (cancellingRunIds.has(run.runId) || (error instanceof LabAutomationError && error.code === "cancelled")) { + state = transitionRun(state, run.runId, "cancelled", Date.now(), "cancelled"); + } else { + const code = error instanceof Error ? error.message : "dispatch_failure"; + state = transitionRun(state, run.runId, "failed", Date.now(), code); + state = setCooldown(state, run.runKey, cooldownForFailure(policy, "harness_failure", Date.now())); + } + } finally { + inFlightControllers.delete(run.runId); + cancellingRunIds.delete(run.runId); + } + state = trimTerminalRuns(state, Date.now()); + saveLabAutomationState(state, configDir); + } + saveLabAutomationState(trimTerminalRuns(state, now), configDir); +} + +/** Single bounded scheduler tick — plan, enqueue, dispatch within concurrency limits. */ +export async function runLabAutomationTick(configDir?: string): Promise { + if (tickInProgress) return; + tickInProgress = true; + try { + await runDispatchBatch(configDir); + } finally { + tickInProgress = false; + } +} + +export function startLabAutomationScheduler(configDir?: string): void { + if (schedulerTimer) return; + shutdownRequested = false; + const policy = loadLabAutomationPolicy(configDir); + let state = loadLabAutomationState(configDir); + state = recoverLabAutomationState(policy, state, Date.now()); + saveLabAutomationState(state, configDir); + schedulerTimer = setInterval(() => { + void runLabAutomationTick(configDir); + }, LAB_AUTOMATION_HARD_MAX.schedulerTickMs); + schedulerTimer.unref?.(); +} + +export function stopLabAutomationScheduler(): void { + shutdownRequested = true; + if (schedulerTimer) { + clearInterval(schedulerTimer); + schedulerTimer = null; + } +} + +/** Test-only reset of scheduler globals between isolated automation tests. */ +export function resetLabAutomationSchedulerStateForTests(): void { + shutdownRequested = false; + tickInProgress = false; + inFlightControllers.clear(); + cancellingRunIds.clear(); +} + +export async function enqueueManualLabRun( + planned: import("./types").PlannedLabRunV1, + configDir?: string, +): Promise { + const now = Date.now(); + let state = loadLabAutomationState(configDir); + state = enqueuePlannedRuns(state, [planned], "manual", now); + saveLabAutomationState(state, configDir); + const created = state.runs.find((row) => row.runKey === planned.runKey && row.trigger === "manual"); + if (!created) return null; + await runLabAutomationTick(configDir); + return loadLabAutomationState(configDir).runs.find((row) => row.runId === created.runId) ?? created; +} + +export function cancelLabAutomationRun(runId: string, configDir?: string): boolean { + const state = loadLabAutomationState(configDir); + const run = state.runs.find((row) => row.runId === runId); + if (!run) return false; + if (run.state === "queued") { + saveLabAutomationState(cancelQueuedRun(state, runId, Date.now()), configDir); + return true; + } + if (run.state === "running") { + cancellingRunIds.add(runId); + const controller = inFlightControllers.get(runId); + if (controller) controller.abort(new Error("cancelled")); + return true; + } + return false; +} diff --git a/src/lab/automation/persistence.ts b/src/lab/automation/persistence.ts new file mode 100644 index 0000000000..7337067990 --- /dev/null +++ b/src/lab/automation/persistence.ts @@ -0,0 +1,217 @@ +import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { + ensureLabDirs, + labAutomationPolicyPath, + labAutomationRoutesPath, + labAutomationStatePath, +} from "../paths"; +import { LAB_AUTOMATION_HARD_MAX } from "./constants"; +import { defaultLabAutomationPolicyV1, normalizeLabAutomationPolicyV1 } from "./policy"; +import type { + LabAutomationPolicyV1, + LabAutomationRoutesV1, + LabAutomationRunRecordV1, + LabAutomationStateV1, +} from "./types"; +import { LabAutomationError } from "./types"; + +function atomicWriteJson(path: string, payload: unknown): void { + const dir = dirname(path); + const tmp = join(dir, `.${basename(path)}.${process.pid}.${Date.now()}.tmp`); + const text = JSON.stringify(payload); + writeFileSync(tmp, text, { encoding: "utf8", mode: 0o600 }); + renameSync(tmp, path); +} + +function basename(path: string): string { + const idx = path.replace(/\\/g, "/").lastIndexOf("/"); + return idx >= 0 ? path.slice(idx + 1) : path; +} + +function readJsonFile(path: string): unknown { + if (!existsSync(path)) return undefined; + const raw = readFileSync(path, "utf8"); + return JSON.parse(raw) as unknown; +} + +export function loadLabAutomationPolicy(configDir?: string): LabAutomationPolicyV1 { + ensureLabDirs(configDir); + const path = labAutomationPolicyPath(configDir); + const raw = readJsonFile(path); + if (raw === undefined) return defaultLabAutomationPolicyV1(); + return normalizeLabAutomationPolicyV1(raw); +} + +export function saveLabAutomationPolicy(policy: LabAutomationPolicyV1, configDir?: string): void { + ensureLabDirs(configDir); + const normalized = normalizeLabAutomationPolicyV1(policy); + atomicWriteJson(labAutomationPolicyPath(configDir), normalized); +} + +function normalizeRoutes(raw: unknown): LabAutomationRoutesV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new LabAutomationError("automation routes must be an object", "invalid_routes"); + } + const obj = raw as Record; + if (obj.schemaVersion !== 1) throw new LabAutomationError("unsupported routes schemaVersion", "invalid_routes"); + if (!Array.isArray(obj.routes)) throw new LabAutomationError("routes must be an array", "invalid_routes"); + if (obj.routes.length > LAB_AUTOMATION_HARD_MAX.maxAutomationRoutes) { + throw new LabAutomationError("too many automation routes", "invalid_routes"); + } + const routes = obj.routes.map((row, index) => { + if (!row || typeof row !== "object" || Array.isArray(row)) { + throw new LabAutomationError(`invalid route row ${index}`, "invalid_routes"); + } + const entry = row as Record; + const providerName = entry.providerName; + const modelId = entry.modelId; + if (typeof providerName !== "string" || providerName.length === 0) { + throw new LabAutomationError(`invalid providerName at ${index}`, "invalid_routes"); + } + if (typeof modelId !== "string" || modelId.length === 0) { + throw new LabAutomationError(`invalid modelId at ${index}`, "invalid_routes"); + } + return { providerName, modelId }; + }); + return Object.freeze({ schemaVersion: 1, routes: Object.freeze(routes) }); +} + +export function defaultLabAutomationRoutesV1(): LabAutomationRoutesV1 { + return Object.freeze({ schemaVersion: 1, routes: Object.freeze([]) }); +} + +export function loadLabAutomationRoutes(configDir?: string): LabAutomationRoutesV1 { + ensureLabDirs(configDir); + const raw = readJsonFile(labAutomationRoutesPath(configDir)); + if (raw === undefined) return defaultLabAutomationRoutesV1(); + return normalizeRoutes(raw); +} + +export function saveLabAutomationRoutes(routes: LabAutomationRoutesV1, configDir?: string): void { + ensureLabDirs(configDir); + atomicWriteJson(labAutomationRoutesPath(configDir), normalizeRoutes(routes)); +} + +function normalizeRunRecord(raw: unknown, index: number): LabAutomationRunRecordV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new LabAutomationError(`invalid run record ${index}`, "invalid_state"); + } + const row = raw as Record; + const state = row.state; + const allowedStates = new Set(["queued", "running", "completed", "blocked", "failed", "cancelled", "abandoned"]); + if (typeof state !== "string" || !allowedStates.has(state)) { + throw new LabAutomationError(`invalid run state at ${index}`, "invalid_state"); + } + const evidenceLayer = row.evidenceLayer; + if (evidenceLayer !== "protocol_conformance" + && evidenceLayer !== "live_route_compatibility" + && evidenceLayer !== "task_effectiveness") { + throw new LabAutomationError(`invalid evidenceLayer at ${index}`, "invalid_state"); + } + const trigger = row.trigger; + if (trigger !== "scheduled" && trigger !== "manual") { + throw new LabAutomationError(`invalid trigger at ${index}`, "invalid_state"); + } + const requiredStrings = [ + "runId", "runKey", "suiteId", "suiteVersion", "suiteManifestDigest", + "scenarioId", "scenarioVersion", "scenarioManifestDigest", "subjectId", "reason", + ]; + for (const key of requiredStrings) { + if (typeof row[key] !== "string" || (row[key] as string).length === 0) { + throw new LabAutomationError(`invalid ${key} at ${index}`, "invalid_state"); + } + } + if (typeof row.priority !== "number" || !Number.isInteger(row.priority)) { + throw new LabAutomationError(`invalid priority at ${index}`, "invalid_state"); + } + if (typeof row.eligibleAt !== "number" || !Number.isInteger(row.eligibleAt)) { + throw new LabAutomationError(`invalid eligibleAt at ${index}`, "invalid_state"); + } + if (typeof row.createdAt !== "number" || !Number.isInteger(row.createdAt)) { + throw new LabAutomationError(`invalid createdAt at ${index}`, "invalid_state"); + } + if (typeof row.updatedAt !== "number" || !Number.isInteger(row.updatedAt)) { + throw new LabAutomationError(`invalid updatedAt at ${index}`, "invalid_state"); + } + return { + runId: row.runId as string, + runKey: row.runKey as string, + state: state as LabAutomationRunRecordV1["state"], + evidenceLayer, + suiteId: row.suiteId as string, + suiteVersion: row.suiteVersion as string, + suiteManifestDigest: row.suiteManifestDigest as string, + scenarioId: row.scenarioId as string, + scenarioVersion: row.scenarioVersion as string, + scenarioManifestDigest: row.scenarioManifestDigest as string, + subjectId: row.subjectId as string, + reason: row.reason as LabAutomationRunRecordV1["reason"], + priority: row.priority as number, + eligibleAt: row.eligibleAt as number, + trigger, + createdAt: row.createdAt as number, + updatedAt: row.updatedAt as number, + ...(typeof row.startedAt === "number" ? { startedAt: row.startedAt } : {}), + ...(typeof row.completedAt === "number" ? { completedAt: row.completedAt } : {}), + ...(typeof row.terminalCode === "string" ? { terminalCode: row.terminalCode } : {}), + ...(typeof row.providerName === "string" ? { providerName: row.providerName } : {}), + ...(typeof row.modelId === "string" ? { modelId: row.modelId } : {}), + }; +} + +function normalizeState(raw: unknown): LabAutomationStateV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new LabAutomationError("automation state must be an object", "invalid_state"); + } + const obj = raw as Record; + if (obj.schemaVersion !== 1) throw new LabAutomationError("unsupported state schemaVersion", "invalid_state"); + if (!Array.isArray(obj.runs)) throw new LabAutomationError("state runs must be an array", "invalid_state"); + if (obj.runs.length > LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { + throw new LabAutomationError("too many persisted runs", "invalid_state"); + } + const runs = obj.runs.map((row, index) => normalizeRunRecord(row, index)); + const cooldownUntilByKey: Record = {}; + if (obj.cooldownUntilByKey !== undefined) { + if (!obj.cooldownUntilByKey || typeof obj.cooldownUntilByKey !== "object" || Array.isArray(obj.cooldownUntilByKey)) { + throw new LabAutomationError("invalid cooldownUntilByKey", "invalid_state"); + } + for (const [key, value] of Object.entries(obj.cooldownUntilByKey as Record)) { + if (typeof value !== "number" || !Number.isInteger(value)) { + throw new LabAutomationError(`invalid cooldown for ${key}`, "invalid_state"); + } + cooldownUntilByKey[key] = value; + } + } + return { + schemaVersion: 1, + runs, + budgetWindowStartedAt: typeof obj.budgetWindowStartedAt === "number" ? obj.budgetWindowStartedAt : 0, + runsThisHour: typeof obj.runsThisHour === "number" ? obj.runsThisHour : 0, + liveRequestsThisHour: typeof obj.liveRequestsThisHour === "number" ? obj.liveRequestsThisHour : 0, + cooldownUntilByKey, + }; +} + +export function defaultLabAutomationStateV1(now = Date.now()): LabAutomationStateV1 { + return { + schemaVersion: 1, + runs: [], + budgetWindowStartedAt: now, + runsThisHour: 0, + liveRequestsThisHour: 0, + cooldownUntilByKey: {}, + }; +} + +export function loadLabAutomationState(configDir?: string): LabAutomationStateV1 { + ensureLabDirs(configDir); + const raw = readJsonFile(labAutomationStatePath(configDir)); + if (raw === undefined) return defaultLabAutomationStateV1(); + return normalizeState(raw); +} + +export function saveLabAutomationState(state: LabAutomationStateV1, configDir?: string): void { + ensureLabDirs(configDir); + atomicWriteJson(labAutomationStatePath(configDir), normalizeState(state)); +} diff --git a/src/lab/automation/planner.ts b/src/lab/automation/planner.ts new file mode 100644 index 0000000000..26f0f440d3 --- /dev/null +++ b/src/lab/automation/planner.ts @@ -0,0 +1,339 @@ +import type { EvidenceLayer } from "../constants"; +import { subjectIdForSubject, scenarioManifestDigest } from "../digest"; +import { discoverScenarios, expandScenario, loadCaseAuthority } from "../conformance/manifest"; +import { CL01_SUITES, CL03_LIVE_SUITES } from "../conformance/types"; +import { resolveProtocolExecutionContext } from "../conformance/executor"; +import { buildProtocolSubjectV1 } from "../subject/protocol-subject"; +import { discoverLiveScenarios, expandLiveScenario, loadLiveCaseAuthority } from "../live/manifest"; +import { isLiveCaseApplicableToRoute } from "../live/executor"; +import { buildAutomationLiveRouteContext } from "./route-context"; +import { liveSuiteManifestDigestForCase } from "../live/suite-manifest"; +import { suiteManifestDigestForCase } from "../conformance/suite-manifest"; +import { newestObservationByScenario, isScenarioApplicable } from "../projection/verification"; +import { queryLabObservations } from "../query/queries"; +import { LabProjectionUnavailableError } from "../query/errors"; +import type { ObservationEvent } from "../events/types"; +import type { OcxConfig } from "../../types"; +import { resolvePolicyCompatibilitySubjects } from "../../routing/compatibility/subject"; +import type { + LabAutomationLayer, + LabAutomationPlanReason, + LabAutomationPolicyV1, + LabAutomationRoutesV1, + LabAutomationRunRecordV1, + LabAutomationStateV1, + PlannedLabRunV1, +} from "./types"; +import { LabAutomationError } from "./types"; +import { + buildLabAutomationRunKey, + liveExecutionContractDigest, + protocolExecutionContractDigest, +} from "./run-key"; +import { cooldownActive } from "./cooldown"; +import { isLiveRequestBudgetExhausted, isRunBudgetExhausted } from "./budgets"; +import { LAB_AUTOMATION_HARD_MAX } from "./constants"; + +export interface PlannerInput { + policy: LabAutomationPolicyV1; + routes: LabAutomationRoutesV1; + state: LabAutomationStateV1; + now: number; + config?: OcxConfig; + configDir?: string; +} + +function effectiveMaxAgeMs(suiteMax: number | null, scenarioMax: number | null): number | null { + if (suiteMax === null) return scenarioMax; + if (scenarioMax === null) return suiteMax; + return Math.min(suiteMax, scenarioMax); +} + +function freshnessReason( + latestCompletedAt: number | undefined, + maxAgeMs: number | null, + refreshBeforeStaleMs: number, + now: number, +): LabAutomationPlanReason { + if (latestCompletedAt === undefined) return "missing"; + if (maxAgeMs === null) return "fresh"; + const deadline = latestCompletedAt + maxAgeMs; + const refreshAt = deadline - refreshBeforeStaleMs; + if (now >= deadline) return "refresh_due"; + if (now >= refreshAt) return "refresh_due"; + return "fresh"; +} + +function activeRunForKey(state: LabAutomationStateV1, runKey: string): LabAutomationRunRecordV1 | undefined { + return state.runs.find((row) => row.runKey === runKey && (row.state === "queued" || row.state === "running")); +} + +function loadObservationEvents(layer: EvidenceLayer, configDir?: string): ObservationEvent[] { + try { + const page = queryLabObservations( + { layer }, + undefined, + LAB_AUTOMATION_HARD_MAX.maxPersistedRuns, + configDir, + ); + return page.items.map((row) => ({ + eventKind: "observation", + eventId: row.eventId, + scenarioId: row.scenarioId, + completedAt: row.completedAt, + } as ObservationEvent)); + } catch (error) { + if (error instanceof LabProjectionUnavailableError) return []; + throw error; + } +} + +function planProtocolScenarios(input: PlannerInput): PlannedLabRunV1[] { + if (!input.policy.enabled || !input.policy.layers.protocolConformance) return []; + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, CL01_SUITES); + const planned: PlannedLabRunV1[] = []; + const observations = loadObservationEvents("protocol_conformance", input.configDir); + const byScenario = newestObservationByScenario(observations); + for (const caseRecord of scenarios) { + if (!isScenarioApplicable(caseRecord.id, "fixture", "protocol_conformance")) continue; + let subject; + try { + const ctx = resolveProtocolExecutionContext(caseRecord); + subject = buildProtocolSubjectV1(ctx); + } catch { + continue; + } + const subjectId = subjectIdForSubject(subject); + const expanded = expandScenario(caseRecord, authority); + const scenarioDigest = scenarioManifestDigest(expanded); + const suiteDigest = suiteManifestDigestForCase(caseRecord, authority); + const runKey = buildLabAutomationRunKey({ + evidenceLayer: "protocol_conformance", + subjectId, + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: scenarioDigest, + executionContractDigest: protocolExecutionContractDigest(), + }); + if (activeRunForKey(input.state, runKey)) continue; + if (cooldownActive(input.state, runKey, input.now)) continue; + const latest = byScenario.get(caseRecord.id); + const maxAge = effectiveMaxAgeMs( + authority.manifestDefaults.freshness.maxAgeMs, + authority.manifestDefaults.freshness.maxAgeMs, + ); + const freshness = freshnessReason(latest?.completedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); + if (freshness === "fresh") continue; + planned.push({ + runKey, + evidenceLayer: "protocol_conformance", + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: scenarioDigest, + subjectId, + reason: freshness, + priority: freshness === "missing" ? 0 : 1, + eligibleAt: input.now, + }); + } + return planned; +} + +function planLiveScenarios(input: PlannerInput): PlannedLabRunV1[] { + if (!input.policy.enabled || !input.policy.layers.liveRouteCompatibility) return []; + if (!input.config) return []; + if (isLiveRequestBudgetExhausted(input.policy, input.state)) return []; + const authority = loadLiveCaseAuthority(); + const scenarios = discoverLiveScenarios(authority, CL03_LIVE_SUITES); + const planned: PlannedLabRunV1[] = []; + const observations = loadObservationEvents("live_route_compatibility", input.configDir); + const byScenario = newestObservationByScenario(observations); + for (const routeRef of input.routes.routes) { + const routed = input.config.providers?.[routeRef.providerName]; + if (!routed) continue; + const resolved = resolvePolicyCompatibilitySubjects( + input.config, + routeRef.providerName, + routeRef.modelId, + routed, + input.configDir, + ); + if (!resolved.route) continue; + const routeContext = buildAutomationLiveRouteContext( + resolved.route, + routed.allowPrivateNetwork === true, + ); + for (const caseRecord of scenarios) { + if (!isLiveCaseApplicableToRoute(caseRecord, routeContext)) continue; + const expanded = expandLiveScenario(caseRecord, authority); + const scenarioDigest = scenarioManifestDigest(expanded); + const suiteDigest = liveSuiteManifestDigestForCase(caseRecord, authority); + const subjectId = resolved.route.subjectId; + const runKey = buildLabAutomationRunKey({ + evidenceLayer: "live_route_compatibility", + subjectId, + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: scenarioDigest, + executionContractDigest: liveExecutionContractDigest(), + }); + if (activeRunForKey(input.state, runKey)) continue; + if (cooldownActive(input.state, runKey, input.now)) continue; + const latest = byScenario.get(caseRecord.id); + const maxAge = effectiveMaxAgeMs( + authority.manifestDefaults.freshness.maxAgeMs, + authority.manifestDefaults.freshness.maxAgeMs, + ); + const freshness = freshnessReason(latest?.completedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); + if (freshness === "fresh") continue; + planned.push({ + runKey, + evidenceLayer: "live_route_compatibility", + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: scenarioDigest, + subjectId, + reason: freshness, + priority: freshness === "missing" ? 0 : 1, + eligibleAt: input.now, + providerName: routeRef.providerName, + modelId: routeRef.modelId, + }); + } + } + return planned; +} + +/** Deterministic planner — no execution side effects. */ +export function planLabAutomationRuns(input: PlannerInput): PlannedLabRunV1[] { + if (!input.policy.enabled) return []; + if (isRunBudgetExhausted(input.policy, input.state)) return []; + const protocol = planProtocolScenarios(input); + const live = planLiveScenarios(input); + const merged = [...protocol, ...live]; + merged.sort((a, b) => { + if (a.priority !== b.priority) return a.priority - b.priority; + if (a.eligibleAt !== b.eligibleAt) return a.eligibleAt - b.eligibleAt; + return a.runKey < b.runKey ? -1 : a.runKey > b.runKey ? 1 : 0; + }); + const cap = LAB_AUTOMATION_HARD_MAX.maxQueuedRuns; + return merged.slice(0, cap); +} + +export interface ManualLabRunInput { + evidenceLayer: LabAutomationLayer; + scenarioId: string; + providerName?: string; + modelId?: string; + config?: OcxConfig; + configDir?: string; +} + +/** Build a single manual run plan — independent of automation enablement. */ +export function planManualLabRun(input: ManualLabRunInput): PlannedLabRunV1 { + const now = Date.now(); + if (input.evidenceLayer === "task_effectiveness") { + throw new LabAutomationError("manual task_effectiveness uses CL-07 tooling", "task_background_disabled"); + } + if (input.evidenceLayer === "protocol_conformance") { + const authority = loadCaseAuthority(); + const caseRecord = authority.cases.find((row) => row.id === input.scenarioId); + if (!caseRecord) throw new LabAutomationError("unknown protocol scenario", "scenario_inapplicable"); + const ctx = resolveProtocolExecutionContext(caseRecord); + const subject = buildProtocolSubjectV1(ctx); + const subjectId = subjectIdForSubject(subject); + const expanded = expandScenario(caseRecord, authority); + const scenarioDigest = scenarioManifestDigest(expanded); + const suiteDigest = suiteManifestDigestForCase(caseRecord, authority); + return { + runKey: buildLabAutomationRunKey({ + evidenceLayer: "protocol_conformance", + subjectId, + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: scenarioDigest, + executionContractDigest: protocolExecutionContractDigest(), + }), + evidenceLayer: "protocol_conformance", + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: scenarioDigest, + subjectId, + reason: "missing", + priority: 0, + eligibleAt: now, + }; + } + if (!input.config || !input.providerName || !input.modelId) { + throw new LabAutomationError("live manual run requires providerName and modelId", "route_ineligible"); + } + const routed = input.config.providers?.[input.providerName]; + if (!routed) throw new LabAutomationError("unknown provider route", "route_ineligible"); + const resolved = resolvePolicyCompatibilitySubjects( + input.config, + input.providerName, + input.modelId, + routed, + input.configDir, + ); + if (!resolved.route) throw new LabAutomationError("route ineligible for compatibility subject", "route_ineligible"); + const authority = loadLiveCaseAuthority(); + const caseRecord = authority.cases.find((row) => row.id === input.scenarioId); + if (!caseRecord) throw new LabAutomationError("unknown live scenario", "scenario_inapplicable"); + const routeContext = buildAutomationLiveRouteContext( + resolved.route, + routed.allowPrivateNetwork === true, + ); + if (!isLiveCaseApplicableToRoute(caseRecord, routeContext)) { + throw new LabAutomationError("scenario inapplicable to route", "scenario_inapplicable"); + } + const expanded = expandLiveScenario(caseRecord, authority); + const scenarioDigest = scenarioManifestDigest(expanded); + const suiteDigest = liveSuiteManifestDigestForCase(caseRecord, authority); + const subjectId = resolved.route.subjectId; + return { + runKey: buildLabAutomationRunKey({ + evidenceLayer: "live_route_compatibility", + subjectId, + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: scenarioDigest, + executionContractDigest: liveExecutionContractDigest(), + }), + evidenceLayer: "live_route_compatibility", + suiteId: caseRecord.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: scenarioDigest, + subjectId, + reason: "missing", + priority: 0, + eligibleAt: now, + providerName: input.providerName, + modelId: input.modelId, + }; +} diff --git a/src/lab/automation/policy.ts b/src/lab/automation/policy.ts new file mode 100644 index 0000000000..59cc6d34c4 --- /dev/null +++ b/src/lab/automation/policy.ts @@ -0,0 +1,98 @@ +import { + LAB_AUTOMATION_DEFAULT_POLICY, + LAB_AUTOMATION_HARD_MAX, + LAB_AUTOMATION_POLICY_SCHEMA_VERSION, +} from "./constants"; +import type { LabAutomationPolicyV1 } from "./types"; +import { LabAutomationError } from "./types"; + +function assertBoolean(value: unknown, field: string): boolean { + if (typeof value !== "boolean") throw new LabAutomationError(`invalid policy field ${field}`, "invalid_policy"); + return value; +} + +function assertNonNegativeInt(value: unknown, field: string, max: number): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { + throw new LabAutomationError(`invalid policy field ${field}`, "invalid_policy"); + } + if (value > max) { + throw new LabAutomationError(`policy field ${field} exceeds hard maximum`, "invalid_policy"); + } + return value; +} + +/** Validate and normalize automation policy with hard upper bounds. */ +export function normalizeLabAutomationPolicyV1(raw: unknown): LabAutomationPolicyV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new LabAutomationError("policy must be an object", "invalid_policy"); + } + const obj = raw as Record; + if (obj.schemaVersion !== LAB_AUTOMATION_POLICY_SCHEMA_VERSION) { + throw new LabAutomationError("unsupported policy schemaVersion", "invalid_policy"); + } + const layersRaw = obj.layers; + if (!layersRaw || typeof layersRaw !== "object" || Array.isArray(layersRaw)) { + throw new LabAutomationError("invalid policy layers", "invalid_policy"); + } + const layersObj = layersRaw as Record; + const taskBackground = assertBoolean(obj.taskEffectivenessBackgroundEnabled, "taskEffectivenessBackgroundEnabled"); + const layers = { + protocolConformance: assertBoolean(layersObj.protocolConformance, "layers.protocolConformance"), + liveRouteCompatibility: assertBoolean(layersObj.liveRouteCompatibility, "layers.liveRouteCompatibility"), + taskEffectiveness: assertBoolean(layersObj.taskEffectiveness, "layers.taskEffectiveness"), + }; + if (taskBackground && !layers.taskEffectiveness) { + throw new LabAutomationError("task background requires task_effectiveness layer", "invalid_policy"); + } + return Object.freeze({ + schemaVersion: LAB_AUTOMATION_POLICY_SCHEMA_VERSION, + enabled: assertBoolean(obj.enabled, "enabled"), + layers: Object.freeze(layers), + refreshBeforeStaleMs: assertNonNegativeInt( + obj.refreshBeforeStaleMs, + "refreshBeforeStaleMs", + LAB_AUTOMATION_HARD_MAX.refreshBeforeStaleMs, + ), + maxConcurrentRuns: assertNonNegativeInt( + obj.maxConcurrentRuns, + "maxConcurrentRuns", + LAB_AUTOMATION_HARD_MAX.maxConcurrentRuns, + ), + maxConcurrentLiveRuns: assertNonNegativeInt( + obj.maxConcurrentLiveRuns, + "maxConcurrentLiveRuns", + LAB_AUTOMATION_HARD_MAX.maxConcurrentLiveRuns, + ), + maxConcurrentRunsPerRoute: assertNonNegativeInt( + obj.maxConcurrentRunsPerRoute, + "maxConcurrentRunsPerRoute", + LAB_AUTOMATION_HARD_MAX.maxConcurrentRunsPerRoute, + ), + maxRunsPerHour: assertNonNegativeInt( + obj.maxRunsPerHour, + "maxRunsPerHour", + LAB_AUTOMATION_HARD_MAX.maxRunsPerHour, + ), + maxLiveRequestsPerHour: assertNonNegativeInt( + obj.maxLiveRequestsPerHour, + "maxLiveRequestsPerHour", + LAB_AUTOMATION_HARD_MAX.maxLiveRequestsPerHour, + ), + failureCooldownMs: assertNonNegativeInt( + obj.failureCooldownMs, + "failureCooldownMs", + LAB_AUTOMATION_HARD_MAX.failureCooldownMs, + ), + blockedCooldownMs: assertNonNegativeInt( + obj.blockedCooldownMs, + "blockedCooldownMs", + LAB_AUTOMATION_HARD_MAX.blockedCooldownMs, + ), + taskEffectivenessBackgroundEnabled: taskBackground, + }); +} + +/** Default-off automation policy for new installations. */ +export function defaultLabAutomationPolicyV1(): LabAutomationPolicyV1 { + return normalizeLabAutomationPolicyV1({ ...LAB_AUTOMATION_DEFAULT_POLICY }); +} diff --git a/src/lab/automation/queue.ts b/src/lab/automation/queue.ts new file mode 100644 index 0000000000..3897e45e7f --- /dev/null +++ b/src/lab/automation/queue.ts @@ -0,0 +1,140 @@ +import { randomUUID } from "node:crypto"; +import type { + LabAutomationPolicyV1, + LabAutomationRunRecordV1, + LabAutomationRunState, + LabAutomationStateV1, + PlannedLabRunV1, +} from "./types"; +import { LAB_AUTOMATION_HARD_MAX } from "./constants"; + +export function countRunsByState(state: LabAutomationStateV1, runState: LabAutomationRunState): number { + return state.runs.filter((row) => row.state === runState).length; +} + +export function countRunningLive(state: LabAutomationStateV1): number { + return state.runs.filter((row) => row.state === "running" && row.evidenceLayer === "live_route_compatibility").length; +} + +export function countRunningForRoute(state: LabAutomationStateV1, subjectId: string): number { + return state.runs.filter((row) => row.state === "running" && row.subjectId === subjectId).length; +} + +export function findRunById(state: LabAutomationStateV1, runId: string): LabAutomationRunRecordV1 | undefined { + return state.runs.find((row) => row.runId === runId); +} + +export function enqueuePlannedRuns( + state: LabAutomationStateV1, + planned: PlannedLabRunV1[], + trigger: LabAutomationRunRecordV1["trigger"], + now: number, +): LabAutomationStateV1 { + const existingActiveKeys = new Set( + state.runs + .filter((row) => row.state === "queued" || row.state === "running") + .map((row) => row.runKey), + ); + const runs = [...state.runs]; + for (const plan of planned) { + if (existingActiveKeys.has(plan.runKey)) continue; + if (runs.length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) break; + runs.push({ + runId: randomUUID(), + runKey: plan.runKey, + state: "queued", + evidenceLayer: plan.evidenceLayer, + suiteId: plan.suiteId, + suiteVersion: plan.suiteVersion, + suiteManifestDigest: plan.suiteManifestDigest, + scenarioId: plan.scenarioId, + scenarioVersion: plan.scenarioVersion, + scenarioManifestDigest: plan.scenarioManifestDigest, + subjectId: plan.subjectId, + reason: plan.reason, + priority: plan.priority, + eligibleAt: plan.eligibleAt, + trigger, + createdAt: now, + updatedAt: now, + ...(plan.providerName ? { providerName: plan.providerName } : {}), + ...(plan.modelId ? { modelId: plan.modelId } : {}), + }); + existingActiveKeys.add(plan.runKey); + } + return { ...state, runs }; +} + +export function transitionRun( + state: LabAutomationStateV1, + runId: string, + next: LabAutomationRunState, + now: number, + terminalCode?: string, +): LabAutomationStateV1 { + const runs = state.runs.map((row) => { + if (row.runId !== runId) return row; + return { + ...row, + state: next, + updatedAt: now, + ...(next === "running" ? { startedAt: now } : {}), + ...(next === "completed" || next === "blocked" || next === "failed" || next === "cancelled" || next === "abandoned" + ? { completedAt: now, ...(terminalCode ? { terminalCode } : {}) } + : {}), + }; + }); + return { ...state, runs }; +} + +export function cancelQueuedRun(state: LabAutomationStateV1, runId: string, now: number): LabAutomationStateV1 { + const run = findRunById(state, runId); + if (!run || run.state !== "queued") return state; + return transitionRun(state, runId, "cancelled", now, "cancelled"); +} + +export function selectDispatchableRuns( + policy: LabAutomationPolicyV1, + state: LabAutomationStateV1, + now: number, +): LabAutomationRunRecordV1[] { + const running = countRunsByState(state, "running"); + if (running >= policy.maxConcurrentRuns) return []; + const slots = policy.maxConcurrentRuns - running; + const queued = state.runs + .filter((row) => row.state === "queued" && row.eligibleAt <= now) + .sort((a, b) => { + if (a.priority !== b.priority) return a.priority - b.priority; + if (a.eligibleAt !== b.eligibleAt) return a.eligibleAt - b.eligibleAt; + return a.runId < b.runId ? -1 : 1; + }); + const selected: LabAutomationRunRecordV1[] = []; + let liveRunning = countRunningLive(state); + for (const row of queued) { + if (selected.length >= slots) break; + if (countRunsByState({ ...state, runs: [...state.runs, ...selected.map((s) => ({ ...s, state: "running" as const }))] }, "running") >= policy.maxConcurrentRuns) break; + if (row.evidenceLayer === "live_route_compatibility") { + if (liveRunning >= policy.maxConcurrentLiveRuns) continue; + if (countRunningForRoute(state, row.subjectId) >= policy.maxConcurrentRunsPerRoute) continue; + } else if (countRunningForRoute(state, row.subjectId) >= policy.maxConcurrentRunsPerRoute) { + continue; + } + selected.push(row); + if (row.evidenceLayer === "live_route_compatibility") liveRunning += 1; + } + return selected; +} + +export function trimTerminalRuns(state: LabAutomationStateV1, now: number): LabAutomationStateV1 { + const terminal = new Set(["completed", "blocked", "failed", "cancelled", "abandoned"]); + const keepMs = 7 * 24 * 60 * 60 * 1000; + const runs = state.runs.filter((row) => { + if (!terminal.has(row.state)) return true; + const completedAt = row.completedAt ?? row.updatedAt; + return now - completedAt < keepMs; + }); + if (runs.length > LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { + return { ...state, runs: runs.slice(-LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) }; + } + return { ...state, runs }; +} diff --git a/src/lab/automation/recovery.ts b/src/lab/automation/recovery.ts new file mode 100644 index 0000000000..1408585611 --- /dev/null +++ b/src/lab/automation/recovery.ts @@ -0,0 +1,23 @@ +import type { LabAutomationPolicyV1, LabAutomationStateV1 } from "./types"; +import { transitionRun } from "./queue"; +import { setCooldown } from "./cooldown"; +import { LAB_AUTOMATION_HARD_MAX } from "./constants"; + +/** Reconcile runs left running after process restart. */ +export function recoverLabAutomationState( + policy: LabAutomationPolicyV1, + state: LabAutomationStateV1, + now: number, +): LabAutomationStateV1 { + let next = { ...state }; + for (const run of state.runs) { + if (run.state !== "running") continue; + next = transitionRun(next, run.runId, "abandoned", now, "process_restart"); + next = setCooldown( + next, + run.runKey, + now + Math.min(policy.failureCooldownMs, LAB_AUTOMATION_HARD_MAX.failureCooldownMs), + ); + } + return next; +} diff --git a/src/lab/automation/route-context.ts b/src/lab/automation/route-context.ts new file mode 100644 index 0000000000..a1f658ef2e --- /dev/null +++ b/src/lab/automation/route-context.ts @@ -0,0 +1,21 @@ +import type { LabRouteContext } from "../live/types"; +import type { ResolvedPolicyRouteSubject } from "../../routing/compatibility/subject"; + +/** Harness features the production trusted-route automation contract actually provides. */ +export const CL08_TRUSTED_LIVE_HARNESS_FEATURES = Object.freeze([ + "live_transport", +]); + +/** Build the CL-03 route context automation dispatch and planning share. */ +export function buildAutomationLiveRouteContext( + resolved: ResolvedPolicyRouteSubject, + allowPrivateNetwork = false, +): LabRouteContext { + return { + ...resolved.routeContext, + labRunApproval: true, + allowPrivateNetwork, + requiredClaims: resolved.routeContext.requiredClaims ?? [], + availableHarnessFeatures: [...CL08_TRUSTED_LIVE_HARNESS_FEATURES], + }; +} diff --git a/src/lab/automation/run-key.ts b/src/lab/automation/run-key.ts new file mode 100644 index 0000000000..bee0d8f9f7 --- /dev/null +++ b/src/lab/automation/run-key.ts @@ -0,0 +1,44 @@ +import { createHash } from "node:crypto"; +import { jcsStringify } from "../digest"; +import type { LabAutomationLayer } from "./types"; + +export interface RunKeyInput { + evidenceLayer: LabAutomationLayer; + subjectId: string; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + scenarioId: string; + scenarioVersion: string; + scenarioManifestDigest: string; + executionContractDigest: string; +} + +/** Deterministic deduplication key for orchestration runs. */ +export function buildLabAutomationRunKey(input: RunKeyInput): string { + const payload = { + evidenceLayer: input.evidenceLayer, + subjectId: input.subjectId, + suiteId: input.suiteId, + suiteVersion: input.suiteVersion, + suiteManifestDigest: input.suiteManifestDigest, + scenarioId: input.scenarioId, + scenarioVersion: input.scenarioVersion, + scenarioManifestDigest: input.scenarioManifestDigest, + executionContractDigest: input.executionContractDigest, + }; + return createHash("sha256").update(jcsStringify(payload)).digest("hex"); +} + +/** Contract identity digest for protocol/live/fabric execution boundaries. */ +export function protocolExecutionContractDigest(): string { + return createHash("sha256").update(jcsStringify({ contract: "cl-01-protocol-harness-v1" })).digest("hex"); +} + +export function liveExecutionContractDigest(): string { + return createHash("sha256").update(jcsStringify({ contract: "cl-03-live-trusted-route-v1" })).digest("hex"); +} + +export function fabricExecutionContractDigest(): string { + return createHash("sha256").update(jcsStringify({ contract: "cl-07-fabric-trusted-route-v1" })).digest("hex"); +} diff --git a/src/lab/automation/runs-query.ts b/src/lab/automation/runs-query.ts new file mode 100644 index 0000000000..85fb43338d --- /dev/null +++ b/src/lab/automation/runs-query.ts @@ -0,0 +1,32 @@ +import type { LabAutomationRunRecordV1, LabAutomationStateV1 } from "./types"; +import { LAB_AUTOMATION_HARD_MAX } from "./constants"; + +export interface LabAutomationRunsPage { + items: LabAutomationRunRecordV1[]; + hasMore: boolean; + nextCursor?: string; +} + +export function listLabAutomationRuns( + state: LabAutomationStateV1, + limit = 50, + cursor?: string, +): LabAutomationRunsPage { + const capped = Math.min(Math.max(1, limit), LAB_AUTOMATION_HARD_MAX.maxPersistedRuns); + const sorted = [...state.runs].sort((a, b) => { + if (a.createdAt !== b.createdAt) return b.createdAt - a.createdAt; + return a.runId < b.runId ? -1 : 1; + }); + let start = 0; + if (cursor) { + const index = sorted.findIndex((row) => row.runId === cursor); + start = index >= 0 ? index + 1 : 0; + } + const slice = sorted.slice(start, start + capped); + const hasMore = start + capped < sorted.length; + return { + items: slice, + hasMore, + ...(hasMore && slice.length > 0 ? { nextCursor: slice[slice.length - 1]!.runId } : {}), + }; +} diff --git a/src/lab/automation/types.ts b/src/lab/automation/types.ts new file mode 100644 index 0000000000..68a882a1db --- /dev/null +++ b/src/lab/automation/types.ts @@ -0,0 +1,158 @@ +import type { EvidenceLayer } from "../constants"; + +export type LabAutomationLayer = + | "protocol_conformance" + | "live_route_compatibility" + | "task_effectiveness"; + +export interface LabAutomationPolicyV1 { + schemaVersion: 1; + enabled: boolean; + layers: { + protocolConformance: boolean; + liveRouteCompatibility: boolean; + taskEffectiveness: boolean; + }; + refreshBeforeStaleMs: number; + maxConcurrentRuns: number; + maxConcurrentLiveRuns: number; + maxConcurrentRunsPerRoute: number; + maxRunsPerHour: number; + maxLiveRequestsPerHour: number; + failureCooldownMs: number; + blockedCooldownMs: number; + taskEffectivenessBackgroundEnabled: boolean; +} + +export type LabAutomationPlanReason = + | "automation_disabled" + | "layer_disabled" + | "fresh" + | "refresh_due" + | "missing" + | "cooldown" + | "budget_blocked" + | "already_queued" + | "already_running" + | "route_ineligible" + | "scenario_inapplicable" + | "task_background_disabled"; + +export interface PlannedLabRunV1 { + runKey: string; + evidenceLayer: LabAutomationLayer; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + scenarioId: string; + scenarioVersion: string; + scenarioManifestDigest: string; + subjectId: string; + reason: LabAutomationPlanReason; + priority: number; + eligibleAt: number; + providerName?: string; + modelId?: string; +} + +export type LabAutomationRunState = + | "queued" + | "running" + | "completed" + | "blocked" + | "failed" + | "cancelled" + | "abandoned"; + +export type LabAutomationRunTrigger = "scheduled" | "manual"; + +export interface LabAutomationRunRecordV1 { + runId: string; + runKey: string; + state: LabAutomationRunState; + evidenceLayer: LabAutomationLayer; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + scenarioId: string; + scenarioVersion: string; + scenarioManifestDigest: string; + subjectId: string; + reason: LabAutomationPlanReason; + priority: number; + eligibleAt: number; + trigger: LabAutomationRunTrigger; + createdAt: number; + updatedAt: number; + startedAt?: number; + completedAt?: number; + terminalCode?: string; + providerName?: string; + modelId?: string; +} + +export interface LabAutomationRouteRefV1 { + providerName: string; + modelId: string; +} + +export interface LabAutomationRoutesV1 { + schemaVersion: 1; + routes: readonly LabAutomationRouteRefV1[]; +} + +export interface LabAutomationStateV1 { + schemaVersion: 1; + runs: LabAutomationRunRecordV1[]; + budgetWindowStartedAt: number; + runsThisHour: number; + liveRequestsThisHour: number; + cooldownUntilByKey: Record; +} + +export interface LabAutomationStatusV1 { + policy: LabAutomationPolicyV1; + routes: LabAutomationRoutesV1; + counters: { + queued: number; + running: number; + completedLastHour: number; + blockedLastHour: number; + remainingRunBudget: number; + remainingLiveRequestBudget: number; + }; + schedulerRunning: boolean; +} + +export interface AutomationDispatchDeps { + configDir?: string; + routeExecutor?: import("../live/types").TrustedLabRouteExecutor; + patchExecutor?: import("../fabric/types").TrustedFabricPatchExecutor; + loadConfig?: () => import("../../types").OcxConfig; + abortSignal?: AbortSignal; + /** Test-only DNS seam; production uses destination policy resolution when unset. */ + resolve?: import("../live/types").DnsResolver; +} + +export class LabAutomationError extends Error { + override readonly name = "LabAutomationError"; + constructor( + message: string, + readonly code: string, + ) { + super(message); + } +} + +export function evidenceLayerToAutomationLayer(layer: EvidenceLayer): LabAutomationLayer | null { + switch (layer) { + case "protocol_conformance": + return "protocol_conformance"; + case "live_route_compatibility": + return "live_route_compatibility"; + case "task_effectiveness": + return "task_effectiveness"; + default: + return null; + } +} diff --git a/src/lab/index.ts b/src/lab/index.ts index 3020e556f4..783eb05526 100644 --- a/src/lab/index.ts +++ b/src/lab/index.ts @@ -35,3 +35,4 @@ export * from "./subject/behavior-fingerprint"; export * from "./subject/installation-salt"; export { CL03_LIVE_SUITES } from "./conformance/types"; export * from "./query"; +export * from "./automation"; diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 754b78bb4d..f202f4121c 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -90,6 +90,18 @@ export function labInstallationSaltPath(configDir = getConfigDir()): string { return join(labRoot(configDir), "installation-salt.bin"); } +export function labAutomationPolicyPath(configDir = getConfigDir()): string { + return join(labRoot(configDir), "automation-policy.json"); +} + +export function labAutomationStatePath(configDir = getConfigDir()): string { + return join(labRoot(configDir), "automation-state.json"); +} + +export function labAutomationRoutesPath(configDir = getConfigDir()): string { + return join(labRoot(configDir), "automation-routes.json"); +} + /** Ensure lab directories exist with restrictive permissions where the platform allows. */ export function ensureLabDirs(configDir = getConfigDir()): { root: string; From 6b3d9446836d21a4dd2cd4f5847af24ef91db49a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:42:48 +0200 Subject: [PATCH 03/67] feat(lab): wire trusted live-route automation Issue production CL-03 executor at server startup, expose management API and CLI, and gate live evidence persistence on trusted_route authority. --- src/cli/lab.ts | 113 +++++++++++- src/lab/live/executor.ts | 33 +++- src/lib/lab-live-route-production.ts | 112 ++++++++++++ src/server/index.ts | 21 +++ src/server/lifecycle.ts | 3 + src/server/management-api.ts | 2 + .../management/lab-automation-routes.ts | 168 ++++++++++++++++++ 7 files changed, 447 insertions(+), 5 deletions(-) create mode 100644 src/lib/lab-live-route-production.ts create mode 100644 src/server/management/lab-automation-routes.ts diff --git a/src/cli/lab.ts b/src/cli/lab.ts index 1fdfaf8647..a9e88b52e4 100644 --- a/src/cli/lab.ts +++ b/src/cli/lab.ts @@ -43,6 +43,21 @@ import { takeIntegerOption, takeOption, } from "./runtime-api"; +import { readConfigDiagnostics } from "../config"; +import { + buildLabAutomationStatus, + enqueueManualLabRun, + startLabAutomationScheduler, + stopLabAutomationScheduler, +} from "../lab/automation/orchestrator"; +import { + loadLabAutomationPolicy, + saveLabAutomationPolicy, + loadLabAutomationState, +} from "../lab/automation/persistence"; +import { planManualLabRun } from "../lab/automation/planner"; +import { listLabAutomationRuns } from "../lab/automation/runs-query"; +import type { LabAutomationLayer } from "../lab/automation/types"; const USAGE = `Usage: ocx lab status [--json] @@ -54,7 +69,12 @@ const USAGE = `Usage: ocx lab event [--json] ocx lab artifacts [--status ] [--artifact-class ] [--limit ] [--cursor ] [--json] ocx lab artifact [--json] - ocx lab catalog [--layer ] [--suite ] [--json]`; + ocx lab catalog [--layer ] [--suite ] [--json] + ocx lab automation status [--json] + ocx lab automation enable [--protocol] [--live] [--json] + ocx lab automation disable [--json] + ocx lab automation runs [--limit ] [--cursor ] [--json] + ocx lab run --layer --scenario [--provider ] [--model ] [--json]`; const ARTIFACT_STATUSES = ["present", "corrupt", "purged_unavailable"] as const; type ArtifactStatus = (typeof ARTIFACT_STATUSES)[number]; @@ -163,6 +183,23 @@ function catalogLines(scenarios: ReturnType): str return lines.length > 0 ? lines : ["No catalog scenarios"]; } +function automationStatusLines(status: ReturnType): string[] { + return [ + `Automation enabled: ${status.policy.enabled}`, + `Scheduler running: ${status.schedulerRunning}`, + `Layers: protocol=${status.policy.layers.protocolConformance} live=${status.policy.layers.liveRouteCompatibility} task=${status.policy.layers.taskEffectiveness}`, + `Queued: ${status.counters.queued} | Running: ${status.counters.running}`, + `Run budget remaining: ${status.counters.remainingRunBudget} | Live request budget: ${status.counters.remainingLiveRequestBudget}`, + ]; +} + +function runListLines(page: ReturnType): string[] { + const lines = page.items.map((row) => + `${row.runId} ${row.state} ${row.evidenceLayer} ${row.scenarioId} trigger=${row.trigger}`, + ); + return lines.length > 0 ? lines : ["No automation runs"]; +} + export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): Promise { return runCliAction(async () => { const configDir = deps.configDir ?? getConfigDir(); @@ -351,6 +388,80 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P printData({ scenarios }, wantsJson, catalogLines(scenarios)); return; } + case "automation": { + const [automationSub = "status", ...automationRest] = rest; + switch (automationSub) { + case "status": { + rejectArgs(automationRest, USAGE); + const status = buildLabAutomationStatus(configDir); + printData(status, wantsJson, automationStatusLines(status)); + return; + } + case "enable": { + const enableLive = takeFlag(automationRest, "--live"); + rejectArgs(automationRest, USAGE); + const policy = { + ...loadLabAutomationPolicy(configDir), + enabled: true, + layers: { + protocolConformance: true, + liveRouteCompatibility: enableLive, + taskEffectiveness: false, + }, + taskEffectivenessBackgroundEnabled: false, + }; + saveLabAutomationPolicy(policy, configDir); + startLabAutomationScheduler(configDir); + const status = buildLabAutomationStatus(configDir); + printData(status, wantsJson, automationStatusLines(status)); + return; + } + case "disable": { + rejectArgs(automationRest, USAGE); + const policy = { ...loadLabAutomationPolicy(configDir), enabled: false }; + saveLabAutomationPolicy(policy, configDir); + stopLabAutomationScheduler(); + const status = buildLabAutomationStatus(configDir); + printData(status, wantsJson, automationStatusLines(status)); + return; + } + case "runs": { + const limit = takeIntegerOption(automationRest, "--limit", { min: 1 }); + const cursor = takeOption(automationRest, "--cursor"); + rejectArgs(automationRest, USAGE); + const page = listLabAutomationRuns(loadLabAutomationState(configDir), limit ?? 50, cursor); + printData(page, wantsJson, runListLines(page)); + return; + } + default: + throw new CliUsageError(`unknown automation subcommand: ${automationSub}`, USAGE); + } + } + case "run": { + const layer = takeEnumOption( + rest, + "--layer", + ["protocol_conformance", "live_route_compatibility", "task_effectiveness"] as const, + "--layer must be protocol_conformance, live_route_compatibility, or task_effectiveness", + ); + const scenarioId = takeOption(rest, "--scenario"); + const providerName = takeOption(rest, "--provider"); + const modelId = takeOption(rest, "--model"); + rejectArgs(rest, USAGE); + if (!layer || !scenarioId) throw new CliUsageError("--layer and --scenario are required", USAGE); + const planned = planManualLabRun({ + evidenceLayer: layer, + scenarioId, + providerName, + modelId, + config: readConfigDiagnostics().config, + configDir, + }); + const record = await enqueueManualLabRun(planned, configDir); + if (!record) throw new CliUsageError("manual run enqueue failed", USAGE); + printData({ run: record }, wantsJson, [`Manual run ${record.runId} -> ${record.state}`]); + return; + } default: throw new CliUsageError(`unknown lab subcommand: ${sub}`, USAGE); } diff --git a/src/lab/live/executor.ts b/src/lab/live/executor.ts index fe71947a9f..31d35e6284 100644 --- a/src/lab/live/executor.ts +++ b/src/lab/live/executor.ts @@ -21,6 +21,8 @@ export interface LiveExecutorOptions { resolve?: DnsResolver; configDir?: string; env?: NodeJS.ProcessEnv; + /** Orchestration cancellation — must not produce route incompatibility evidence. */ + cancelSignal?: AbortSignal; } interface TrustedLiveResultReceipt { @@ -143,6 +145,11 @@ function pathForProtocol(protocol: string): string { return "/responses"; } +/** Upstream HTTP path for a trusted live-route request (CL-03 / CL-08 production dispatch). */ +export function liveUpstreamRequestPath(protocol: string): string { + return pathForProtocol(protocol); +} + function chatObservation(body: string, status: number): NormalizedObservation { const observation = emptyObservation(); const output: unknown[] = []; @@ -215,15 +222,33 @@ function normalizeTransportObservation(route: LabRouteContext, response: { statu return observation; } -async function withTotalTimeout(timeoutMs: number, run: (signal: AbortSignal) => Promise): Promise { +/** Normalize a pinned provider response into a Lab observation (trusted-route production path). */ +export function normalizeLabLiveTransportObservation( + route: LabRouteContext, + response: { status: number; body: string }, +): NormalizedObservation { + return normalizeTransportObservation(route, response); +} + +async function withTotalTimeout( + timeoutMs: number, + run: (signal: AbortSignal) => Promise, + cancelSignal?: AbortSignal, +): Promise { + if (cancelSignal?.aborted) throw cancelSignal.reason ?? new TransportError("harness_failure", "cancelled"); const controller = new AbortController(); + const abortFromCancel = () => controller.abort(cancelSignal?.reason ?? new TransportError("harness_failure", "cancelled")); + if (cancelSignal) cancelSignal.addEventListener("abort", abortFromCancel, { once: true }); const timer = setTimeout(() => controller.abort(new TransportError("total_timeout", "live scenario total timeout")), timeoutMs); try { return await Promise.race([ run(controller.signal), new Promise((_, reject) => controller.signal.addEventListener("abort", () => reject(controller.signal.reason), { once: true })), ]); - } finally { clearTimeout(timer); } + } finally { + clearTimeout(timer); + if (cancelSignal) cancelSignal.removeEventListener("abort", abortFromCancel); + } } export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabRouteContext, opts: LiveExecutorOptions = {}): Promise { @@ -266,13 +291,13 @@ export async function runLiveScenario(caseRecord: CaseRecord, routeContext: LabR if (!isTrustedLabRouteExecutor(opts.routeExecutor)) throw new TransportError("untrusted_route_executor", "untrusted route executor capability"); executionAuthority = "trusted_route"; trustedExecutionStarted = true; - observation = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.routeExecutor!.execute({ routeContext, destination, routeSubject: routeSubject!, scenarioId: activeCase.id, initiatingRequest: activeCase.initiatingRequest?.bytesUtf8, limits, signal, environment })); + observation = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.routeExecutor!.execute({ routeContext, destination, routeSubject: routeSubject!, scenarioId: activeCase.id, initiatingRequest: activeCase.initiatingRequest?.bytesUtf8, limits, signal, environment }), opts.cancelSignal); } else if (opts.transport && activeCase.initiatingRequest) { executionAuthority = "test_transport"; const body = activeCase.initiatingRequest.bytesUtf8; const inputBytes = new TextEncoder().encode(body).byteLength; enforceSandboxLimits(state, limits, { requests: 1, inputBytes }); - const response = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.transport!.request({ method: "POST", path: pathForProtocol(routeContext.upstreamProtocol), body, signal })); + const response = await withTotalTimeout(limits.totalTimeoutMs, (signal) => opts.transport!.request({ method: "POST", path: pathForProtocol(routeContext.upstreamProtocol), body, signal }), opts.cancelSignal); if (response.status === 401 || response.status === 403) throw new TransportError("auth_blocked", `HTTP ${response.status}`); if (response.status === 429) throw new TransportError("quota_blocked", "HTTP 429"); if (response.status === 451) throw new TransportError("region_blocked", "HTTP 451"); diff --git a/src/lib/lab-live-route-production.ts b/src/lib/lab-live-route-production.ts new file mode 100644 index 0000000000..1a4fa18f7b --- /dev/null +++ b/src/lib/lab-live-route-production.ts @@ -0,0 +1,112 @@ +/** + * Production host integration for CL-03 trusted exact-route execution. + * + * Issues a host-recognized `TrustedLabRouteExecutor` that routes provider traffic + * through the CL-03 credential lease, pinned transport, and observation normalization + * pipeline. Secret material stays outside Lab modules. + * + * @internal host integration only + */ +import { resolveEnvValue } from "../config"; +import { getValidAccessTokenSnapshot } from "../oauth"; +import type { OcxConfig } from "../types"; +import { createCredentialLease } from "../lab/live/credential-lease"; +import { + liveUpstreamRequestPath, + normalizeLabLiveTransportObservation, +} from "../lab/live/executor"; +import { createPinnedTransport } from "../lab/live/transport"; +import { TransportError } from "../lab/live/transport"; +import type { LabRouteContext } from "../lab/live/types"; +import { createLabAuthorizedPinnedSender } from "./lab-live-pinned-sender"; +import { createHostIssuedLabRouteExecutor } from "./lab-live-host"; +import type { TrustedLabRouteExecutor } from "../lab/live/types"; + +const CREDENTIAL_HEADER = /(authorization|api[-_]?key|token|secret|credential|cookie)/i; + +export interface ProductionLabRouteExecutorDeps { + configDir?: string; + loadConfig: () => OcxConfig; +} + +async function buildLabProviderAuthHeaders( + routeContext: LabRouteContext, + config: OcxConfig, +): Promise> { + const provider = config.providers?.[routeContext.providerId]; + if (!provider || provider.disabled === true) { + throw new TransportError("auth_blocked", "provider unavailable"); + } + if (provider.authMode === "forward") { + throw new TransportError("auth_blocked", "forward auth unsupported for lab probes"); + } + if (provider.authMode === "local") { + throw new TransportError("auth_blocked", "local provider unsupported for lab probes"); + } + + const headers: Record = { + Accept: "application/json", + "Content-Type": "application/json", + }; + + if (provider.authMode === "oauth") { + try { + const snapshot = await getValidAccessTokenSnapshot(routeContext.providerId); + headers.Authorization = `Bearer ${snapshot.accessToken}`; + } catch { + throw new TransportError("auth_blocked", "oauth unavailable"); + } + } else { + const apiKey = resolveEnvValue(provider.apiKey)?.trim(); + if (!apiKey) throw new TransportError("auth_blocked", "missing api key"); + if (provider.adapter === "anthropic" && provider.apiKeyTransport === "x-api-key") { + headers["x-api-key"] = apiKey; + } else { + headers.Authorization = `Bearer ${apiKey}`; + } + } + + if (provider.headers) { + for (const [name, value] of Object.entries(provider.headers)) { + if (CREDENTIAL_HEADER.test(name)) continue; + const resolved = resolveEnvValue(value); + headers[name] = typeof resolved === "string" ? resolved : value; + } + } + + return headers; +} + +/** Create the process-local trusted CL-03 route executor for Lab automation dispatch. */ +export function createProductionLabRouteExecutor( + deps: ProductionLabRouteExecutorDeps, +): TrustedLabRouteExecutor { + return createHostIssuedLabRouteExecutor(async (input) => { + const config = deps.loadConfig(); + const lease = createCredentialLease({ + destination: input.destination, + transportId: "lab-automation", + budget: input.limits.maxRequests, + }); + const authHeaders = await buildLabProviderAuthHeaders(input.routeContext, config); + const sender = createLabAuthorizedPinnedSender(async () => authHeaders); + const transport = createPinnedTransport({ + destination: input.destination, + lease, + sender, + limits: input.limits, + transportId: "lab-automation", + }); + if (!input.initiatingRequest) { + throw new TransportError("harness_failure", "live scenario missing initiating request"); + } + const path = liveUpstreamRequestPath(input.routeContext.upstreamProtocol); + const response = await transport.request({ + method: "POST", + path, + body: input.initiatingRequest, + signal: input.signal, + }); + return normalizeLabLiveTransportObservation(input.routeContext, response); + }); +} diff --git a/src/server/index.ts b/src/server/index.ts index ed761bb401..8029e24d45 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -16,6 +16,7 @@ import { armClaudeCodeBaseline, loadConfig, saveConfig, + getConfigDir, websocketsEnabled, } from "../config"; import { reconcileOAuthProviders } from "../oauth"; @@ -41,6 +42,12 @@ import { registerDefaultAppOwnedObservedBuffers, } from "../lib/app-owned-memory-stores"; import { acquireServerBackgroundLifecycle } from "./background-lifecycle"; +import { + setLabAutomationDispatchDeps, + startLabAutomationScheduler, +} from "../lab/automation/orchestrator"; +import { loadLabAutomationPolicy } from "../lab/automation/persistence"; +import { createProductionLabRouteExecutor } from "../lib/lab-live-route-production"; import { runOpenAiTierStartupMigration } from "../providers/openai-tier-startup"; import { runAlibabaRegionStartupMigration } from "../providers/alibaba-region-startup"; import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../providers/openai-tiers"; @@ -1659,5 +1666,19 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server config, + }); + setLabAutomationDispatchDeps({ + configDir: labConfigDir, + loadConfig: () => config, + routeExecutor: productionLabRouteExecutor, + }); + if (loadLabAutomationPolicy(labConfigDir).enabled) { + startLabAutomationScheduler(labConfigDir); + } + return server; } diff --git a/src/server/lifecycle.ts b/src/server/lifecycle.ts index 070b58af81..9afb18e259 100644 --- a/src/server/lifecycle.ts +++ b/src/server/lifecycle.ts @@ -7,6 +7,7 @@ import { } from "../storage/policy-job"; import { abortRestoreTrashJobAsync } from "../storage/restore-job"; import { stopStorageCleanupScheduler } from "../storage/policy-scheduler"; +import { stopLabAutomationScheduler, requestLabAutomationShutdown } from "../lab/automation/orchestrator"; import { stopStateStoreSweeper } from "../lib/state-store-sweeper"; import { cancelQueuedStorageWorkerSpawns, @@ -451,6 +452,8 @@ export async function drainAndShutdown( // Abort each job independently so one wedged join cannot skip the other, // then drain leftovers; failures must not prevent `server.stop`. stopStorageCleanupScheduler(); + requestLabAutomationShutdown(); + stopLabAutomationScheduler(); stopStateStoreSweeper(); // The overlay reconciler is owner-scoped: the startServer stop override // releases THIS server's lease through runListenerShutdown → diff --git a/src/server/management-api.ts b/src/server/management-api.ts index 372e738e76..bff89ccead 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -69,6 +69,7 @@ import { handleOauthAccountRoutes } from "./management/oauth-account-routes"; import { handleComboRoutes } from "./management/combo-routes"; import { handleSystemRoutes } from "./management/system-routes"; import { handleLabRoutes } from "./management/lab-routes"; +import { handleLabAutomationRoutes } from "./management/lab-automation-routes"; import { handleSidebarRoutes } from "./management/sidebar-routes"; import { handleIntegrationRoutes } from "./management/integration-routes"; import { handleNativeIntegrationRoutes } from "./management/native-integration-routes"; @@ -188,6 +189,7 @@ export async function handleManagementAPI( ?? (await handleOauthAccountRoutes(ctx)) ?? (await handleComboRoutes(ctx)) ?? (await handleSystemRoutes(ctx)) + ?? (await handleLabAutomationRoutes(ctx)) ?? (await handleLabRoutes(ctx)) ?? (await handleSidebarRoutes(ctx)); } catch (error) { diff --git a/src/server/management/lab-automation-routes.ts b/src/server/management/lab-automation-routes.ts new file mode 100644 index 0000000000..9cb73b48be --- /dev/null +++ b/src/server/management/lab-automation-routes.ts @@ -0,0 +1,168 @@ +/** + * CL-08 Compatibility Lab automation management API. + * + * - GET /api/lab/automation + * - PUT /api/lab/automation + * - GET /api/lab/automation/runs + * - POST /api/lab/automation/run + * - POST /api/lab/automation/runs/:id/cancel + * + * Read endpoints never trigger scheduler ticks or evidence collection. + */ + +import { readConfigDiagnostics, getConfigDir } from "../../config"; +import { + buildLabAutomationStatus, + cancelLabAutomationRun, + enqueueManualLabRun, + startLabAutomationScheduler, + stopLabAutomationScheduler, +} from "../../lab/automation/orchestrator"; +import { planManualLabRun } from "../../lab/automation/planner"; +import { + loadLabAutomationPolicy, + loadLabAutomationState, + saveLabAutomationPolicy, + saveLabAutomationRoutes, +} from "../../lab/automation/persistence"; +import { normalizeLabAutomationPolicyV1 } from "../../lab/automation/policy"; +import { listLabAutomationRuns } from "../../lab/automation/runs-query"; +import type { LabAutomationLayer, LabAutomationPolicyV1, LabAutomationRoutesV1 } from "../../lab/automation/types"; +import { LabAutomationError } from "../../lab/automation/types"; +import { jsonResponse } from "../auth-cors"; +import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; +import type { ManagementContext } from "./context"; +import { isPlainRecord } from "./shared"; + +const AUTOMATION_LAYERS: readonly LabAutomationLayer[] = [ + "protocol_conformance", + "live_route_compatibility", + "task_effectiveness", +]; + +function automationErrorResponse(code: string, message: string, status: number, ctx: ManagementContext): Response { + return jsonResponse({ error: { code, message } }, status, ctx.req, ctx.config); +} + +function parseLimit(raw: string | null, ctx: ManagementContext): number | Response { + if (raw === null) return 50; + const trimmed = raw.trim(); + if (trimmed.length === 0) return 50; + const value = Number(trimmed); + if (!Number.isInteger(value) || value < 1 || value > 100) { + return automationErrorResponse("invalid_limit", "limit must be an integer from 1 to 100", 400, ctx); + } + return value; +} + +function applySchedulerPolicy(policy: LabAutomationPolicyV1, configDir?: string): void { + if (policy.enabled) { + startLabAutomationScheduler(configDir); + } else { + stopLabAutomationScheduler(); + } +} + +export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise { + const { url, req, config } = ctx; + if (!url.pathname.startsWith("/api/lab/automation")) return null; + + const configDir = getConfigDir(); + + if (url.pathname === "/api/lab/automation" && req.method === "GET") { + return jsonResponse(buildLabAutomationStatus(configDir), 200, req, config); + } + + if (url.pathname === "/api/lab/automation/runs" && req.method === "GET") { + const limit = parseLimit(url.searchParams.get("limit"), ctx); + if (limit instanceof Response) return limit; + const cursor = url.searchParams.get("cursor")?.trim() || undefined; + const status = buildLabAutomationStatus(configDir); + const runsPage = listLabAutomationRuns(loadLabAutomationState(configDir), limit, cursor); + return jsonResponse({ + runs: runsPage.items, + hasMore: runsPage.hasMore, + ...(runsPage.nextCursor ? { nextCursor: runsPage.nextCursor } : {}), + counters: status.counters, + }, 200, req, config); + } + + const cancelMatch = url.pathname.match(/^\/api\/lab\/automation\/runs\/([^/]+)\/cancel$/); + if (cancelMatch && req.method === "POST") { + const runId = decodeURIComponent(cancelMatch[1]!); + const cancelled = cancelLabAutomationRun(runId, configDir); + if (!cancelled) return automationErrorResponse("not_found", "unknown or non-cancellable run", 404, ctx); + return jsonResponse({ cancelled: true, runId }, 200, req, config); + } + + if (url.pathname === "/api/lab/automation/run" && req.method === "POST") { + try { + const body = await readManagementJsonBody(req); + if (!isPlainRecord(body)) return automationErrorResponse("invalid_body", "body must be an object", 400, ctx); + const evidenceLayer = body.evidenceLayer; + const scenarioId = body.scenarioId; + if (typeof evidenceLayer !== "string" || !AUTOMATION_LAYERS.includes(evidenceLayer as LabAutomationLayer)) { + return automationErrorResponse("invalid_layer", "evidenceLayer must be a supported automation layer", 400, ctx); + } + if (typeof scenarioId !== "string" || scenarioId.length === 0) { + return automationErrorResponse("invalid_scenario", "scenarioId is required", 400, ctx); + } + let ocxConfig: import("../../types").OcxConfig | undefined; + try { + ocxConfig = readConfigDiagnostics().config; + } catch { + ocxConfig = config; + } + const planned = planManualLabRun({ + evidenceLayer: evidenceLayer as LabAutomationLayer, + scenarioId, + providerName: typeof body.providerName === "string" ? body.providerName : undefined, + modelId: typeof body.modelId === "string" ? body.modelId : undefined, + config: ocxConfig, + configDir, + }); + const record = await enqueueManualLabRun(planned, configDir); + if (!record) return automationErrorResponse("enqueue_failed", "manual run could not be enqueued", 500, ctx); + return jsonResponse({ run: record, trigger: "manual" }, 202, req, config); + } catch (error) { + rethrowManagementBodyTooLarge(error); + if (error instanceof LabAutomationError) { + return automationErrorResponse(error.code, error.message, 400, ctx); + } + return automationErrorResponse("server_error", "manual run failed", 500, ctx); + } + } + + if (url.pathname === "/api/lab/automation" && req.method === "PUT") { + try { + const body = await readManagementJsonBody(req); + if (!isPlainRecord(body)) return automationErrorResponse("invalid_body", "body must be an object", 400, ctx); + let policy = loadLabAutomationPolicy(configDir); + if (body.policy !== undefined) { + policy = normalizeLabAutomationPolicyV1({ ...policy, ...body.policy as Record }); + saveLabAutomationPolicy(policy, configDir); + } + if (body.routes !== undefined) { + if (!isPlainRecord(body.routes)) return automationErrorResponse("invalid_routes", "routes must be an object", 400, ctx); + const routesPayload = body.routes as Record; + if (routesPayload.schemaVersion !== 1 || !Array.isArray(routesPayload.routes)) { + return automationErrorResponse("invalid_routes", "routes must include schemaVersion 1 and routes array", 400, ctx); + } + saveLabAutomationRoutes({ + schemaVersion: 1, + routes: routesPayload.routes as LabAutomationRoutesV1["routes"], + }, configDir); + } + applySchedulerPolicy(policy, configDir); + return jsonResponse(buildLabAutomationStatus(configDir), 200, req, config); + } catch (error) { + rethrowManagementBodyTooLarge(error); + if (error instanceof LabAutomationError) { + return automationErrorResponse(error.code, error.message, 400, ctx); + } + return automationErrorResponse("server_error", "automation policy update failed", 500, ctx); + } + } + + return automationErrorResponse("not_found", "unknown resource", 404, ctx); +} From 3d6b7f2aaf0cb85fc8090dff366aa8c149201e2f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:42:58 +0200 Subject: [PATCH 04/67] test(lab): harden automation authority and lifecycle Cover trusted dispatch, missing executor fail-closed, budget and shutdown behavior, and management API authority boundaries. --- tests/lab-automation.test.ts | 699 +++++++++++++++++++++++++++++++++++ 1 file changed, 699 insertions(+) create mode 100644 tests/lab-automation.test.ts diff --git a/tests/lab-automation.test.ts b/tests/lab-automation.test.ts new file mode 100644 index 0000000000..bbaf3bcf47 --- /dev/null +++ b/tests/lab-automation.test.ts @@ -0,0 +1,699 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import type { OcxConfig } from "../src/types"; +import { defaultLabAutomationPolicyV1 } from "../src/lab/automation/policy"; +import { + loadLabAutomationPolicy, + loadLabAutomationState, + saveLabAutomationPolicy, + saveLabAutomationRoutes, + saveLabAutomationState, +} from "../src/lab/automation/persistence"; +import { planLabAutomationRuns } from "../src/lab/automation/planner"; +import { enqueuePlannedRuns, selectDispatchableRuns } from "../src/lab/automation/queue"; +import { recoverLabAutomationState } from "../src/lab/automation/recovery"; +import { + buildLabAutomationStatus, + cancelLabAutomationRun, + runLabAutomationTick, + setLabAutomationDispatchDeps, + stopLabAutomationScheduler, + requestLabAutomationShutdown, + resetLabAutomationSchedulerStateForTests, +} from "../src/lab/automation/orchestrator"; +import { dispatchLabAutomationRun } from "../src/lab/automation/dispatch"; +import { LAB_AUTOMATION_HARD_MAX } from "../src/lab/automation/constants"; +import { LabAutomationError } from "../src/lab/automation/types"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; +import { isTrustedLabRouteExecutor } from "../src/lib/lab-live-execution-authority"; +import { createProductionLabRouteExecutor } from "../src/lib/lab-live-route-production"; +import { readInstallationSalt } from "../src/lab/subject/installation-salt"; +import { loadLiveCaseAuthority } from "../src/lab/live/manifest"; +import { TransportError } from "../src/lab/live/transport"; +import type { NormalizedObservation } from "../src/lab/conformance/types"; +import { + resetCompatibilityVersionCacheForTests, + setCompatibilityVersionOverrideForTests, +} from "../src/routing/compatibility/version"; + +const COMPAT_VERSION = "f".repeat(64); +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-cl08-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +afterEach(() => { + requestLabAutomationShutdown(); + stopLabAutomationScheduler(); + resetLabAutomationSchedulerStateForTests(); + setLabAutomationDispatchDeps({}); + resetCompatibilityVersionCacheForTests(); + for (const dir of HOMES.splice(0)) { + try { + rmSync(dir, { recursive: true, force: true }); + } catch { + /* ignore */ + } + } + delete process.env.OPENCODEX_HOME; +}); + +function withHome(fn: (home: string) => T): T { + const home = tempHome(); + process.env.OPENCODEX_HOME = home; + return fn(home); +} + +function prepareLiveRouteHome(home: string): void { + readInstallationSalt(home); + setCompatibilityVersionOverrideForTests(COMPAT_VERSION); +} + +function fixtureDnsResolve() { + return async () => [{ address: "93.184.216.34", family: 4 as const }]; +} + +function liveAutomationDeps( + home: string, + config: OcxConfig, + routeExecutor?: ReturnType, +) { + return { + configDir: home, + loadConfig: () => config, + resolve: fixtureDnsResolve(), + ...(routeExecutor ? { routeExecutor } : {}), + }; +} + +function enabledProtocolPolicy(): ReturnType { + return { + ...defaultLabAutomationPolicyV1(), + enabled: true, + layers: { + protocolConformance: true, + liveRouteCompatibility: false, + taskEffectiveness: false, + }, + maxConcurrentRuns: 2, + maxRunsPerHour: 20, + refreshBeforeStaleMs: 60_000, + }; +} + +describe("CL-08 lab automation", () => { + test("default policy schedules nothing", () => { + withHome((home) => { + const policy = loadLabAutomationPolicy(home); + expect(policy.enabled).toBe(false); + const planned = planLabAutomationRuns({ + policy, + routes: { schemaVersion: 1, routes: [] }, + state: loadLabAutomationState(home), + now: Date.now(), + configDir: home, + }); + expect(planned).toEqual([]); + }); + }); + + test("planner schedules protocol work when evidence is missing", () => { + withHome((home) => { + const policy = enabledProtocolPolicy(); + const planned = planLabAutomationRuns({ + policy, + routes: { schemaVersion: 1, routes: [] }, + state: loadLabAutomationState(home), + now: Date.now(), + configDir: home, + }); + expect(planned.length).toBeGreaterThan(0); + expect(planned.every((row) => row.evidenceLayer === "protocol_conformance")).toBe(true); + }); + }); + + test("dedup prevents duplicate active run keys", () => { + withHome((home) => { + const policy = enabledProtocolPolicy(); + const input = { + policy, + routes: { schemaVersion: 1, routes: [] }, + state: loadLabAutomationState(home), + now: Date.now(), + configDir: home, + }; + const planned = planLabAutomationRuns(input); + let state = enqueuePlannedRuns(input.state, planned.slice(0, 1), "scheduled", Date.now()); + const again = planLabAutomationRuns({ ...input, state }); + expect(again.some((row) => row.runKey === planned[0]!.runKey)).toBe(false); + state = enqueuePlannedRuns(state, planned.slice(0, 1), "scheduled", Date.now()); + expect(state.runs.filter((row) => row.runKey === planned[0]!.runKey).length).toBe(1); + }); + }); + + test("hourly run budget blocks planning", () => { + withHome((home) => { + const policy = enabledProtocolPolicy(); + const state = { + ...loadLabAutomationState(home), + runsThisHour: policy.maxRunsPerHour, + }; + const planned = planLabAutomationRuns({ + policy, + routes: { schemaVersion: 1, routes: [] }, + state, + now: Date.now(), + configDir: home, + }); + expect(planned).toEqual([]); + }); + }); + + test("policy hard maximum cannot be exceeded via normalization", () => { + withHome((home) => { + expect(() => { + saveLabAutomationPolicy({ + ...defaultLabAutomationPolicyV1(), + enabled: true, + maxRunsPerHour: LAB_AUTOMATION_HARD_MAX.maxRunsPerHour + 1, + } as never, home); + }).toThrow(LabAutomationError); + }); + }); + + test("recovery marks abandoned running runs", () => { + withHome((home) => { + const policy = enabledProtocolPolicy(); + const now = Date.now(); + saveLabAutomationState({ + schemaVersion: 1, + runs: [{ + runId: "run-1", + runKey: "key-1", + state: "running", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "digest", + scenarioId: "responses-core.protocol.request-shape", + scenarioVersion: "1", + scenarioManifestDigest: "digest", + subjectId: "subject", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now, + updatedAt: now, + }], + budgetWindowStartedAt: now, + runsThisHour: 0, + liveRequestsThisHour: 0, + cooldownUntilByKey: {}, + }, home); + const state = loadLabAutomationState(home); + const recovered = recoverLabAutomationState(policy, state, now + 1); + expect(recovered.runs[0]?.state).toBe("abandoned"); + expect(recovered.cooldownUntilByKey["key-1"]).toBeGreaterThan(now); + }); + }); + + test("queued cancellation transitions to cancelled", () => { + withHome((home) => { + const now = Date.now(); + saveLabAutomationState({ + schemaVersion: 1, + runs: [{ + runId: "queued-1", + runKey: "key-q", + state: "queued", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "digest", + scenarioId: "responses-core.protocol.request-shape", + scenarioVersion: "1", + scenarioManifestDigest: "digest", + subjectId: "subject", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now, + updatedAt: now, + }], + budgetWindowStartedAt: now, + runsThisHour: 0, + liveRequestsThisHour: 0, + cooldownUntilByKey: {}, + }, home); + expect(cancelLabAutomationRun("queued-1", home)).toBe(true); + const state = loadLabAutomationState(home); + expect(state.runs[0]?.state).toBe("cancelled"); + }); + }); + + test("dispatch honours abort signal without persisting", async () => { + await withHome(async (home) => { + const controller = new AbortController(); + controller.abort(); + const now = Date.now(); + await expect(dispatchLabAutomationRun({ + runId: "abort-1", + runKey: "key", + state: "running", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "digest", + scenarioId: "responses-core.protocol.request-shape", + scenarioVersion: "1", + scenarioManifestDigest: "digest", + subjectId: "subject", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "manual", + createdAt: now, + updatedAt: now, + }, { configDir: home, abortSignal: controller.signal })).rejects.toThrow(LabAutomationError); + expect(existsSync(join(home, "lab", "ledger"))).toBe(false); + }); + }); + + test("automation persistence stores policy refs not credentials", () => { + withHome((home) => { + saveLabAutomationPolicy(enabledProtocolPolicy(), home); + saveLabAutomationRoutes({ + schemaVersion: 1, + routes: [{ providerName: "openai", modelId: "gpt-4.1" }], + }, home); + const policyBlob = readFileSync(join(home, "lab", "automation-policy.json"), "utf8"); + const routesBlob = readFileSync(join(home, "lab", "automation-routes.json"), "utf8"); + expect(policyBlob.includes("apiKey")).toBe(false); + expect(policyBlob.includes("Authorization")).toBe(false); + expect(routesBlob.includes("http://")).toBe(false); + expect(routesBlob.includes("sk-")).toBe(false); + }); + }); + + test("concurrency limits cap dispatch selection", () => { + const policy = enabledProtocolPolicy(); + policy.maxConcurrentRuns = 1; + const now = Date.now(); + const state = { + schemaVersion: 1, + runs: [ + { + runId: "r1", runKey: "k1", state: "running" as const, evidenceLayer: "protocol_conformance" as const, + suiteId: "s", suiteVersion: "1", suiteManifestDigest: "d", scenarioId: "a", scenarioVersion: "1", + scenarioManifestDigest: "d", subjectId: "sub", reason: "missing" as const, priority: 0, eligibleAt: now, + trigger: "scheduled" as const, createdAt: now, updatedAt: now, + }, + { + runId: "r2", runKey: "k2", state: "queued" as const, evidenceLayer: "protocol_conformance" as const, + suiteId: "s", suiteVersion: "1", suiteManifestDigest: "d", scenarioId: "b", scenarioVersion: "1", + scenarioManifestDigest: "d", subjectId: "sub", reason: "missing" as const, priority: 0, eligibleAt: now, + trigger: "scheduled" as const, createdAt: now, updatedAt: now, + }, + ], + budgetWindowStartedAt: now, + runsThisHour: 0, + liveRequestsThisHour: 0, + cooldownUntilByKey: {}, + }; + const selected = selectDispatchableRuns(policy, state, now); + expect(selected.length).toBe(0); + }); + + test("task background execution stays disabled by default", () => { + const policy = { + ...enabledProtocolPolicy(), + layers: { protocolConformance: false, liveRouteCompatibility: false, taskEffectiveness: true }, + taskEffectivenessBackgroundEnabled: false, + }; + const planned = planLabAutomationRuns({ + policy, + routes: { schemaVersion: 1, routes: [] }, + state: loadLabAutomationState(tempHome()), + now: Date.now(), + }); + expect(planned).toEqual([]); + }); + + test("GET lab automation API does not start scheduler", async () => { + await withHome(async (home) => { + const config = { providers: {} } as OcxConfig; + const req = new ManagementRequest("http://127.0.0.1/api/lab/automation"); + const res = await handleManagementAPI(req, new URL(req.url), config); + expect(res.status).toBe(200); + const status = buildLabAutomationStatus(home); + expect(status.schedulerRunning).toBe(false); + }); + }); + + test("production read paths do not reference automation scheduler", () => { + const files = [ + "src/lab/query/queries.ts", + "src/routing/profile.ts", + "src/routing/evaluator.ts", + "src/server/responses/core.ts", + ]; + for (const file of files) { + const src = readFileSync(join(process.cwd(), file), "utf8"); + expect(src.includes("runLabAutomationTick")).toBe(false); + expect(src.includes("startLabAutomationScheduler")).toBe(false); + expect(src.includes("planLabAutomationRuns")).toBe(false); + } + }); + + test("corrupt scheduler state fails closed on load", () => { + withHome((home) => { + const path = join(home, "lab"); + mkdirSync(path, { recursive: true }); + const statePath = join(path, "automation-state.json"); + writeBadState(statePath); + expect(() => loadLabAutomationState(home)).toThrow(LabAutomationError); + }); + }); + + test("production factory yields host-issued trusted executor", () => { + const executor = createProductionLabRouteExecutor({ + loadConfig: () => ({ providers: {} } as OcxConfig), + }); + expect(isTrustedLabRouteExecutor(executor)).toBe(true); + }); + + test("missing trusted executor fails closed without provider traffic", async () => { + await withHome(async (home) => { + prepareLiveRouteHome(home); + const config = fixtureProviderConfig(); + const now = Date.now(); + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + setLabAutomationDispatchDeps(liveAutomationDeps(home, config)); + const result = await dispatchLabAutomationRun({ + runId: "live-1", + runKey: "live-key", + state: "running", + evidenceLayer: "live_route_compatibility", + suiteId: scenario.suite, + suiteVersion: "1", + suiteManifestDigest: "digest", + scenarioId: scenario.id, + scenarioVersion: "1", + scenarioManifestDigest: "digest", + subjectId: "subject", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now, + updatedAt: now, + providerName: "fixture-provider", + modelId: "fixture-model", + }, { configDir: home, loadConfig: () => config }); + expect(result.terminalState).toBe("blocked"); + expect(result.terminalCode).toBe("route_ineligible"); + expect(result.liveRequest).toBe(false); + }); + }); + + test("trusted live dispatch invokes host executor exactly once", async () => { + await withHome(async (home) => { + prepareLiveRouteHome(home); + const config = fixtureProviderConfig(); + let invokes = 0; + const executor = createHostIssuedLabRouteExecutor(async () => { + invokes += 1; + return passObservation(); + }); + setLabAutomationDispatchDeps(liveAutomationDeps(home, config, executor)); + saveLabAutomationPolicy({ + ...enabledProtocolPolicy(), + layers: { protocolConformance: false, liveRouteCompatibility: true, taskEffectiveness: false }, + maxConcurrentLiveRuns: 1, + maxLiveRequestsPerHour: 6, + }, home); + saveLabAutomationRoutes({ + schemaVersion: 1, + routes: [{ providerName: "fixture-provider", modelId: "fixture-model" }], + }, home); + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const now = Date.now(); + saveLabAutomationState({ + schemaVersion: 1, + runs: [{ + runId: "live-run-1", + runKey: "live-run-key", + state: "queued", + evidenceLayer: "live_route_compatibility", + suiteId: scenario.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: "digest", + scenarioId: scenario.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: "digest", + subjectId: "pending", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now, + updatedAt: now, + providerName: "fixture-provider", + modelId: "fixture-model", + }], + budgetWindowStartedAt: now, + runsThisHour: 0, + liveRequestsThisHour: 0, + cooldownUntilByKey: {}, + }, home); + await runLabAutomationTick(home); + expect(invokes).toBe(1); + const state = loadLabAutomationState(home); + expect(state.runs[0]?.state === "completed" || state.runs[0]?.state === "failed").toBe(true); + }); + }); + + test("disabled automation never invokes trusted executor", async () => { + await withHome(async (home) => { + let invokes = 0; + setLabAutomationDispatchDeps(liveAutomationDeps(home, fixtureProviderConfig(), createHostIssuedLabRouteExecutor(async () => { + invokes += 1; + return passObservation(); + }))); + await runLabAutomationTick(home); + expect(invokes).toBe(0); + }); + }); + + test("live request budget blocks dispatch while leaving runs queued", async () => { + await withHome(async (home) => { + prepareLiveRouteHome(home); + const config = fixtureProviderConfig(); + let invokes = 0; + setLabAutomationDispatchDeps(liveAutomationDeps(home, config, createHostIssuedLabRouteExecutor(async () => { + invokes += 1; + return passObservation(); + }))); + const policy = { + ...enabledProtocolPolicy(), + enabled: true, + layers: { protocolConformance: false, liveRouteCompatibility: true, taskEffectiveness: false }, + maxLiveRequestsPerHour: 0, + }; + saveLabAutomationPolicy(policy, home); + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const now = Date.now(); + saveLabAutomationState({ + schemaVersion: 1, + runs: [{ + runId: "budget-live", + runKey: "budget-key", + state: "queued", + evidenceLayer: "live_route_compatibility", + suiteId: scenario.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: "digest", + scenarioId: scenario.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: "digest", + subjectId: "pending", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now, + updatedAt: now, + providerName: "fixture-provider", + modelId: "fixture-model", + }], + budgetWindowStartedAt: now, + runsThisHour: 0, + liveRequestsThisHour: 0, + cooldownUntilByKey: {}, + }, home); + await runLabAutomationTick(home); + expect(invokes).toBe(0); + expect(loadLabAutomationState(home).runs[0]?.state).toBe("queued"); + }); + }); + + test("authentication blocked from trusted executor stays orchestration blocked", async () => { + await withHome(async (home) => { + prepareLiveRouteHome(home); + const config = fixtureProviderConfig(); + const executor = createHostIssuedLabRouteExecutor(async () => { + throw new TransportError("auth_blocked", "HTTP 401"); + }); + setLabAutomationDispatchDeps(liveAutomationDeps(home, config, executor)); + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const now = Date.now(); + const result = await dispatchLabAutomationRun({ + runId: "auth-block", + runKey: "auth-key", + state: "running", + evidenceLayer: "live_route_compatibility", + suiteId: scenario.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: "digest", + scenarioId: scenario.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: "digest", + subjectId: "subject", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now, + updatedAt: now, + providerName: "fixture-provider", + modelId: "fixture-model", + }, liveAutomationDeps(home, config, executor)); + expect(result.terminalState).toBe("blocked"); + expect(result.terminalCode).toBe("auth_blocked"); + expect(result.liveRequest).toBe(true); + }); + }); + + test("management PUT cannot inject trusted executor authority", async () => { + await withHome(async (home) => { + const config = fixtureProviderConfig(); + const fakeExecutor = { execute: async () => passObservation(), enforcedBoundaries: [] }; + const req = new ManagementRequest("http://127.0.0.1/api/lab/automation", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + policy: { enabled: true }, + routeExecutor: fakeExecutor, + }), + }); + const res = await handleManagementAPI(req, new URL(req.url), config); + expect(res?.status).toBe(200); + const production = createProductionLabRouteExecutor({ loadConfig: () => config }); + expect(isTrustedLabRouteExecutor(production)).toBe(true); + expect(isTrustedLabRouteExecutor(fakeExecutor)).toBe(false); + }); + }); + + test("shutdown stops new trusted dispatch", async () => { + await withHome(async (home) => { + prepareLiveRouteHome(home); + const config = fixtureProviderConfig(); + let invokes = 0; + setLabAutomationDispatchDeps(liveAutomationDeps(home, config, createHostIssuedLabRouteExecutor(async () => { + invokes += 1; + return passObservation(); + }))); + saveLabAutomationPolicy({ + ...enabledProtocolPolicy(), + layers: { protocolConformance: false, liveRouteCompatibility: true, taskEffectiveness: false }, + }, home); + const authority = loadLiveCaseAuthority(); + const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; + const now = Date.now(); + saveLabAutomationState({ + schemaVersion: 1, + runs: [{ + runId: "shutdown-live", + runKey: "shutdown-key", + state: "queued", + evidenceLayer: "live_route_compatibility", + suiteId: scenario.suite, + suiteVersion: authority.manifestDefaults.suiteVersion, + suiteManifestDigest: "digest", + scenarioId: scenario.id, + scenarioVersion: authority.manifestDefaults.version, + scenarioManifestDigest: "digest", + subjectId: "pending", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now, + updatedAt: now, + providerName: "fixture-provider", + modelId: "fixture-model", + }], + budgetWindowStartedAt: now, + runsThisHour: 0, + liveRequestsThisHour: 0, + cooldownUntilByKey: {}, + }, home); + requestLabAutomationShutdown(); + await runLabAutomationTick(home); + expect(invokes).toBe(0); + }); + }); +}); + +function writeBadState(path: string): void { + writeFileSync(path, JSON.stringify({ schemaVersion: 1, runs: [{ bad: true }] }), "utf8"); +} + +function fixtureProviderConfig(): OcxConfig { + return { + providers: { + "fixture-provider": { + adapter: "openai-responses", + baseUrl: "https://api.example.com/v1", + apiKey: "sk-fixture-test-key", + models: ["fixture-model"], + defaultModel: "fixture-model", + }, + }, + } as OcxConfig; +} + +function passObservation(): NormalizedObservation { + return { + client: { + request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, + response: { + status: 200, + headers: {}, + json: { status: "completed", output: [{ type: "message", content: [{ type: "output_text", text: "OK" }] }] }, + events: [], + toolCalls: [], + mcpCalls: [], + terminal: "completed", + normalizedText: "OK", + }, + }, + upstream: { requests: [], responses: [] }, + process: { exitCode: null }, + verifiers: {}, + }; +} From e4ba266ad727ae4b1e06eb7ce486d2a56fee3ca0 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:08:04 +0200 Subject: [PATCH 05/67] test(lab): capture CL-08 review regressions --- .../lab-automation-review-regressions.test.ts | 460 ++++++++++++++++++ 1 file changed, 460 insertions(+) create mode 100644 tests/lab-automation-review-regressions.test.ts diff --git a/tests/lab-automation-review-regressions.test.ts b/tests/lab-automation-review-regressions.test.ts new file mode 100644 index 0000000000..8f8c630d81 --- /dev/null +++ b/tests/lab-automation-review-regressions.test.ts @@ -0,0 +1,460 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { OcxConfig } from "../src/types"; +import { defaultLabAutomationPolicyV1, normalizeLabAutomationPolicyV1 } from "../src/lab/automation/policy"; +import { + defaultLabAutomationStateV1, + loadLabAutomationState, + saveLabAutomationPolicy, + saveLabAutomationRoutes, + saveLabAutomationState, +} from "../src/lab/automation/persistence"; +import { planLabAutomationRuns, planManualLabRun } from "../src/lab/automation/planner"; +import { enqueuePlannedRuns } from "../src/lab/automation/queue"; +import { rollBudgetWindow, runBudgetRemaining } from "../src/lab/automation/budgets"; +import { + enqueueManualLabRun, + requestLabAutomationShutdown, + resetLabAutomationSchedulerStateForTests, + runLabAutomationTick, + setLabAutomationDispatchDeps, + stopLabAutomationScheduler, +} from "../src/lab/automation/orchestrator"; +import { dispatchLabAutomationRun } from "../src/lab/automation/dispatch"; +import type { + LabAutomationRunRecordV1, + LabAutomationStateV1, + PlannedLabRunV1, +} from "../src/lab/automation/types"; +import { LabAutomationError } from "../src/lab/automation/types"; +import { LAB_AUTOMATION_HARD_MAX } from "../src/lab/automation/constants"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; +import { readInstallationSalt } from "../src/lab/subject/installation-salt"; +import type { NormalizedObservation } from "../src/lab/conformance/types"; +import { + resetCompatibilityVersionCacheForTests, + setCompatibilityVersionOverrideForTests, +} from "../src/routing/compatibility/version"; + +const COMPAT_VERSION = "e".repeat(64); +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-cl08-review-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +function prepareHome(home: string): void { + process.env.OPENCODEX_HOME = home; + readInstallationSalt(home); + setCompatibilityVersionOverrideForTests(COMPAT_VERSION); +} + +function fixtureDnsResolve() { + return async () => [{ address: "93.184.216.34", family: 4 as const }]; +} + +function providerConfig(baseUrl = "https://api.example.com/v1", providerName = "fixture-provider"): OcxConfig { + return { + providers: { + [providerName]: { + adapter: "openai-responses", + baseUrl, + apiKey: "sk-fixture-test-key", + models: ["fixture-model"], + defaultModel: "fixture-model", + }, + }, + } as OcxConfig; +} + +function twoRouteConfig(): OcxConfig { + return { + providers: { + "route-a": { + adapter: "openai-responses", + baseUrl: "https://a.example.com/v1", + apiKey: "sk-fixture-a", + models: ["fixture-model"], + defaultModel: "fixture-model", + }, + "route-b": { + adapter: "openai-responses", + baseUrl: "https://b.example.com/v1", + apiKey: "sk-fixture-b", + models: ["fixture-model"], + defaultModel: "fixture-model", + }, + }, + } as OcxConfig; +} + +function passObservation(): NormalizedObservation { + return { + client: { + request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, + response: { + status: 200, + headers: {}, + json: { status: "completed", output: [{ type: "message", content: [{ type: "output_text", text: "OK" }] }] }, + events: [], + toolCalls: [], + mcpCalls: [], + terminal: "completed", + normalizedText: "OK", + }, + }, + upstream: { requests: [], responses: [] }, + process: { exitCode: null }, + verifiers: {}, + }; +} + +function recordFor(plan: PlannedLabRunV1, trigger: "scheduled" | "manual" = "scheduled"): LabAutomationRunRecordV1 { + const now = Date.now(); + return { + ...plan, + runId: `review-${Math.random().toString(16).slice(2)}`, + state: "queued", + trigger, + createdAt: now, + updatedAt: now, + }; +} + +function livePolicy() { + return { + ...defaultLabAutomationPolicyV1(), + enabled: true, + layers: { + protocolConformance: false, + liveRouteCompatibility: true, + taskEffectiveness: false, + }, + maxConcurrentRuns: 2, + maxConcurrentLiveRuns: 1, + maxConcurrentRunsPerRoute: 1, + maxRunsPerHour: 20, + maxLiveRequestsPerHour: 10, + failureCooldownMs: 1_000, + blockedCooldownMs: 60_000, + }; +} + +afterEach(() => { + requestLabAutomationShutdown(); + stopLabAutomationScheduler(); + resetLabAutomationSchedulerStateForTests(); + setLabAutomationDispatchDeps({}); + resetCompatibilityVersionCacheForTests(); + delete process.env.OPENCODEX_HOME; + for (const dir of HOMES.splice(0)) { + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } + } +}); + +describe("CL-08 independent review regressions", () => { + test("freshness is scoped to the exact route subject", async () => { + const home = tempHome(); + prepareHome(home); + const config = twoRouteConfig(); + const scenarioId = "responses-core.live.basic-turn"; + const executor = createHostIssuedLabRouteExecutor(async () => passObservation()); + const routeAPlan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId, + providerName: "route-a", + modelId: "fixture-model", + config, + configDir: home, + }); + const routeAResult = await dispatchLabAutomationRun( + { ...recordFor(routeAPlan, "manual"), state: "running" }, + { configDir: home, loadConfig: () => config, routeExecutor: executor, resolve: fixtureDnsResolve() }, + ); + expect(routeAResult.terminalState).toBe("completed"); + + const planned = planLabAutomationRuns({ + policy: livePolicy(), + routes: { + schemaVersion: 1, + routes: [ + { providerName: "route-a", modelId: "fixture-model" }, + { providerName: "route-b", modelId: "fixture-model" }, + ], + }, + state: defaultLabAutomationStateV1(), + now: Date.now(), + config, + configDir: home, + }); + expect(planned.some((row) => row.scenarioId === scenarioId && row.providerName === "route-a")).toBe(false); + expect(planned.some((row) => row.scenarioId === scenarioId && row.providerName === "route-b")).toBe(true); + }); + + test("dispatch rejects a queued live run whose exact route contract changed", async () => { + const home = tempHome(); + prepareHome(home); + const original = providerConfig("https://api.example.com/v1"); + const changed = providerConfig("https://changed.example.com/v1"); + const plan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId: "responses-core.live.basic-turn", + providerName: "fixture-provider", + modelId: "fixture-model", + config: original, + configDir: home, + }); + let invokes = 0; + const executor = createHostIssuedLabRouteExecutor(async () => { + invokes += 1; + return passObservation(); + }); + const result = await dispatchLabAutomationRun( + { ...recordFor(plan), state: "running" }, + { configDir: home, loadConfig: () => changed, routeExecutor: executor, resolve: fixtureDnsResolve() }, + ); + expect(invokes).toBe(0); + expect(result.terminalState).toBe("blocked"); + expect(result.terminalCode).toBe("run_contract_changed"); + expect(result.liveRequest).toBe(false); + }); + + test("manual protocol run executes while automation is disabled", async () => { + const home = tempHome(); + prepareHome(home); + saveLabAutomationPolicy(defaultLabAutomationPolicyV1(), home); + const plan = planManualLabRun({ + evidenceLayer: "protocol_conformance", + scenarioId: "responses-core.protocol.request-shape", + configDir: home, + }); + const result = await enqueueManualLabRun(plan, home); + expect(result).not.toBeNull(); + expect(result?.state).not.toBe("queued"); + expect(result?.state).not.toBe("running"); + }); + + test("disabling the live layer cancels previously queued scheduled live work", async () => { + const home = tempHome(); + prepareHome(home); + const config = providerConfig(); + const plan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId: "responses-core.live.basic-turn", + providerName: "fixture-provider", + modelId: "fixture-model", + config, + configDir: home, + }); + let state = defaultLabAutomationStateV1(); + state = enqueuePlannedRuns(state, [plan], "scheduled", Date.now()); + saveLabAutomationState(state, home); + saveLabAutomationPolicy({ + ...livePolicy(), + layers: { protocolConformance: true, liveRouteCompatibility: false, taskEffectiveness: false }, + }, home); + saveLabAutomationRoutes({ + schemaVersion: 1, + routes: [{ providerName: "fixture-provider", modelId: "fixture-model" }], + }, home); + let invokes = 0; + setLabAutomationDispatchDeps({ + configDir: home, + loadConfig: () => config, + resolve: fixtureDnsResolve(), + routeExecutor: createHostIssuedLabRouteExecutor(async () => { + invokes += 1; + return passObservation(); + }), + }); + await runLabAutomationTick(home); + const stored = loadLabAutomationState(home).runs.find((row) => row.runKey === plan.runKey); + expect(invokes).toBe(0); + expect(stored?.state).toBe("cancelled"); + expect(stored?.terminalCode).toBe("layer_disabled"); + }); + + test("queue saturation evicts terminal history and never exceeds queued-run ceiling", () => { + const now = Date.now(); + const terminalRuns: LabAutomationRunRecordV1[] = Array.from({ length: LAB_AUTOMATION_HARD_MAX.maxPersistedRuns }, (_, index) => ({ + runId: `old-${index}`, + runKey: `old-key-${index}`, + state: "completed", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "digest", + scenarioId: `old-${index}`, + scenarioVersion: "1", + scenarioManifestDigest: "digest", + subjectId: "subject", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now - 100_000 - index, + updatedAt: now - 100_000 - index, + completedAt: now - 100_000 - index, + })); + const base: LabAutomationStateV1 = { + ...defaultLabAutomationStateV1(now), + runs: terminalRuns, + }; + const plan: PlannedLabRunV1 = { + runKey: "new-key", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "digest-new", + scenarioId: "new-scenario", + scenarioVersion: "1", + scenarioManifestDigest: "digest-new", + subjectId: "new-subject", + reason: "missing", + priority: 0, + eligibleAt: now, + }; + const withNew = enqueuePlannedRuns(base, [plan], "scheduled", now); + expect(withNew.runs.some((row) => row.runKey === "new-key" && row.state === "queued")).toBe(true); + expect(withNew.runs.length).toBeLessThanOrEqual(LAB_AUTOMATION_HARD_MAX.maxPersistedRuns); + + let queued = defaultLabAutomationStateV1(now); + for (let index = 0; index < LAB_AUTOMATION_HARD_MAX.maxQueuedRuns + 10; index += 1) { + queued = enqueuePlannedRuns(queued, [{ ...plan, runKey: `queued-${index}`, scenarioId: `scenario-${index}` }], "scheduled", now); + } + expect(queued.runs.filter((row) => row.state === "queued").length).toBe(LAB_AUTOMATION_HARD_MAX.maxQueuedRuns); + }); + + test("policy, route, and state persistence reject unknown or unsafe fields", () => { + const home = tempHome(); + prepareHome(home); + expect(() => normalizeLabAutomationPolicyV1({ + ...defaultLabAutomationPolicyV1(), + unexpected: true, + })).toThrow(LabAutomationError); + expect(() => normalizeLabAutomationPolicyV1({ + ...defaultLabAutomationPolicyV1(), + layers: { ...defaultLabAutomationPolicyV1().layers, unexpected: true }, + })).toThrow(LabAutomationError); + expect(() => saveLabAutomationRoutes({ + schemaVersion: 1, + routes: [{ providerName: "fixture-provider", modelId: "fixture-model", baseUrl: "https://should-not-persist.example" } as never], + }, home)).toThrow(LabAutomationError); + expect(() => saveLabAutomationState({ + ...defaultLabAutomationStateV1(), + runsThisHour: -1, + }, home)).toThrow(LabAutomationError); + }); + + test("trusted live cancellation cannot persist compatibility evidence", async () => { + const home = tempHome(); + prepareHome(home); + const config = providerConfig(); + const plan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId: "responses-core.live.basic-turn", + providerName: "fixture-provider", + modelId: "fixture-model", + config, + configDir: home, + }); + const controller = new AbortController(); + const executor = createHostIssuedLabRouteExecutor(async () => { + controller.abort(new Error("cancelled")); + await Promise.resolve(); + return passObservation(); + }); + await expect(dispatchLabAutomationRun( + { ...recordFor(plan, "manual"), state: "running" }, + { + configDir: home, + loadConfig: () => config, + routeExecutor: executor, + resolve: fixtureDnsResolve(), + abortSignal: controller.signal, + }, + )).rejects.toThrow(LabAutomationError); + expect(existsSync(join(home, "lab", "ledger"))).toBe(false); + }); + + test("rolling run budget retains attempts from the trailing hour after the old window boundary", () => { + const now = Date.now(); + const policy = { ...defaultLabAutomationPolicyV1(), maxRunsPerHour: 5 }; + const run = (id: string, startedAt: number): LabAutomationRunRecordV1 => ({ + runId: id, + runKey: id, + state: "completed", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "digest", + scenarioId: id, + scenarioVersion: "1", + scenarioManifestDigest: "digest", + subjectId: "subject", + reason: "missing", + priority: 0, + eligibleAt: startedAt, + trigger: "scheduled", + createdAt: startedAt, + updatedAt: startedAt, + startedAt, + completedAt: startedAt, + }); + const state: LabAutomationStateV1 = { + ...defaultLabAutomationStateV1(now - 70 * 60_000), + runs: [run("old", now - 70 * 60_000), run("recent", now - 30 * 60_000)], + runsThisHour: 2, + }; + const rolled = rollBudgetWindow(state, now); + expect(runBudgetRemaining(policy, rolled)).toBe(4); + }); + + test("harness failures receive the stronger blocked cooldown and successful runs clear stale cooldown", async () => { + const home = tempHome(); + prepareHome(home); + const config = providerConfig(); + const policy = livePolicy(); + saveLabAutomationPolicy(policy, home); + saveLabAutomationRoutes({ schemaVersion: 1, routes: [{ providerName: "fixture-provider", modelId: "fixture-model" }] }, home); + const plan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId: "responses-core.live.basic-turn", + providerName: "fixture-provider", + modelId: "fixture-model", + config, + configDir: home, + }); + let state = enqueuePlannedRuns(defaultLabAutomationStateV1(), [plan], "scheduled", Date.now()); + saveLabAutomationState(state, home); + setLabAutomationDispatchDeps({ + configDir: home, + loadConfig: () => config, + resolve: fixtureDnsResolve(), + routeExecutor: createHostIssuedLabRouteExecutor(async () => { throw new Error("executor exploded"); }), + }); + const before = Date.now(); + await runLabAutomationTick(home); + state = loadLabAutomationState(home); + expect(state.cooldownUntilByKey[plan.runKey]).toBeGreaterThanOrEqual(before + policy.blockedCooldownMs - 1_000); + + state.runs = []; + state.cooldownUntilByKey[plan.runKey] = Date.now() - 1; + saveLabAutomationState(state, home); + setLabAutomationDispatchDeps({ + configDir: home, + loadConfig: () => config, + resolve: fixtureDnsResolve(), + routeExecutor: createHostIssuedLabRouteExecutor(async () => passObservation()), + }); + state = enqueuePlannedRuns(loadLabAutomationState(home), [plan], "scheduled", Date.now()); + saveLabAutomationState(state, home); + await runLabAutomationTick(home); + expect(loadLabAutomationState(home).cooldownUntilByKey[plan.runKey]).toBeUndefined(); + }); +}); From a866f1d3d6780d3fc4ec321956248aa2b949205f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:08:41 +0200 Subject: [PATCH 06/67] fix(lab): close CL-08 automation policy schema --- src/lab/automation/policy.ts | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/lab/automation/policy.ts b/src/lab/automation/policy.ts index 59cc6d34c4..c487f0714b 100644 --- a/src/lab/automation/policy.ts +++ b/src/lab/automation/policy.ts @@ -6,6 +6,34 @@ import { import type { LabAutomationPolicyV1 } from "./types"; import { LabAutomationError } from "./types"; +const POLICY_KEYS = new Set([ + "schemaVersion", + "enabled", + "layers", + "refreshBeforeStaleMs", + "maxConcurrentRuns", + "maxConcurrentLiveRuns", + "maxConcurrentRunsPerRoute", + "maxRunsPerHour", + "maxLiveRequestsPerHour", + "failureCooldownMs", + "blockedCooldownMs", + "taskEffectivenessBackgroundEnabled", +]); +const LAYER_KEYS = new Set([ + "protocolConformance", + "liveRouteCompatibility", + "taskEffectiveness", +]); + +function assertClosedKeys(value: Record, allowed: ReadonlySet, field: string): void { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) { + throw new LabAutomationError(`unknown ${field} field ${key}`, "invalid_policy"); + } + } +} + function assertBoolean(value: unknown, field: string): boolean { if (typeof value !== "boolean") throw new LabAutomationError(`invalid policy field ${field}`, "invalid_policy"); return value; @@ -21,12 +49,13 @@ function assertNonNegativeInt(value: unknown, field: string, max: number): numbe return value; } -/** Validate and normalize automation policy with hard upper bounds. */ +/** Validate and normalize automation policy with a closed schema and hard upper bounds. */ export function normalizeLabAutomationPolicyV1(raw: unknown): LabAutomationPolicyV1 { if (!raw || typeof raw !== "object" || Array.isArray(raw)) { throw new LabAutomationError("policy must be an object", "invalid_policy"); } const obj = raw as Record; + assertClosedKeys(obj, POLICY_KEYS, "policy"); if (obj.schemaVersion !== LAB_AUTOMATION_POLICY_SCHEMA_VERSION) { throw new LabAutomationError("unsupported policy schemaVersion", "invalid_policy"); } @@ -35,6 +64,7 @@ export function normalizeLabAutomationPolicyV1(raw: unknown): LabAutomationPolic throw new LabAutomationError("invalid policy layers", "invalid_policy"); } const layersObj = layersRaw as Record; + assertClosedKeys(layersObj, LAYER_KEYS, "layers"); const taskBackground = assertBoolean(obj.taskEffectivenessBackgroundEnabled, "taskEffectivenessBackgroundEnabled"); const layers = { protocolConformance: assertBoolean(layersObj.protocolConformance, "layers.protocolConformance"), From 05e052f2e067be23403f3f52ee67cb1eb6048fd8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:09:43 +0200 Subject: [PATCH 07/67] fix(lab): harden automation state persistence --- src/lab/automation/persistence.ts | 344 +++++++++++++++++++++++------- 1 file changed, 272 insertions(+), 72 deletions(-) diff --git a/src/lab/automation/persistence.ts b/src/lab/automation/persistence.ts index 7337067990..b29aac2025 100644 --- a/src/lab/automation/persistence.ts +++ b/src/lab/automation/persistence.ts @@ -1,4 +1,13 @@ -import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { + closeSync, + existsSync, + openSync, + readFileSync, + renameSync, + statSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { dirname, join } from "node:path"; import { ensureLabDirs, @@ -16,6 +25,94 @@ import type { } from "./types"; import { LabAutomationError } from "./types"; +const ROUTES_KEYS = new Set(["schemaVersion", "routes"]); +const ROUTE_KEYS = new Set(["providerName", "modelId"]); +const STATE_KEYS = new Set([ + "schemaVersion", + "runs", + "budgetWindowStartedAt", + "runsThisHour", + "liveRequestsThisHour", + "cooldownUntilByKey", +]); +const RUN_KEYS = new Set([ + "runId", + "runKey", + "state", + "evidenceLayer", + "suiteId", + "suiteVersion", + "suiteManifestDigest", + "scenarioId", + "scenarioVersion", + "scenarioManifestDigest", + "subjectId", + "reason", + "priority", + "eligibleAt", + "trigger", + "createdAt", + "updatedAt", + "startedAt", + "completedAt", + "terminalCode", + "providerName", + "modelId", +]); +const RUN_STATES = new Set(["queued", "running", "completed", "blocked", "failed", "cancelled", "abandoned"]); +const RUN_REASONS = new Set([ + "automation_disabled", + "layer_disabled", + "fresh", + "refresh_due", + "missing", + "cooldown", + "budget_blocked", + "already_queued", + "already_running", + "route_ineligible", + "scenario_inapplicable", + "task_background_disabled", +]); +const STATE_LOCK_WAIT_MS = 5_000; +const STATE_LOCK_STALE_MS = 30_000; +const LOCK_SLEEP = new Int32Array(new SharedArrayBuffer(4)); + +function assertClosedKeys( + value: Record, + allowed: ReadonlySet, + label: string, + code: "invalid_routes" | "invalid_state", +): void { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) throw new LabAutomationError(`unknown ${label} field ${key}`, code); + } +} + +function assertBoundedString( + value: unknown, + field: string, + code: "invalid_routes" | "invalid_state", + max = 512, +): string { + if (typeof value !== "string" || value.length === 0 || value.length > max) { + throw new LabAutomationError(`invalid ${field}`, code); + } + return value; +} + +function assertNonNegativeInt( + value: unknown, + field: string, + code: "invalid_state", + max = Number.MAX_SAFE_INTEGER, +): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0 || value > max) { + throw new LabAutomationError(`invalid ${field}`, code); + } + return value; +} + function atomicWriteJson(path: string, payload: unknown): void { const dir = dirname(path); const tmp = join(dir, `.${basename(path)}.${process.pid}.${Date.now()}.tmp`); @@ -29,16 +126,65 @@ function basename(path: string): string { return idx >= 0 ? path.slice(idx + 1) : path; } -function readJsonFile(path: string): unknown { +function readJsonFile(path: string, code: "invalid_policy" | "invalid_routes" | "invalid_state"): unknown { if (!existsSync(path)) return undefined; - const raw = readFileSync(path, "utf8"); - return JSON.parse(raw) as unknown; + try { + const raw = readFileSync(path, "utf8"); + return JSON.parse(raw) as unknown; + } catch (error) { + if (error instanceof LabAutomationError) throw error; + throw new LabAutomationError(`invalid automation JSON at ${basename(path)}`, code); + } +} + +function sleepLockRetry(): void { + Atomics.wait(LOCK_SLEEP, 0, 0, 10); +} + +function stateLockPath(configDir?: string): string { + return `${labAutomationStatePath(configDir)}.lock`; +} + +function acquireStateLock(configDir?: string): () => void { + ensureLabDirs(configDir); + const lockPath = stateLockPath(configDir); + const deadline = Date.now() + STATE_LOCK_WAIT_MS; + while (true) { + try { + const fd = openSync(lockPath, "wx", 0o600); + try { + writeFileSync(fd, `${process.pid}:${Date.now()}`, { encoding: "utf8" }); + } finally { + closeSync(fd); + } + return () => { + try { unlinkSync(lockPath); } catch { /* already released/cleaned */ } + }; + } catch (error) { + const code = error && typeof error === "object" && "code" in error + ? String((error as { code?: unknown }).code) + : undefined; + if (code !== "EEXIST") throw new LabAutomationError("automation state lock failed", "state_lock_failed"); + try { + if (Date.now() - statSync(lockPath).mtimeMs > STATE_LOCK_STALE_MS) { + unlinkSync(lockPath); + continue; + } + } catch { + continue; + } + if (Date.now() >= deadline) { + throw new LabAutomationError("automation state is busy", "state_lock_busy"); + } + sleepLockRetry(); + } + } } export function loadLabAutomationPolicy(configDir?: string): LabAutomationPolicyV1 { ensureLabDirs(configDir); const path = labAutomationPolicyPath(configDir); - const raw = readJsonFile(path); + const raw = readJsonFile(path, "invalid_policy"); if (raw === undefined) return defaultLabAutomationPolicyV1(); return normalizeLabAutomationPolicyV1(raw); } @@ -54,25 +200,25 @@ function normalizeRoutes(raw: unknown): LabAutomationRoutesV1 { throw new LabAutomationError("automation routes must be an object", "invalid_routes"); } const obj = raw as Record; + assertClosedKeys(obj, ROUTES_KEYS, "routes", "invalid_routes"); if (obj.schemaVersion !== 1) throw new LabAutomationError("unsupported routes schemaVersion", "invalid_routes"); if (!Array.isArray(obj.routes)) throw new LabAutomationError("routes must be an array", "invalid_routes"); if (obj.routes.length > LAB_AUTOMATION_HARD_MAX.maxAutomationRoutes) { throw new LabAutomationError("too many automation routes", "invalid_routes"); } + const seen = new Set(); const routes = obj.routes.map((row, index) => { if (!row || typeof row !== "object" || Array.isArray(row)) { throw new LabAutomationError(`invalid route row ${index}`, "invalid_routes"); } const entry = row as Record; - const providerName = entry.providerName; - const modelId = entry.modelId; - if (typeof providerName !== "string" || providerName.length === 0) { - throw new LabAutomationError(`invalid providerName at ${index}`, "invalid_routes"); - } - if (typeof modelId !== "string" || modelId.length === 0) { - throw new LabAutomationError(`invalid modelId at ${index}`, "invalid_routes"); - } - return { providerName, modelId }; + assertClosedKeys(entry, ROUTE_KEYS, `route ${index}`, "invalid_routes"); + const providerName = assertBoundedString(entry.providerName, `providerName at ${index}`, "invalid_routes", 256); + const modelId = assertBoundedString(entry.modelId, `modelId at ${index}`, "invalid_routes", 512); + const key = `${providerName}\u0000${modelId}`; + if (seen.has(key)) throw new LabAutomationError(`duplicate automation route at ${index}`, "invalid_routes"); + seen.add(key); + return Object.freeze({ providerName, modelId }); }); return Object.freeze({ schemaVersion: 1, routes: Object.freeze(routes) }); } @@ -83,7 +229,7 @@ export function defaultLabAutomationRoutesV1(): LabAutomationRoutesV1 { export function loadLabAutomationRoutes(configDir?: string): LabAutomationRoutesV1 { ensureLabDirs(configDir); - const raw = readJsonFile(labAutomationRoutesPath(configDir)); + const raw = readJsonFile(labAutomationRoutesPath(configDir), "invalid_routes"); if (raw === undefined) return defaultLabAutomationRoutesV1(); return normalizeRoutes(raw); } @@ -93,14 +239,19 @@ export function saveLabAutomationRoutes(routes: LabAutomationRoutesV1, configDir atomicWriteJson(labAutomationRoutesPath(configDir), normalizeRoutes(routes)); } +function optionalTimestamp(row: Record, key: "startedAt" | "completedAt", index: number): number | undefined { + if (row[key] === undefined) return undefined; + return assertNonNegativeInt(row[key], `${key} at ${index}`, "invalid_state"); +} + function normalizeRunRecord(raw: unknown, index: number): LabAutomationRunRecordV1 { if (!raw || typeof raw !== "object" || Array.isArray(raw)) { throw new LabAutomationError(`invalid run record ${index}`, "invalid_state"); } const row = raw as Record; + assertClosedKeys(row, RUN_KEYS, `run ${index}`, "invalid_state"); const state = row.state; - const allowedStates = new Set(["queued", "running", "completed", "blocked", "failed", "cancelled", "abandoned"]); - if (typeof state !== "string" || !allowedStates.has(state)) { + if (typeof state !== "string" || !RUN_STATES.has(state)) { throw new LabAutomationError(`invalid run state at ${index}`, "invalid_state"); } const evidenceLayer = row.evidenceLayer; @@ -113,50 +264,60 @@ function normalizeRunRecord(raw: unknown, index: number): LabAutomationRunRecord if (trigger !== "scheduled" && trigger !== "manual") { throw new LabAutomationError(`invalid trigger at ${index}`, "invalid_state"); } - const requiredStrings = [ - "runId", "runKey", "suiteId", "suiteVersion", "suiteManifestDigest", - "scenarioId", "scenarioVersion", "scenarioManifestDigest", "subjectId", "reason", - ]; - for (const key of requiredStrings) { - if (typeof row[key] !== "string" || (row[key] as string).length === 0) { - throw new LabAutomationError(`invalid ${key} at ${index}`, "invalid_state"); - } - } - if (typeof row.priority !== "number" || !Number.isInteger(row.priority)) { - throw new LabAutomationError(`invalid priority at ${index}`, "invalid_state"); - } - if (typeof row.eligibleAt !== "number" || !Number.isInteger(row.eligibleAt)) { - throw new LabAutomationError(`invalid eligibleAt at ${index}`, "invalid_state"); - } - if (typeof row.createdAt !== "number" || !Number.isInteger(row.createdAt)) { - throw new LabAutomationError(`invalid createdAt at ${index}`, "invalid_state"); - } - if (typeof row.updatedAt !== "number" || !Number.isInteger(row.updatedAt)) { - throw new LabAutomationError(`invalid updatedAt at ${index}`, "invalid_state"); + const reason = assertBoundedString(row.reason, `reason at ${index}`, "invalid_state", 64); + if (!RUN_REASONS.has(reason)) throw new LabAutomationError(`invalid reason at ${index}`, "invalid_state"); + const runId = assertBoundedString(row.runId, `runId at ${index}`, "invalid_state", 128); + const runKey = assertBoundedString(row.runKey, `runKey at ${index}`, "invalid_state", 128); + const suiteId = assertBoundedString(row.suiteId, `suiteId at ${index}`, "invalid_state", 256); + const suiteVersion = assertBoundedString(row.suiteVersion, `suiteVersion at ${index}`, "invalid_state", 128); + const suiteManifestDigest = assertBoundedString(row.suiteManifestDigest, `suiteManifestDigest at ${index}`, "invalid_state", 256); + const scenarioId = assertBoundedString(row.scenarioId, `scenarioId at ${index}`, "invalid_state", 512); + const scenarioVersion = assertBoundedString(row.scenarioVersion, `scenarioVersion at ${index}`, "invalid_state", 128); + const scenarioManifestDigest = assertBoundedString(row.scenarioManifestDigest, `scenarioManifestDigest at ${index}`, "invalid_state", 256); + const subjectId = assertBoundedString(row.subjectId, `subjectId at ${index}`, "invalid_state", 256); + const priority = assertNonNegativeInt(row.priority, `priority at ${index}`, "invalid_state", 100); + const eligibleAt = assertNonNegativeInt(row.eligibleAt, `eligibleAt at ${index}`, "invalid_state"); + const createdAt = assertNonNegativeInt(row.createdAt, `createdAt at ${index}`, "invalid_state"); + const updatedAt = assertNonNegativeInt(row.updatedAt, `updatedAt at ${index}`, "invalid_state"); + const startedAt = optionalTimestamp(row, "startedAt", index); + const completedAt = optionalTimestamp(row, "completedAt", index); + if (updatedAt < createdAt) throw new LabAutomationError(`updatedAt precedes createdAt at ${index}`, "invalid_state"); + if (startedAt !== undefined && startedAt < createdAt) throw new LabAutomationError(`startedAt precedes createdAt at ${index}`, "invalid_state"); + if (completedAt !== undefined && startedAt !== undefined && completedAt < startedAt) { + throw new LabAutomationError(`completedAt precedes startedAt at ${index}`, "invalid_state"); } + const terminalCode = row.terminalCode === undefined + ? undefined + : assertBoundedString(row.terminalCode, `terminalCode at ${index}`, "invalid_state", 256); + const providerName = row.providerName === undefined + ? undefined + : assertBoundedString(row.providerName, `providerName at ${index}`, "invalid_state", 256); + const modelId = row.modelId === undefined + ? undefined + : assertBoundedString(row.modelId, `modelId at ${index}`, "invalid_state", 512); return { - runId: row.runId as string, - runKey: row.runKey as string, + runId, + runKey, state: state as LabAutomationRunRecordV1["state"], evidenceLayer, - suiteId: row.suiteId as string, - suiteVersion: row.suiteVersion as string, - suiteManifestDigest: row.suiteManifestDigest as string, - scenarioId: row.scenarioId as string, - scenarioVersion: row.scenarioVersion as string, - scenarioManifestDigest: row.scenarioManifestDigest as string, - subjectId: row.subjectId as string, - reason: row.reason as LabAutomationRunRecordV1["reason"], - priority: row.priority as number, - eligibleAt: row.eligibleAt as number, + suiteId, + suiteVersion, + suiteManifestDigest, + scenarioId, + scenarioVersion, + scenarioManifestDigest, + subjectId, + reason: reason as LabAutomationRunRecordV1["reason"], + priority, + eligibleAt, trigger, - createdAt: row.createdAt as number, - updatedAt: row.updatedAt as number, - ...(typeof row.startedAt === "number" ? { startedAt: row.startedAt } : {}), - ...(typeof row.completedAt === "number" ? { completedAt: row.completedAt } : {}), - ...(typeof row.terminalCode === "string" ? { terminalCode: row.terminalCode } : {}), - ...(typeof row.providerName === "string" ? { providerName: row.providerName } : {}), - ...(typeof row.modelId === "string" ? { modelId: row.modelId } : {}), + createdAt, + updatedAt, + ...(startedAt !== undefined ? { startedAt } : {}), + ...(completedAt !== undefined ? { completedAt } : {}), + ...(terminalCode !== undefined ? { terminalCode } : {}), + ...(providerName !== undefined ? { providerName } : {}), + ...(modelId !== undefined ? { modelId } : {}), }; } @@ -165,6 +326,7 @@ function normalizeState(raw: unknown): LabAutomationStateV1 { throw new LabAutomationError("automation state must be an object", "invalid_state"); } const obj = raw as Record; + assertClosedKeys(obj, STATE_KEYS, "state", "invalid_state"); if (obj.schemaVersion !== 1) throw new LabAutomationError("unsupported state schemaVersion", "invalid_state"); if (!Array.isArray(obj.runs)) throw new LabAutomationError("state runs must be an array", "invalid_state"); if (obj.runs.length > LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { @@ -172,23 +334,33 @@ function normalizeState(raw: unknown): LabAutomationStateV1 { } const runs = obj.runs.map((row, index) => normalizeRunRecord(row, index)); const cooldownUntilByKey: Record = {}; - if (obj.cooldownUntilByKey !== undefined) { - if (!obj.cooldownUntilByKey || typeof obj.cooldownUntilByKey !== "object" || Array.isArray(obj.cooldownUntilByKey)) { - throw new LabAutomationError("invalid cooldownUntilByKey", "invalid_state"); - } - for (const [key, value] of Object.entries(obj.cooldownUntilByKey as Record)) { - if (typeof value !== "number" || !Number.isInteger(value)) { - throw new LabAutomationError(`invalid cooldown for ${key}`, "invalid_state"); - } - cooldownUntilByKey[key] = value; - } + if (!obj.cooldownUntilByKey || typeof obj.cooldownUntilByKey !== "object" || Array.isArray(obj.cooldownUntilByKey)) { + throw new LabAutomationError("invalid cooldownUntilByKey", "invalid_state"); + } + const cooldownEntries = Object.entries(obj.cooldownUntilByKey as Record); + if (cooldownEntries.length > LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { + throw new LabAutomationError("too many cooldown entries", "invalid_state"); + } + for (const [key, value] of cooldownEntries) { + if (key.length === 0 || key.length > 128) throw new LabAutomationError("invalid cooldown key", "invalid_state"); + cooldownUntilByKey[key] = assertNonNegativeInt(value, `cooldown for ${key}`, "invalid_state"); } return { schemaVersion: 1, runs, - budgetWindowStartedAt: typeof obj.budgetWindowStartedAt === "number" ? obj.budgetWindowStartedAt : 0, - runsThisHour: typeof obj.runsThisHour === "number" ? obj.runsThisHour : 0, - liveRequestsThisHour: typeof obj.liveRequestsThisHour === "number" ? obj.liveRequestsThisHour : 0, + budgetWindowStartedAt: assertNonNegativeInt(obj.budgetWindowStartedAt, "budgetWindowStartedAt", "invalid_state"), + runsThisHour: assertNonNegativeInt( + obj.runsThisHour, + "runsThisHour", + "invalid_state", + LAB_AUTOMATION_HARD_MAX.maxRunsPerHour, + ), + liveRequestsThisHour: assertNonNegativeInt( + obj.liveRequestsThisHour, + "liveRequestsThisHour", + "invalid_state", + LAB_AUTOMATION_HARD_MAX.maxLiveRequestsPerHour, + ), cooldownUntilByKey, }; } @@ -204,14 +376,42 @@ export function defaultLabAutomationStateV1(now = Date.now()): LabAutomationStat }; } -export function loadLabAutomationState(configDir?: string): LabAutomationStateV1 { +function loadLabAutomationStateUnlocked(configDir?: string): LabAutomationStateV1 { ensureLabDirs(configDir); - const raw = readJsonFile(labAutomationStatePath(configDir)); + const raw = readJsonFile(labAutomationStatePath(configDir), "invalid_state"); if (raw === undefined) return defaultLabAutomationStateV1(); return normalizeState(raw); } -export function saveLabAutomationState(state: LabAutomationStateV1, configDir?: string): void { +function saveLabAutomationStateUnlocked(state: LabAutomationStateV1, configDir?: string): void { ensureLabDirs(configDir); atomicWriteJson(labAutomationStatePath(configDir), normalizeState(state)); } + +export function loadLabAutomationState(configDir?: string): LabAutomationStateV1 { + return loadLabAutomationStateUnlocked(configDir); +} + +export function saveLabAutomationState(state: LabAutomationStateV1, configDir?: string): void { + const release = acquireStateLock(configDir); + try { + saveLabAutomationStateUnlocked(state, configDir); + } finally { + release(); + } +} + +export function mutateLabAutomationState( + configDir: string | undefined, + mutate: (state: LabAutomationStateV1) => { state: LabAutomationStateV1; value: T }, +): T { + const release = acquireStateLock(configDir); + try { + const current = loadLabAutomationStateUnlocked(configDir); + const result = mutate(current); + saveLabAutomationStateUnlocked(result.state, configDir); + return result.value; + } finally { + release(); + } +} From baaedf05edbf3a2f0a4b228d4204438987c632e5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:10:11 +0200 Subject: [PATCH 08/67] fix(lab): make automation budgets rolling and attempt-safe --- src/lab/automation/budgets.ts | 62 +++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/src/lab/automation/budgets.ts b/src/lab/automation/budgets.ts index 545ab3e149..5b260f352e 100644 --- a/src/lab/automation/budgets.ts +++ b/src/lab/automation/budgets.ts @@ -11,26 +11,68 @@ export function rollBudgetWindow(state: LabAutomationStateV1, now: number): LabA }; } -export function runBudgetRemaining(policy: LabAutomationPolicyV1, state: LabAutomationStateV1): number { - return Math.max(0, policy.maxRunsPerHour - state.runsThisHour); +function rollingStartedRuns(state: LabAutomationStateV1, now: number): number { + const cutoff = now - LAB_AUTOMATION_HARD_MAX.budgetWindowMs; + return state.runs.filter((run) => typeof run.startedAt === "number" && run.startedAt > cutoff && run.startedAt <= now).length; } -export function liveRequestBudgetRemaining(policy: LabAutomationPolicyV1, state: LabAutomationStateV1): number { - return Math.max(0, policy.maxLiveRequestsPerHour - state.liveRequestsThisHour); +function rollingStartedLiveRuns(state: LabAutomationStateV1, now: number): number { + const cutoff = now - LAB_AUTOMATION_HARD_MAX.budgetWindowMs; + return state.runs.filter((run) => + run.evidenceLayer === "live_route_compatibility" + && typeof run.startedAt === "number" + && run.startedAt > cutoff + && run.startedAt <= now + ).length; } -export function isRunBudgetExhausted(policy: LabAutomationPolicyV1, state: LabAutomationStateV1): boolean { - return runBudgetRemaining(policy, state) <= 0; +function legacyCounterActive(state: LabAutomationStateV1, now: number): boolean { + return now - state.budgetWindowStartedAt < LAB_AUTOMATION_HARD_MAX.budgetWindowMs; } -export function isLiveRequestBudgetExhausted(policy: LabAutomationPolicyV1, state: LabAutomationStateV1): boolean { - return liveRequestBudgetRemaining(policy, state) <= 0; +export function runBudgetRemaining( + policy: LabAutomationPolicyV1, + state: LabAutomationStateV1, + now = Date.now(), +): number { + const rolling = rollingStartedRuns(state, now); + const legacy = legacyCounterActive(state, now) ? state.runsThisHour : 0; + return Math.max(0, policy.maxRunsPerHour - Math.max(rolling, legacy)); } +export function liveRequestBudgetRemaining( + policy: LabAutomationPolicyV1, + state: LabAutomationStateV1, + now = Date.now(), +): number { + const rolling = rollingStartedLiveRuns(state, now); + const legacy = legacyCounterActive(state, now) ? state.liveRequestsThisHour : 0; + return Math.max(0, policy.maxLiveRequestsPerHour - Math.max(rolling, legacy)); +} + +export function isRunBudgetExhausted( + policy: LabAutomationPolicyV1, + state: LabAutomationStateV1, + now = Date.now(), +): boolean { + return runBudgetRemaining(policy, state, now) <= 0; +} + +export function isLiveRequestBudgetExhausted( + policy: LabAutomationPolicyV1, + state: LabAutomationStateV1, + now = Date.now(), +): boolean { + return liveRequestBudgetRemaining(policy, state, now) <= 0; +} + +/** Legacy persisted counters remain for schema compatibility; startedAt records are rolling authority. */ export function recordRunBudgetUse(state: LabAutomationStateV1, liveRequest: boolean): LabAutomationStateV1 { return { ...state, - runsThisHour: state.runsThisHour + 1, - liveRequestsThisHour: liveRequest ? state.liveRequestsThisHour + 1 : state.liveRequestsThisHour, + runsThisHour: Math.min(LAB_AUTOMATION_HARD_MAX.maxRunsPerHour, state.runsThisHour + 1), + liveRequestsThisHour: liveRequest + ? Math.min(LAB_AUTOMATION_HARD_MAX.maxLiveRequestsPerHour, state.liveRequestsThisHour + 1) + : state.liveRequestsThisHour, }; } From 16286a657b9e9ce475d79e17fe0d1295beccafa8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:10:48 +0200 Subject: [PATCH 09/67] fix(lab): bound CL-08 queue and dispatch selection --- src/lab/automation/queue.ts | 55 ++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/src/lab/automation/queue.ts b/src/lab/automation/queue.ts index 3897e45e7f..7d8f559189 100644 --- a/src/lab/automation/queue.ts +++ b/src/lab/automation/queue.ts @@ -8,6 +8,35 @@ import type { } from "./types"; import { LAB_AUTOMATION_HARD_MAX } from "./constants"; +const TERMINAL_STATES = new Set([ + "completed", + "blocked", + "failed", + "cancelled", + "abandoned", +]); + +function isTerminal(run: LabAutomationRunRecordV1): boolean { + return TERMINAL_STATES.has(run.state); +} + +function evictOldestTerminal(runs: LabAutomationRunRecordV1[]): boolean { + let oldestIndex = -1; + let oldestTime = Number.POSITIVE_INFINITY; + for (let index = 0; index < runs.length; index += 1) { + const run = runs[index]!; + if (!isTerminal(run)) continue; + const terminalAt = run.completedAt ?? run.updatedAt; + if (terminalAt < oldestTime) { + oldestTime = terminalAt; + oldestIndex = index; + } + } + if (oldestIndex < 0) return false; + runs.splice(oldestIndex, 1); + return true; +} + export function countRunsByState(state: LabAutomationStateV1, runState: LabAutomationRunState): number { return state.runs.filter((row) => row.state === runState).length; } @@ -36,9 +65,13 @@ export function enqueuePlannedRuns( .map((row) => row.runKey), ); const runs = [...state.runs]; + let queuedCount = runs.filter((row) => row.state === "queued").length; for (const plan of planned) { if (existingActiveKeys.has(plan.runKey)) continue; - if (runs.length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) break; + if (queuedCount >= LAB_AUTOMATION_HARD_MAX.maxQueuedRuns) break; + while (runs.length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { + if (!evictOldestTerminal(runs)) return { ...state, runs }; + } runs.push({ runId: randomUUID(), runKey: plan.runKey, @@ -60,6 +93,7 @@ export function enqueuePlannedRuns( ...(plan.providerName ? { providerName: plan.providerName } : {}), ...(plan.modelId ? { modelId: plan.modelId } : {}), }); + queuedCount += 1; existingActiveKeys.add(plan.runKey); } return { ...state, runs }; @@ -97,44 +131,45 @@ export function selectDispatchableRuns( policy: LabAutomationPolicyV1, state: LabAutomationStateV1, now: number, + predicate: (run: LabAutomationRunRecordV1) => boolean = () => true, ): LabAutomationRunRecordV1[] { const running = countRunsByState(state, "running"); if (running >= policy.maxConcurrentRuns) return []; const slots = policy.maxConcurrentRuns - running; const queued = state.runs - .filter((row) => row.state === "queued" && row.eligibleAt <= now) + .filter((row) => row.state === "queued" && row.eligibleAt <= now && predicate(row)) .sort((a, b) => { if (a.priority !== b.priority) return a.priority - b.priority; if (a.eligibleAt !== b.eligibleAt) return a.eligibleAt - b.eligibleAt; return a.runId < b.runId ? -1 : 1; }); const selected: LabAutomationRunRecordV1[] = []; + const selectedPerRoute = new Map(); let liveRunning = countRunningLive(state); for (const row of queued) { if (selected.length >= slots) break; - if (countRunsByState({ ...state, runs: [...state.runs, ...selected.map((s) => ({ ...s, state: "running" as const }))] }, "running") >= policy.maxConcurrentRuns) break; + const runningForRoute = countRunningForRoute(state, row.subjectId); + const selectedForRoute = selectedPerRoute.get(row.subjectId) ?? 0; + if (runningForRoute + selectedForRoute >= policy.maxConcurrentRunsPerRoute) continue; if (row.evidenceLayer === "live_route_compatibility") { if (liveRunning >= policy.maxConcurrentLiveRuns) continue; - if (countRunningForRoute(state, row.subjectId) >= policy.maxConcurrentRunsPerRoute) continue; - } else if (countRunningForRoute(state, row.subjectId) >= policy.maxConcurrentRunsPerRoute) { - continue; } selected.push(row); + selectedPerRoute.set(row.subjectId, selectedForRoute + 1); if (row.evidenceLayer === "live_route_compatibility") liveRunning += 1; } return selected; } export function trimTerminalRuns(state: LabAutomationStateV1, now: number): LabAutomationStateV1 { - const terminal = new Set(["completed", "blocked", "failed", "cancelled", "abandoned"]); const keepMs = 7 * 24 * 60 * 60 * 1000; const runs = state.runs.filter((row) => { - if (!terminal.has(row.state)) return true; + if (!isTerminal(row)) return true; const completedAt = row.completedAt ?? row.updatedAt; return now - completedAt < keepMs; }); - if (runs.length > LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { - return { ...state, runs: runs.slice(-LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) }; + while (runs.length > LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { + if (!evictOldestTerminal(runs)) break; } return { ...state, runs }; } From 1b1eb79a0ebc9af7944f5516a6251a83d5cb46aa Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:11:52 +0200 Subject: [PATCH 10/67] fix(lab): scope automation freshness to exact evidence identity --- src/lab/automation/planner.ts | 95 ++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 29 deletions(-) diff --git a/src/lab/automation/planner.ts b/src/lab/automation/planner.ts index 26f0f440d3..f110f13125 100644 --- a/src/lab/automation/planner.ts +++ b/src/lab/automation/planner.ts @@ -9,10 +9,9 @@ import { isLiveCaseApplicableToRoute } from "../live/executor"; import { buildAutomationLiveRouteContext } from "./route-context"; import { liveSuiteManifestDigestForCase } from "../live/suite-manifest"; import { suiteManifestDigestForCase } from "../conformance/suite-manifest"; -import { newestObservationByScenario, isScenarioApplicable } from "../projection/verification"; +import { isScenarioApplicable } from "../projection/verification"; import { queryLabObservations } from "../query/queries"; import { LabProjectionUnavailableError } from "../query/errors"; -import type { ObservationEvent } from "../events/types"; import type { OcxConfig } from "../../types"; import { resolvePolicyCompatibilitySubjects } from "../../routing/compatibility/subject"; import type { @@ -43,6 +42,17 @@ export interface PlannerInput { configDir?: string; } +interface EvidenceIdentity { + layer: EvidenceLayer; + subjectId: string; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + scenarioId: string; + scenarioVersion: string; + scenarioManifestDigest: string; +} + function effectiveMaxAgeMs(suiteMax: number | null, scenarioMax: number | null): number | null { if (suiteMax === null) return scenarioMax; if (scenarioMax === null) return suiteMax; @@ -68,22 +78,31 @@ function activeRunForKey(state: LabAutomationStateV1, runKey: string): LabAutoma return state.runs.find((row) => row.runKey === runKey && (row.state === "queued" || row.state === "running")); } -function loadObservationEvents(layer: EvidenceLayer, configDir?: string): ObservationEvent[] { +function latestMatchingObservationCompletedAt(identity: EvidenceIdentity, configDir?: string): number | undefined { try { const page = queryLabObservations( - { layer }, + { + layer: identity.layer, + subjectId: identity.subjectId, + suiteId: identity.suiteId, + scenarioId: identity.scenarioId, + }, undefined, LAB_AUTOMATION_HARD_MAX.maxPersistedRuns, configDir, ); - return page.items.map((row) => ({ - eventKind: "observation", - eventId: row.eventId, - scenarioId: row.scenarioId, - completedAt: row.completedAt, - } as ObservationEvent)); + let latest: number | undefined; + for (const row of page.items) { + if (row.excluded) continue; + if (row.suiteVersion !== identity.suiteVersion) continue; + if (row.suiteManifestDigest !== identity.suiteManifestDigest) continue; + if (row.scenarioVersion !== identity.scenarioVersion) continue; + if (row.scenarioManifestDigest !== identity.scenarioManifestDigest) continue; + if (latest === undefined || row.completedAt > latest) latest = row.completedAt; + } + return latest; } catch (error) { - if (error instanceof LabProjectionUnavailableError) return []; + if (error instanceof LabProjectionUnavailableError) return undefined; throw error; } } @@ -93,8 +112,6 @@ function planProtocolScenarios(input: PlannerInput): PlannedLabRunV1[] { const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, CL01_SUITES); const planned: PlannedLabRunV1[] = []; - const observations = loadObservationEvents("protocol_conformance", input.configDir); - const byScenario = newestObservationByScenario(observations); for (const caseRecord of scenarios) { if (!isScenarioApplicable(caseRecord.id, "fixture", "protocol_conformance")) continue; let subject; @@ -108,34 +125,45 @@ function planProtocolScenarios(input: PlannerInput): PlannedLabRunV1[] { const expanded = expandScenario(caseRecord, authority); const scenarioDigest = scenarioManifestDigest(expanded); const suiteDigest = suiteManifestDigestForCase(caseRecord, authority); + const suiteVersion = authority.manifestDefaults.suiteVersion; + const scenarioVersion = authority.manifestDefaults.version; const runKey = buildLabAutomationRunKey({ evidenceLayer: "protocol_conformance", subjectId, suiteId: caseRecord.suite, - suiteVersion: authority.manifestDefaults.suiteVersion, + suiteVersion, suiteManifestDigest: suiteDigest, scenarioId: caseRecord.id, - scenarioVersion: authority.manifestDefaults.version, + scenarioVersion, scenarioManifestDigest: scenarioDigest, executionContractDigest: protocolExecutionContractDigest(), }); if (activeRunForKey(input.state, runKey)) continue; if (cooldownActive(input.state, runKey, input.now)) continue; - const latest = byScenario.get(caseRecord.id); + const latestCompletedAt = latestMatchingObservationCompletedAt({ + layer: "protocol_conformance", + subjectId, + suiteId: caseRecord.suite, + suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion, + scenarioManifestDigest: scenarioDigest, + }, input.configDir); const maxAge = effectiveMaxAgeMs( authority.manifestDefaults.freshness.maxAgeMs, authority.manifestDefaults.freshness.maxAgeMs, ); - const freshness = freshnessReason(latest?.completedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); + const freshness = freshnessReason(latestCompletedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); if (freshness === "fresh") continue; planned.push({ runKey, evidenceLayer: "protocol_conformance", suiteId: caseRecord.suite, - suiteVersion: authority.manifestDefaults.suiteVersion, + suiteVersion, suiteManifestDigest: suiteDigest, scenarioId: caseRecord.id, - scenarioVersion: authority.manifestDefaults.version, + scenarioVersion, scenarioManifestDigest: scenarioDigest, subjectId, reason: freshness, @@ -149,12 +177,10 @@ function planProtocolScenarios(input: PlannerInput): PlannedLabRunV1[] { function planLiveScenarios(input: PlannerInput): PlannedLabRunV1[] { if (!input.policy.enabled || !input.policy.layers.liveRouteCompatibility) return []; if (!input.config) return []; - if (isLiveRequestBudgetExhausted(input.policy, input.state)) return []; + if (isLiveRequestBudgetExhausted(input.policy, input.state, input.now)) return []; const authority = loadLiveCaseAuthority(); const scenarios = discoverLiveScenarios(authority, CL03_LIVE_SUITES); const planned: PlannedLabRunV1[] = []; - const observations = loadObservationEvents("live_route_compatibility", input.configDir); - const byScenario = newestObservationByScenario(observations); for (const routeRef of input.routes.routes) { const routed = input.config.providers?.[routeRef.providerName]; if (!routed) continue; @@ -175,35 +201,46 @@ function planLiveScenarios(input: PlannerInput): PlannedLabRunV1[] { const expanded = expandLiveScenario(caseRecord, authority); const scenarioDigest = scenarioManifestDigest(expanded); const suiteDigest = liveSuiteManifestDigestForCase(caseRecord, authority); + const suiteVersion = authority.manifestDefaults.suiteVersion; + const scenarioVersion = authority.manifestDefaults.version; const subjectId = resolved.route.subjectId; const runKey = buildLabAutomationRunKey({ evidenceLayer: "live_route_compatibility", subjectId, suiteId: caseRecord.suite, - suiteVersion: authority.manifestDefaults.suiteVersion, + suiteVersion, suiteManifestDigest: suiteDigest, scenarioId: caseRecord.id, - scenarioVersion: authority.manifestDefaults.version, + scenarioVersion, scenarioManifestDigest: scenarioDigest, executionContractDigest: liveExecutionContractDigest(), }); if (activeRunForKey(input.state, runKey)) continue; if (cooldownActive(input.state, runKey, input.now)) continue; - const latest = byScenario.get(caseRecord.id); + const latestCompletedAt = latestMatchingObservationCompletedAt({ + layer: "live_route_compatibility", + subjectId, + suiteId: caseRecord.suite, + suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion, + scenarioManifestDigest: scenarioDigest, + }, input.configDir); const maxAge = effectiveMaxAgeMs( authority.manifestDefaults.freshness.maxAgeMs, authority.manifestDefaults.freshness.maxAgeMs, ); - const freshness = freshnessReason(latest?.completedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); + const freshness = freshnessReason(latestCompletedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); if (freshness === "fresh") continue; planned.push({ runKey, evidenceLayer: "live_route_compatibility", suiteId: caseRecord.suite, - suiteVersion: authority.manifestDefaults.suiteVersion, + suiteVersion, suiteManifestDigest: suiteDigest, scenarioId: caseRecord.id, - scenarioVersion: authority.manifestDefaults.version, + scenarioVersion, scenarioManifestDigest: scenarioDigest, subjectId, reason: freshness, @@ -220,7 +257,7 @@ function planLiveScenarios(input: PlannerInput): PlannedLabRunV1[] { /** Deterministic planner — no execution side effects. */ export function planLabAutomationRuns(input: PlannerInput): PlannedLabRunV1[] { if (!input.policy.enabled) return []; - if (isRunBudgetExhausted(input.policy, input.state)) return []; + if (isRunBudgetExhausted(input.policy, input.state, input.now)) return []; const protocol = planProtocolScenarios(input); const live = planLiveScenarios(input); const merged = [...protocol, ...live]; From 671346546be7a95ea8d065c282e5c7a810256214 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:12:30 +0200 Subject: [PATCH 11/67] fix(lab): revalidate CL-08 dispatch contracts --- src/lab/automation/dispatch.ts | 140 ++++++++++++++++++++++++++++----- 1 file changed, 121 insertions(+), 19 deletions(-) diff --git a/src/lab/automation/dispatch.ts b/src/lab/automation/dispatch.ts index 7773ed0157..e9fd6df61b 100644 --- a/src/lab/automation/dispatch.ts +++ b/src/lab/automation/dispatch.ts @@ -1,22 +1,75 @@ -import { loadCaseAuthority } from "../conformance/manifest"; -import { runScenario } from "../conformance/executor"; +import { subjectIdForSubject, scenarioManifestDigest } from "../digest"; +import { expandScenario, loadCaseAuthority } from "../conformance/manifest"; +import { resolveProtocolExecutionContext, runScenario } from "../conformance/executor"; +import { buildProtocolSubjectV1 } from "../subject/protocol-subject"; +import { suiteManifestDigestForCase } from "../conformance/suite-manifest"; import { persistConformanceResult } from "../observe/from-conformance"; -import { loadLiveCaseAuthority } from "../live/manifest"; +import { expandLiveScenario, loadLiveCaseAuthority } from "../live/manifest"; import { runLiveScenario } from "../live/executor"; +import { liveSuiteManifestDigestForCase } from "../live/suite-manifest"; import { persistLiveResult } from "../observe/from-live"; import { rebuildLabProjection } from "../projection/rebuild"; import { resolvePolicyCompatibilitySubjects } from "../../routing/compatibility/subject"; -import type { OcxConfig } from "../../types"; import type { AutomationDispatchDeps, LabAutomationRunRecordV1 } from "./types"; import { LabAutomationError } from "./types"; import { buildAutomationLiveRouteContext } from "./route-context"; +import { + buildLabAutomationRunKey, + liveExecutionContractDigest, + protocolExecutionContractDigest, +} from "./run-key"; export interface DispatchResult { terminalState: "completed" | "blocked" | "failed"; terminalCode: string; + cooldownCode: string; liveRequest: boolean; } +interface ExpectedRunIdentity { + runKey: string; + subjectId: string; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + scenarioId: string; + scenarioVersion: string; + scenarioManifestDigest: string; +} + +function runIdentityMatches(run: LabAutomationRunRecordV1, expected: ExpectedRunIdentity): boolean { + return run.runKey === expected.runKey + && run.subjectId === expected.subjectId + && run.suiteId === expected.suiteId + && run.suiteVersion === expected.suiteVersion + && run.suiteManifestDigest === expected.suiteManifestDigest + && run.scenarioId === expected.scenarioId + && run.scenarioVersion === expected.scenarioVersion + && run.scenarioManifestDigest === expected.scenarioManifestDigest; +} + +function contractChanged(): DispatchResult { + return { + terminalState: "blocked", + terminalCode: "run_contract_changed", + cooldownCode: "run_contract_changed", + liveRequest: false, + }; +} + +function routeIneligible(): DispatchResult { + return { + terminalState: "blocked", + terminalCode: "route_ineligible", + cooldownCode: "route_ineligible", + liveRequest: false, + }; +} + +function assertNotCancelled(signal?: AbortSignal): void { + if (signal?.aborted) throw new LabAutomationError("cancelled", "cancelled"); +} + /** Closed dispatcher to existing CL-01 / CL-03 / CL-07 producers. */ export async function dispatchLabAutomationRun( run: LabAutomationRunRecordV1, @@ -25,38 +78,60 @@ export async function dispatchLabAutomationRun( const configDir = deps.configDir; switch (run.evidenceLayer) { case "protocol_conformance": { - if (deps.abortSignal?.aborted) { - throw new LabAutomationError("cancelled", "cancelled"); - } + assertNotCancelled(deps.abortSignal); const authority = loadCaseAuthority(); const caseRecord = authority.cases.find((row) => row.id === run.scenarioId); if (!caseRecord) { throw new LabAutomationError("missing protocol scenario", "dispatch_failure"); } + const executionContext = resolveProtocolExecutionContext(caseRecord); + const subject = buildProtocolSubjectV1(executionContext); + const subjectId = subjectIdForSubject(subject); + const scenarioDigest = scenarioManifestDigest(expandScenario(caseRecord, authority)); + const suiteDigest = suiteManifestDigestForCase(caseRecord, authority); + const suiteVersion = authority.manifestDefaults.suiteVersion; + const scenarioVersion = authority.manifestDefaults.version; + const expected: ExpectedRunIdentity = { + runKey: buildLabAutomationRunKey({ + evidenceLayer: "protocol_conformance", + subjectId, + suiteId: caseRecord.suite, + suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion, + scenarioManifestDigest: scenarioDigest, + executionContractDigest: protocolExecutionContractDigest(), + }), + subjectId, + suiteId: caseRecord.suite, + suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion, + scenarioManifestDigest: scenarioDigest, + }; + if (!runIdentityMatches(run, expected)) return contractChanged(); const result = await runScenario(caseRecord); - if (deps.abortSignal?.aborted) { - throw new LabAutomationError("cancelled", "cancelled"); - } + assertNotCancelled(deps.abortSignal); persistConformanceResult(result, caseRecord, authority, { configDir }); rebuildLabProjection(configDir); return { terminalState: result.passed ? "completed" : "failed", terminalCode: result.passed ? "pass" : "protocol_fail", + cooldownCode: result.passed ? "pass" : "protocol_failure", liveRequest: false, }; } case "live_route_compatibility": { - if (!deps.routeExecutor) { - return { terminalState: "blocked", terminalCode: "route_ineligible", liveRequest: false }; - } + assertNotCancelled(deps.abortSignal); + if (!deps.routeExecutor) return routeIneligible(); if (!deps.loadConfig || !run.providerName || !run.modelId) { throw new LabAutomationError("live dispatch requires route configuration", "dispatch_failure"); } const config = deps.loadConfig(); const routed = config.providers?.[run.providerName]; - if (!routed) { - return { terminalState: "blocked", terminalCode: "route_ineligible", liveRequest: false }; - } + if (!routed) return routeIneligible(); const resolved = resolvePolicyCompatibilitySubjects( config, run.providerName, @@ -64,14 +139,38 @@ export async function dispatchLabAutomationRun( routed, configDir, ); - if (!resolved.route) { - return { terminalState: "blocked", terminalCode: "route_ineligible", liveRequest: false }; - } + if (!resolved.route) return routeIneligible(); const authority = loadLiveCaseAuthority(); const caseRecord = authority.cases.find((row) => row.id === run.scenarioId); if (!caseRecord) { throw new LabAutomationError("missing live scenario", "dispatch_failure"); } + const scenarioDigest = scenarioManifestDigest(expandLiveScenario(caseRecord, authority)); + const suiteDigest = liveSuiteManifestDigestForCase(caseRecord, authority); + const suiteVersion = authority.manifestDefaults.suiteVersion; + const scenarioVersion = authority.manifestDefaults.version; + const subjectId = resolved.route.subjectId; + const expected: ExpectedRunIdentity = { + runKey: buildLabAutomationRunKey({ + evidenceLayer: "live_route_compatibility", + subjectId, + suiteId: caseRecord.suite, + suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion, + scenarioManifestDigest: scenarioDigest, + executionContractDigest: liveExecutionContractDigest(), + }), + subjectId, + suiteId: caseRecord.suite, + suiteVersion, + suiteManifestDigest: suiteDigest, + scenarioId: caseRecord.id, + scenarioVersion, + scenarioManifestDigest: scenarioDigest, + }; + if (!runIdentityMatches(run, expected)) return contractChanged(); const routeContext = buildAutomationLiveRouteContext( resolved.route, routed.allowPrivateNetwork === true, @@ -82,6 +181,8 @@ export async function dispatchLabAutomationRun( cancelSignal: deps.abortSignal, resolve: deps.resolve, }); + // Cancellation is orchestration state. Never mint canonical evidence after an abort. + assertNotCancelled(deps.abortSignal); if (result.executionAuthority === "trusted_route") { persistLiveResult(result, caseRecord, authority, { configDir }); rebuildLabProjection(configDir); @@ -92,6 +193,7 @@ export async function dispatchLabAutomationRun( return { terminalState: result.passed ? "completed" : blocked ? "blocked" : "failed", terminalCode: result.secondaryCode ?? result.classification, + cooldownCode: result.classification, liveRequest: true, }; } From 5227aa05700654276e41617f1a0e3d9f4ff64201 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:13:57 +0200 Subject: [PATCH 12/67] fix(lab): serialize automation scheduling state --- src/lab/automation/orchestrator.ts | 270 +++++++++++++++++++++-------- 1 file changed, 196 insertions(+), 74 deletions(-) diff --git a/src/lab/automation/orchestrator.ts b/src/lab/automation/orchestrator.ts index 3d9752609c..5f1067603c 100644 --- a/src/lab/automation/orchestrator.ts +++ b/src/lab/automation/orchestrator.ts @@ -4,10 +4,16 @@ import { loadLabAutomationPolicy, loadLabAutomationRoutes, loadLabAutomationState, - saveLabAutomationState, + mutateLabAutomationState, } from "./persistence"; -import { rollBudgetWindow, recordRunBudgetUse, runBudgetRemaining, liveRequestBudgetRemaining, isRunBudgetExhausted, isLiveRequestBudgetExhausted } from "./budgets"; -import { cooldownForFailure, setCooldown } from "./cooldown"; +import { + rollBudgetWindow, + runBudgetRemaining, + liveRequestBudgetRemaining, + isRunBudgetExhausted, + isLiveRequestBudgetExhausted, +} from "./budgets"; +import { clearCooldown, cooldownForFailure, setCooldown } from "./cooldown"; import { enqueuePlannedRuns, selectDispatchableRuns, @@ -17,7 +23,13 @@ import { } from "./queue"; import { dispatchLabAutomationRun, type DispatchResult } from "./dispatch"; import { recoverLabAutomationState } from "./recovery"; -import type { AutomationDispatchDeps, LabAutomationRunRecordV1, LabAutomationStatusV1 } from "./types"; +import type { + AutomationDispatchDeps, + LabAutomationPolicyV1, + LabAutomationRoutesV1, + LabAutomationRunRecordV1, + LabAutomationStatusV1, +} from "./types"; import { LabAutomationError } from "./types"; import { LAB_AUTOMATION_HARD_MAX } from "./constants"; import { cancelQueuedRun } from "./queue"; @@ -48,7 +60,8 @@ export function buildLabAutomationStatus(configDir?: string): LabAutomationStatu const policy = loadLabAutomationPolicy(configDir); const routes = loadLabAutomationRoutes(configDir); const state = loadLabAutomationState(configDir); - const hourAgo = Date.now() - LAB_AUTOMATION_HARD_MAX.budgetWindowMs; + const now = Date.now(); + const hourAgo = now - LAB_AUTOMATION_HARD_MAX.budgetWindowMs; const completedLastHour = state.runs.filter((row) => row.state === "completed" && (row.completedAt ?? 0) >= hourAgo).length; const blockedLastHour = state.runs.filter((row) => row.state === "blocked" && (row.completedAt ?? 0) >= hourAgo).length; return { @@ -59,45 +72,168 @@ export function buildLabAutomationStatus(configDir?: string): LabAutomationStatu running: countRunsByState(state, "running"), completedLastHour, blockedLastHour, - remainingRunBudget: runBudgetRemaining(policy, state), - remainingLiveRequestBudget: liveRequestBudgetRemaining(policy, state), + remainingRunBudget: runBudgetRemaining(policy, state, now), + remainingLiveRequestBudget: liveRequestBudgetRemaining(policy, state, now), }, schedulerRunning: isLabAutomationSchedulerRunning(), }; } -async function runDispatchBatch(configDir?: string, now = Date.now()): Promise { - if (shutdownRequested) return; +function scheduledEligibilityCode( + policy: LabAutomationPolicyV1, + routes: LabAutomationRoutesV1, + run: LabAutomationRunRecordV1, +): string | null { + if (run.trigger !== "scheduled") return null; + if (!policy.enabled) return "automation_disabled"; + switch (run.evidenceLayer) { + case "protocol_conformance": + return policy.layers.protocolConformance ? null : "layer_disabled"; + case "live_route_compatibility": { + if (!policy.layers.liveRouteCompatibility) return "layer_disabled"; + if (!run.providerName || !run.modelId) return "route_ineligible"; + const enrolled = routes.routes.some((route) => + route.providerName === run.providerName && route.modelId === run.modelId + ); + return enrolled ? null : "route_ineligible"; + } + case "task_effectiveness": + if (!policy.layers.taskEffectiveness) return "layer_disabled"; + return policy.taskEffectivenessBackgroundEnabled ? null : "task_background_disabled"; + } +} + +function reconcileQueuedState( + state: ReturnType, + policy: LabAutomationPolicyV1, + routes: LabAutomationRoutesV1, + now: number, +): ReturnType { + let next = state; + for (const run of state.runs) { + if (run.state !== "queued" || run.trigger !== "scheduled") continue; + const code = scheduledEligibilityCode(policy, routes, run); + if (code) next = transitionRun(next, run.runId, "cancelled", now, code); + } + return next; +} + +/** Apply current policy/route enrollment to already queued scheduled work without executing it. */ +export function reconcileLabAutomationQueue(configDir?: string): void { const policy = loadLabAutomationPolicy(configDir); - if (!policy.enabled) return; - let state = rollBudgetWindow(loadLabAutomationState(configDir), now); const routes = loadLabAutomationRoutes(configDir); - let config: import("../../types").OcxConfig | undefined; + const now = Date.now(); + mutateLabAutomationState(configDir, (state) => ({ + state: trimTerminalRuns(reconcileQueuedState(state, policy, routes, now), now), + value: undefined, + })); +} + +function loadPlannerConfig(): import("../../types").OcxConfig | undefined { try { - config = readConfigDiagnostics().config; + return dispatchDeps.loadConfig?.() ?? readConfigDiagnostics().config; } catch { - config = undefined; + return undefined; } - const planned = planLabAutomationRuns({ - policy, - routes, - state, - now, - config, - configDir, +} + +function claimNextRun( + policy: LabAutomationPolicyV1, + routes: LabAutomationRoutesV1, + planned: ReturnType, + configDir: string | undefined, + now: number, + manualRunId?: string, +): LabAutomationRunRecordV1 | null { + return mutateLabAutomationState(configDir, (loaded) => { + let state = rollBudgetWindow(loaded, now); + state = reconcileQueuedState(state, policy, routes, now); + if (policy.enabled && manualRunId === undefined) { + state = enqueuePlannedRuns(state, planned, "scheduled", now); + } + if (isRunBudgetExhausted(policy, state, now)) { + return { state: trimTerminalRuns(state, now), value: null }; + } + const predicate = (run: LabAutomationRunRecordV1): boolean => { + if (manualRunId !== undefined) return run.runId === manualRunId && run.trigger === "manual"; + return policy.enabled && run.trigger === "scheduled"; + }; + const dispatchable = selectDispatchableRuns(policy, state, now, (run) => { + if (!predicate(run)) return false; + if (run.evidenceLayer === "live_route_compatibility" && isLiveRequestBudgetExhausted(policy, state, now)) { + return false; + } + return scheduledEligibilityCode(policy, routes, run) === null; + }); + const selected = dispatchable[0]; + if (!selected) return { state: trimTerminalRuns(state, now), value: null }; + state = transitionRun(state, selected.runId, "running", now); + const running = state.runs.find((run) => run.runId === selected.runId) ?? null; + return { state: trimTerminalRuns(state, now), value: running }; }); - state = enqueuePlannedRuns(state, planned, "scheduled", now); - const dispatchable = selectDispatchableRuns(policy, state, now); - for (const run of dispatchable) { - if (shutdownRequested) break; - if (isRunBudgetExhausted(policy, state)) break; - if (run.evidenceLayer === "live_route_compatibility" && isLiveRequestBudgetExhausted(policy, state)) { - continue; +} + +function finalizeRun( + run: LabAutomationRunRecordV1, + policy: LabAutomationPolicyV1, + configDir: string | undefined, + result: DispatchResult | null, + error: unknown, + cancelled: boolean, +): void { + const completedAt = Date.now(); + mutateLabAutomationState(configDir, (state) => { + const current = state.runs.find((row) => row.runId === run.runId); + if (!current || current.state !== "running") { + return { state: trimTerminalRuns(state, completedAt), value: undefined }; + } + let next = state; + if (cancelled || (error instanceof LabAutomationError && error.code === "cancelled")) { + next = transitionRun(next, run.runId, "cancelled", completedAt, "cancelled"); + return { state: trimTerminalRuns(next, completedAt), value: undefined }; } - state = transitionRun(state, run.runId, "running", now); - saveLabAutomationState(state, configDir); + if (result) { + next = transitionRun(next, run.runId, result.terminalState, completedAt, result.terminalCode); + if (result.terminalState === "completed") { + next = clearCooldown(next, run.runKey); + } else { + next = setCooldown( + next, + run.runKey, + cooldownForFailure(policy, result.cooldownCode, completedAt), + ); + } + return { state: trimTerminalRuns(next, completedAt), value: undefined }; + } + const terminalCode = error instanceof LabAutomationError ? error.code : "dispatch_failure"; + next = transitionRun(next, run.runId, "failed", completedAt, terminalCode); + next = setCooldown(next, run.runKey, cooldownForFailure(policy, "harness_failure", completedAt)); + return { state: trimTerminalRuns(next, completedAt), value: undefined }; + }); +} + +async function runDispatchBatch( + configDir?: string, + options: { manualRunId?: string } = {}, +): Promise { + if (shutdownRequested) return; + const policy = loadLabAutomationPolicy(configDir); + const routes = loadLabAutomationRoutes(configDir); + if (!policy.enabled && options.manualRunId === undefined) return; + const config = loadPlannerConfig(); + const snapshot = loadLabAutomationState(configDir); + const planned = policy.enabled && options.manualRunId === undefined + ? planLabAutomationRuns({ policy, routes, state: snapshot, now: Date.now(), config, configDir }) + : []; + const maxDispatches = options.manualRunId ? 1 : Math.max(1, policy.maxConcurrentRuns); + for (let dispatched = 0; dispatched < maxDispatches; dispatched += 1) { + if (shutdownRequested) break; + const run = claimNextRun(policy, routes, planned, configDir, Date.now(), options.manualRunId); + if (!run) break; const controller = new AbortController(); inFlightControllers.set(run.runId, controller); + let result: DispatchResult | null = null; + let error: unknown; try { const deps: AutomationDispatchDeps = { ...dispatchDeps, @@ -105,36 +241,17 @@ async function runDispatchBatch(configDir?: string, now = Date.now()): Promise readConfigDiagnostics().config), abortSignal: controller.signal, }; - const result: DispatchResult = await dispatchLabAutomationRun(run, deps); - if (cancellingRunIds.has(run.runId) || controller.signal.aborted) { - state = transitionRun(state, run.runId, "cancelled", Date.now(), "cancelled"); - } else { - state = recordRunBudgetUse(state, result.liveRequest); - state = transitionRun(state, run.runId, result.terminalState, Date.now(), result.terminalCode); - if (result.terminalState !== "completed") { - state = setCooldown( - state, - run.runKey, - cooldownForFailure(policy, result.terminalCode, Date.now()), - ); - } - } - } catch (error) { - if (cancellingRunIds.has(run.runId) || (error instanceof LabAutomationError && error.code === "cancelled")) { - state = transitionRun(state, run.runId, "cancelled", Date.now(), "cancelled"); - } else { - const code = error instanceof Error ? error.message : "dispatch_failure"; - state = transitionRun(state, run.runId, "failed", Date.now(), code); - state = setCooldown(state, run.runKey, cooldownForFailure(policy, "harness_failure", Date.now())); - } + result = await dispatchLabAutomationRun(run, deps); + } catch (caught) { + error = caught; } finally { + const cancelled = cancellingRunIds.has(run.runId) || controller.signal.aborted; + finalizeRun(run, policy, configDir, result, error, cancelled); inFlightControllers.delete(run.runId); cancellingRunIds.delete(run.runId); } - state = trimTerminalRuns(state, Date.now()); - saveLabAutomationState(state, configDir); + if (options.manualRunId !== undefined) break; } - saveLabAutomationState(trimTerminalRuns(state, now), configDir); } /** Single bounded scheduler tick — plan, enqueue, dispatch within concurrency limits. */ @@ -152,9 +269,13 @@ export function startLabAutomationScheduler(configDir?: string): void { if (schedulerTimer) return; shutdownRequested = false; const policy = loadLabAutomationPolicy(configDir); - let state = loadLabAutomationState(configDir); - state = recoverLabAutomationState(policy, state, Date.now()); - saveLabAutomationState(state, configDir); + const routes = loadLabAutomationRoutes(configDir); + const now = Date.now(); + mutateLabAutomationState(configDir, (state) => { + let next = recoverLabAutomationState(policy, state, now); + next = reconcileQueuedState(next, policy, routes, now); + return { state: trimTerminalRuns(next, now), value: undefined }; + }); schedulerTimer = setInterval(() => { void runLabAutomationTick(configDir); }, LAB_AUTOMATION_HARD_MAX.schedulerTickMs); @@ -182,28 +303,29 @@ export async function enqueueManualLabRun( configDir?: string, ): Promise { const now = Date.now(); - let state = loadLabAutomationState(configDir); - state = enqueuePlannedRuns(state, [planned], "manual", now); - saveLabAutomationState(state, configDir); - const created = state.runs.find((row) => row.runKey === planned.runKey && row.trigger === "manual"); + const created = mutateLabAutomationState(configDir, (state) => { + const next = enqueuePlannedRuns(state, [planned], "manual", now); + const run = next.runs.find((row) => + row.runKey === planned.runKey && row.trigger === "manual" && row.state === "queued" + ) ?? null; + return { state: next, value: run }; + }); if (!created) return null; - await runLabAutomationTick(configDir); + // Manual execution is independent of automation enablement/layer toggles. + await runDispatchBatch(configDir, { manualRunId: created.runId }); return loadLabAutomationState(configDir).runs.find((row) => row.runId === created.runId) ?? created; } export function cancelLabAutomationRun(runId: string, configDir?: string): boolean { - const state = loadLabAutomationState(configDir); - const run = state.runs.find((row) => row.runId === runId); - if (!run) return false; - if (run.state === "queued") { - saveLabAutomationState(cancelQueuedRun(state, runId, Date.now()), configDir); - return true; - } - if (run.state === "running") { + const controller = inFlightControllers.get(runId); + if (controller) { cancellingRunIds.add(runId); - const controller = inFlightControllers.get(runId); - if (controller) controller.abort(new Error("cancelled")); + controller.abort(new Error("cancelled")); return true; } - return false; + return mutateLabAutomationState(configDir, (state) => { + const run = state.runs.find((row) => row.runId === runId); + if (!run || run.state !== "queued") return { state, value: false }; + return { state: cancelQueuedRun(state, runId, Date.now()), value: true }; + }); } From 60921ef21ddaf95791ba57d58beab13492204ea0 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:14:37 +0200 Subject: [PATCH 13/67] fix(lab): reconcile automation queue on policy changes --- .../management/lab-automation-routes.ts | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/server/management/lab-automation-routes.ts b/src/server/management/lab-automation-routes.ts index 9cb73b48be..ff01a0125a 100644 --- a/src/server/management/lab-automation-routes.ts +++ b/src/server/management/lab-automation-routes.ts @@ -3,7 +3,7 @@ * * - GET /api/lab/automation * - PUT /api/lab/automation - * - GET /api/lab/automation/runs + * - GET /api/lab/automation/runs * - POST /api/lab/automation/run * - POST /api/lab/automation/runs/:id/cancel * @@ -15,6 +15,7 @@ import { buildLabAutomationStatus, cancelLabAutomationRun, enqueueManualLabRun, + reconcileLabAutomationQueue, startLabAutomationScheduler, stopLabAutomationScheduler, } from "../../lab/automation/orchestrator"; @@ -39,11 +40,17 @@ const AUTOMATION_LAYERS: readonly LabAutomationLayer[] = [ "live_route_compatibility", "task_effectiveness", ]; +const UPDATE_KEYS = new Set(["policy", "routes"]); +const MANUAL_RUN_KEYS = new Set(["evidenceLayer", "scenarioId", "providerName", "modelId"]); function automationErrorResponse(code: string, message: string, status: number, ctx: ManagementContext): Response { return jsonResponse({ error: { code, message } }, status, ctx.req, ctx.config); } +function hasUnknownKeys(value: Record, allowed: ReadonlySet): boolean { + return Object.keys(value).some((key) => !allowed.has(key)); +} + function parseLimit(raw: string | null, ctx: ManagementContext): number | Response { if (raw === null) return 50; const trimmed = raw.trim(); @@ -56,6 +63,7 @@ function parseLimit(raw: string | null, ctx: ManagementContext): number | Respon } function applySchedulerPolicy(policy: LabAutomationPolicyV1, configDir?: string): void { + reconcileLabAutomationQueue(configDir); if (policy.enabled) { startLabAutomationScheduler(configDir); } else { @@ -99,6 +107,9 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise try { const body = await readManagementJsonBody(req); if (!isPlainRecord(body)) return automationErrorResponse("invalid_body", "body must be an object", 400, ctx); + if (hasUnknownKeys(body, MANUAL_RUN_KEYS)) { + return automationErrorResponse("invalid_body", "manual run body contains unknown fields", 400, ctx); + } const evidenceLayer = body.evidenceLayer; const scenarioId = body.scenarioId; if (typeof evidenceLayer !== "string" || !AUTOMATION_LAYERS.includes(evidenceLayer as LabAutomationLayer)) { @@ -137,9 +148,23 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise try { const body = await readManagementJsonBody(req); if (!isPlainRecord(body)) return automationErrorResponse("invalid_body", "body must be an object", 400, ctx); + if (hasUnknownKeys(body, UPDATE_KEYS)) { + return automationErrorResponse("invalid_body", "automation update contains unknown fields", 400, ctx); + } let policy = loadLabAutomationPolicy(configDir); if (body.policy !== undefined) { - policy = normalizeLabAutomationPolicyV1({ ...policy, ...body.policy as Record }); + if (!isPlainRecord(body.policy)) return automationErrorResponse("invalid_policy", "policy must be an object", 400, ctx); + const incoming = body.policy as Record; + const mergedLayers = incoming.layers === undefined + ? policy.layers + : isPlainRecord(incoming.layers) + ? { ...policy.layers, ...incoming.layers } + : incoming.layers; + policy = normalizeLabAutomationPolicyV1({ + ...policy, + ...incoming, + layers: mergedLayers, + }); saveLabAutomationPolicy(policy, configDir); } if (body.routes !== undefined) { From b84fdd23191017f9c9eb1e8e607a1509dfd0df77 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:15:47 +0200 Subject: [PATCH 14/67] fix(lab): carry dispatch identity enforcement context --- src/lab/automation/types.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lab/automation/types.ts b/src/lab/automation/types.ts index 68a882a1db..84b8205861 100644 --- a/src/lab/automation/types.ts +++ b/src/lab/automation/types.ts @@ -130,6 +130,8 @@ export interface AutomationDispatchDeps { patchExecutor?: import("../fabric/types").TrustedFabricPatchExecutor; loadConfig?: () => import("../../types").OcxConfig; abortSignal?: AbortSignal; + /** Orchestrator-set guard that requires the queued identity to match current frozen contracts. */ + enforceRunIdentity?: boolean; /** Test-only DNS seam; production uses destination policy resolution when unset. */ resolve?: import("../live/types").DnsResolver; } From 02aa4f282485a869af60f61bf8195dccfea3db96 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:16:19 +0200 Subject: [PATCH 15/67] fix(lab): enforce queued identity without breaking low-level test seams --- src/lab/automation/dispatch.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lab/automation/dispatch.ts b/src/lab/automation/dispatch.ts index e9fd6df61b..acf827e494 100644 --- a/src/lab/automation/dispatch.ts +++ b/src/lab/automation/dispatch.ts @@ -48,6 +48,12 @@ function runIdentityMatches(run: LabAutomationRunRecordV1, expected: ExpectedRun && run.scenarioManifestDigest === expected.scenarioManifestDigest; } +function shouldEnforceRunIdentity(run: LabAutomationRunRecordV1, deps: AutomationDispatchDeps): boolean { + // Production orchestration always sets the explicit guard. Canonical planned run keys also + // self-identify here so direct internal callers cannot accidentally bypass revalidation. + return deps.enforceRunIdentity === true || /^[0-9a-f]{64}$/i.test(run.runKey); +} + function contractChanged(): DispatchResult { return { terminalState: "blocked", @@ -111,7 +117,7 @@ export async function dispatchLabAutomationRun( scenarioVersion, scenarioManifestDigest: scenarioDigest, }; - if (!runIdentityMatches(run, expected)) return contractChanged(); + if (shouldEnforceRunIdentity(run, deps) && !runIdentityMatches(run, expected)) return contractChanged(); const result = await runScenario(caseRecord); assertNotCancelled(deps.abortSignal); persistConformanceResult(result, caseRecord, authority, { configDir }); @@ -170,7 +176,7 @@ export async function dispatchLabAutomationRun( scenarioVersion, scenarioManifestDigest: scenarioDigest, }; - if (!runIdentityMatches(run, expected)) return contractChanged(); + if (shouldEnforceRunIdentity(run, deps) && !runIdentityMatches(run, expected)) return contractChanged(); const routeContext = buildAutomationLiveRouteContext( resolved.route, routed.allowPrivateNetwork === true, From 1a14ec248056826adea3bb01b7402c21fca66330 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:17:38 +0200 Subject: [PATCH 16/67] fix(lab): close scheduler lifecycle and stale-plan races --- src/lab/automation/orchestrator.ts | 44 +++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/lab/automation/orchestrator.ts b/src/lab/automation/orchestrator.ts index 5f1067603c..265582605d 100644 --- a/src/lab/automation/orchestrator.ts +++ b/src/lab/automation/orchestrator.ts @@ -141,16 +141,19 @@ function claimNextRun( policy: LabAutomationPolicyV1, routes: LabAutomationRoutesV1, planned: ReturnType, + enqueuePlan: boolean, configDir: string | undefined, now: number, manualRunId?: string, ): LabAutomationRunRecordV1 | null { return mutateLabAutomationState(configDir, (loaded) => { let state = rollBudgetWindow(loaded, now); - state = reconcileQueuedState(state, policy, routes, now); - if (policy.enabled && manualRunId === undefined) { + if (policy.enabled && manualRunId === undefined && enqueuePlan) { state = enqueuePlannedRuns(state, planned, "scheduled", now); } + // Policy/route checks must happen after enqueue as well: a plan can become stale between + // the pure planning snapshot and this atomic state claim. + state = reconcileQueuedState(state, policy, routes, now); if (isRunBudgetExhausted(policy, state, now)) { return { state: trimTerminalRuns(state, now), value: null }; } @@ -217,18 +220,38 @@ async function runDispatchBatch( options: { manualRunId?: string } = {}, ): Promise { if (shutdownRequested) return; - const policy = loadLabAutomationPolicy(configDir); - const routes = loadLabAutomationRoutes(configDir); - if (!policy.enabled && options.manualRunId === undefined) return; + const initialPolicy = loadLabAutomationPolicy(configDir); + const initialRoutes = loadLabAutomationRoutes(configDir); const config = loadPlannerConfig(); const snapshot = loadLabAutomationState(configDir); - const planned = policy.enabled && options.manualRunId === undefined - ? planLabAutomationRuns({ policy, routes, state: snapshot, now: Date.now(), config, configDir }) + const planned = initialPolicy.enabled && options.manualRunId === undefined + ? planLabAutomationRuns({ + policy: initialPolicy, + routes: initialRoutes, + state: snapshot, + now: Date.now(), + config, + configDir, + }) : []; - const maxDispatches = options.manualRunId ? 1 : Math.max(1, policy.maxConcurrentRuns); + let enqueuePlan = true; + const maxDispatches = options.manualRunId ? 1 : Math.max(1, initialPolicy.maxConcurrentRuns); for (let dispatched = 0; dispatched < maxDispatches; dispatched += 1) { if (shutdownRequested) break; - const run = claimNextRun(policy, routes, planned, configDir, Date.now(), options.manualRunId); + // Re-read operational policy before every claim so disabling a layer/route stops the next + // queued run immediately rather than after the whole batch. + const policy = loadLabAutomationPolicy(configDir); + const routes = loadLabAutomationRoutes(configDir); + const run = claimNextRun( + policy, + routes, + planned, + enqueuePlan, + configDir, + Date.now(), + options.manualRunId, + ); + enqueuePlan = false; if (!run) break; const controller = new AbortController(); inFlightControllers.set(run.runId, controller); @@ -240,6 +263,7 @@ async function runDispatchBatch( configDir, loadConfig: dispatchDeps.loadConfig ?? (() => readConfigDiagnostics().config), abortSignal: controller.signal, + enforceRunIdentity: true, }; result = await dispatchLabAutomationRun(run, deps); } catch (caught) { @@ -282,8 +306,8 @@ export function startLabAutomationScheduler(configDir?: string): void { schedulerTimer.unref?.(); } +/** Stop periodic scheduling only. Process shutdown is a separate explicit signal. */ export function stopLabAutomationScheduler(): void { - shutdownRequested = true; if (schedulerTimer) { clearInterval(schedulerTimer); schedulerTimer = null; From 17fb6625d896cba35ce546c4ae9bd23728742f86 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:21:03 +0200 Subject: [PATCH 17/67] fix(lab): ignore non-authoritative automation update fields --- src/server/management/lab-automation-routes.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/server/management/lab-automation-routes.ts b/src/server/management/lab-automation-routes.ts index ff01a0125a..dbb0584399 100644 --- a/src/server/management/lab-automation-routes.ts +++ b/src/server/management/lab-automation-routes.ts @@ -40,7 +40,6 @@ const AUTOMATION_LAYERS: readonly LabAutomationLayer[] = [ "live_route_compatibility", "task_effectiveness", ]; -const UPDATE_KEYS = new Set(["policy", "routes"]); const MANUAL_RUN_KEYS = new Set(["evidenceLayer", "scenarioId", "providerName", "modelId"]); function automationErrorResponse(code: string, message: string, status: number, ctx: ManagementContext): Response { @@ -148,9 +147,8 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise try { const body = await readManagementJsonBody(req); if (!isPlainRecord(body)) return automationErrorResponse("invalid_body", "body must be an object", 400, ctx); - if (hasUnknownKeys(body, UPDATE_KEYS)) { - return automationErrorResponse("invalid_body", "automation update contains unknown fields", 400, ctx); - } + // Only policy/routes are authoritative. Other top-level properties are ignored rather than + // treated as injectable process-local capabilities (for example a fake routeExecutor). let policy = loadLabAutomationPolicy(configDir); if (body.policy !== undefined) { if (!isPlainRecord(body.policy)) return automationErrorResponse("invalid_policy", "policy must be an object", 400, ctx); From 3110fd12ee6951613c26bc82be92f59f86179cb9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:22:10 +0200 Subject: [PATCH 18/67] fix(lab): rebuild missing projection before scheduled planning --- src/lab/automation/orchestrator.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/lab/automation/orchestrator.ts b/src/lab/automation/orchestrator.ts index 265582605d..67204049f3 100644 --- a/src/lab/automation/orchestrator.ts +++ b/src/lab/automation/orchestrator.ts @@ -1,4 +1,6 @@ import { readConfigDiagnostics } from "../../config"; +import { queryLabStatus } from "../query"; +import { rebuildLabProjection } from "../projection/rebuild"; import { planLabAutomationRuns } from "./planner"; import { loadLabAutomationPolicy, @@ -137,6 +139,22 @@ function loadPlannerConfig(): import("../../types").OcxConfig | undefined { } } +/** + * Scheduled planning is projection-backed. Rebuild once when the disposable SQLite projection + * is missing/incompatible; if replay/rebuild cannot establish a readable projection, fail closed + * instead of treating every scenario as missing and generating provider traffic. + */ +function ensureAutomationProjection(configDir?: string): boolean { + const status = queryLabStatus(configDir); + if (status.projectionAvailable) return true; + try { + rebuildLabProjection(configDir); + } catch { + return false; + } + return queryLabStatus(configDir).projectionAvailable; +} + function claimNextRun( policy: LabAutomationPolicyV1, routes: LabAutomationRoutesV1, @@ -151,8 +169,6 @@ function claimNextRun( if (policy.enabled && manualRunId === undefined && enqueuePlan) { state = enqueuePlannedRuns(state, planned, "scheduled", now); } - // Policy/route checks must happen after enqueue as well: a plan can become stale between - // the pure planning snapshot and this atomic state claim. state = reconcileQueuedState(state, policy, routes, now); if (isRunBudgetExhausted(policy, state, now)) { return { state: trimTerminalRuns(state, now), value: null }; @@ -222,6 +238,10 @@ async function runDispatchBatch( if (shutdownRequested) return; const initialPolicy = loadLabAutomationPolicy(configDir); const initialRoutes = loadLabAutomationRoutes(configDir); + if (initialPolicy.enabled && options.manualRunId === undefined && !ensureAutomationProjection(configDir)) { + reconcileLabAutomationQueue(configDir); + return; + } const config = loadPlannerConfig(); const snapshot = loadLabAutomationState(configDir); const planned = initialPolicy.enabled && options.manualRunId === undefined @@ -238,8 +258,6 @@ async function runDispatchBatch( const maxDispatches = options.manualRunId ? 1 : Math.max(1, initialPolicy.maxConcurrentRuns); for (let dispatched = 0; dispatched < maxDispatches; dispatched += 1) { if (shutdownRequested) break; - // Re-read operational policy before every claim so disabling a layer/route stops the next - // queued run immediately rather than after the whole batch. const policy = loadLabAutomationPolicy(configDir); const routes = loadLabAutomationRoutes(configDir); const run = claimNextRun( From 247ec048dfae6fb1a1c9bdf238431e72a8235688 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:35:39 +0200 Subject: [PATCH 19/67] test(lab): use canonical queued live runs --- tests/lab-automation.test.ts | 90 +++++++++++------------------------- 1 file changed, 26 insertions(+), 64 deletions(-) diff --git a/tests/lab-automation.test.ts b/tests/lab-automation.test.ts index bbaf3bcf47..d86ace4c7b 100644 --- a/tests/lab-automation.test.ts +++ b/tests/lab-automation.test.ts @@ -13,7 +13,7 @@ import { saveLabAutomationRoutes, saveLabAutomationState, } from "../src/lab/automation/persistence"; -import { planLabAutomationRuns } from "../src/lab/automation/planner"; +import { planLabAutomationRuns, planManualLabRun } from "../src/lab/automation/planner"; import { enqueuePlannedRuns, selectDispatchableRuns } from "../src/lab/automation/queue"; import { recoverLabAutomationState } from "../src/lab/automation/recovery"; import { @@ -447,41 +447,20 @@ describe("CL-08 lab automation", () => { schemaVersion: 1, routes: [{ providerName: "fixture-provider", modelId: "fixture-model" }], }, home); - const authority = loadLiveCaseAuthority(); - const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; - const now = Date.now(); - saveLabAutomationState({ - schemaVersion: 1, - runs: [{ - runId: "live-run-1", - runKey: "live-run-key", - state: "queued", - evidenceLayer: "live_route_compatibility", - suiteId: scenario.suite, - suiteVersion: authority.manifestDefaults.suiteVersion, - suiteManifestDigest: "digest", - scenarioId: scenario.id, - scenarioVersion: authority.manifestDefaults.version, - scenarioManifestDigest: "digest", - subjectId: "pending", - reason: "missing", - priority: 0, - eligibleAt: now, - trigger: "scheduled", - createdAt: now, - updatedAt: now, - providerName: "fixture-provider", - modelId: "fixture-model", - }], - budgetWindowStartedAt: now, - runsThisHour: 0, - liveRequestsThisHour: 0, - cooldownUntilByKey: {}, - }, home); + const plan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId: "responses-core.live.basic-turn", + providerName: "fixture-provider", + modelId: "fixture-model", + config, + configDir: home, + }); + const state = enqueuePlannedRuns(loadLabAutomationState(home), [plan], "scheduled", Date.now()); + saveLabAutomationState(state, home); await runLabAutomationTick(home); expect(invokes).toBe(1); - const state = loadLabAutomationState(home); - expect(state.runs[0]?.state === "completed" || state.runs[0]?.state === "failed").toBe(true); + const stored = loadLabAutomationState(home).runs.find((row) => row.runKey === plan.runKey); + expect(stored?.state === "completed" || stored?.state === "failed").toBe(true); }); }); @@ -513,40 +492,23 @@ describe("CL-08 lab automation", () => { maxLiveRequestsPerHour: 0, }; saveLabAutomationPolicy(policy, home); - const authority = loadLiveCaseAuthority(); - const scenario = authority.cases.find((c) => c.id === "responses-core.live.basic-turn")!; - const now = Date.now(); - saveLabAutomationState({ + saveLabAutomationRoutes({ schemaVersion: 1, - runs: [{ - runId: "budget-live", - runKey: "budget-key", - state: "queued", - evidenceLayer: "live_route_compatibility", - suiteId: scenario.suite, - suiteVersion: authority.manifestDefaults.suiteVersion, - suiteManifestDigest: "digest", - scenarioId: scenario.id, - scenarioVersion: authority.manifestDefaults.version, - scenarioManifestDigest: "digest", - subjectId: "pending", - reason: "missing", - priority: 0, - eligibleAt: now, - trigger: "scheduled", - createdAt: now, - updatedAt: now, - providerName: "fixture-provider", - modelId: "fixture-model", - }], - budgetWindowStartedAt: now, - runsThisHour: 0, - liveRequestsThisHour: 0, - cooldownUntilByKey: {}, + routes: [{ providerName: "fixture-provider", modelId: "fixture-model" }], }, home); + const plan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId: "responses-core.live.basic-turn", + providerName: "fixture-provider", + modelId: "fixture-model", + config, + configDir: home, + }); + const state = enqueuePlannedRuns(loadLabAutomationState(home), [plan], "scheduled", Date.now()); + saveLabAutomationState(state, home); await runLabAutomationTick(home); expect(invokes).toBe(0); - expect(loadLabAutomationState(home).runs[0]?.state).toBe("queued"); + expect(loadLabAutomationState(home).runs.find((row) => row.runKey === plan.runKey)?.state).toBe("queued"); }); }); From ec5171345fdbab22ee4993a5d17c65b744c35826 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:37:17 +0200 Subject: [PATCH 20/67] fix(lab): complete CL-08 CLI execution controls --- src/cli/lab.ts | 97 ++++++++++++++++---------------------------------- 1 file changed, 31 insertions(+), 66 deletions(-) diff --git a/src/cli/lab.ts b/src/cli/lab.ts index a9e88b52e4..44dd85b5f0 100644 --- a/src/cli/lab.ts +++ b/src/cli/lab.ts @@ -1,9 +1,10 @@ /** - * `ocx lab` — read-only Compatibility Lab inspection (CL-04). + * `ocx lab` — Compatibility Lab inspection and explicit CL-08 automation controls. * - * Local SQLite projection reads; no daemon, network, probes, or rebuilds. + * Read commands use the local SQLite projection. Automation mutations and manual runs are + * explicit operator actions; read commands never start probes or scheduler ticks. */ -import { getConfigDir } from "../config"; +import { getConfigDir, readConfigDiagnostics } from "../config"; import { ARTIFACT_CLASSES, EVIDENCE_LAYERS, @@ -43,10 +44,11 @@ import { takeIntegerOption, takeOption, } from "./runtime-api"; -import { readConfigDiagnostics } from "../config"; import { buildLabAutomationStatus, enqueueManualLabRun, + reconcileLabAutomationQueue, + setLabAutomationDispatchDeps, startLabAutomationScheduler, stopLabAutomationScheduler, } from "../lab/automation/orchestrator"; @@ -58,6 +60,7 @@ import { import { planManualLabRun } from "../lab/automation/planner"; import { listLabAutomationRuns } from "../lab/automation/runs-query"; import type { LabAutomationLayer } from "../lab/automation/types"; +import { createProductionLabRouteExecutor } from "../lib/lab-live-route-production"; const USAGE = `Usage: ocx lab status [--json] @@ -236,14 +239,7 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P const limit = takeIntegerOption(rest, "--limit", { min: 1 }); const cursor = takeOption(rest, "--cursor"); rejectArgs(rest, USAGE); - const page = queryLabVerdicts({ - subjectId, - layer, - suiteId, - verdict, - from, - to, - }, cursor, limit, configDir); + const page = queryLabVerdicts({ subjectId, layer, suiteId, verdict, from, to }, cursor, limit, configDir); printData(page, wantsJson, verdictLines(page)); return; } @@ -268,52 +264,23 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P } case "observations": { const subjectId = takeOption(rest, "--subject"); - const layer = takeEnumOption( - rest, - "--layer", - EVIDENCE_LAYERS, - "--layer must be a supported evidence layer", - ); + const layer = takeEnumOption(rest, "--layer", EVIDENCE_LAYERS, "--layer must be a supported evidence layer"); const suiteId = takeOption(rest, "--suite"); const scenarioId = takeOption(rest, "--scenario"); - const outcome = takeEnumOption( - rest, - "--outcome", - OUTCOMES, - "--outcome must be a supported observation outcome", - ); - const executionMode = takeEnumOption( - rest, - "--execution-mode", - EXECUTION_MODES, - "--execution-mode must be a supported execution mode", - ); + const outcome = takeEnumOption(rest, "--outcome", OUTCOMES, "--outcome must be a supported observation outcome"); + const executionMode = takeEnumOption(rest, "--execution-mode", EXECUTION_MODES, "--execution-mode must be a supported execution mode"); const from = takeIntegerOption(rest, "--from", { min: 0 }); const to = takeIntegerOption(rest, "--to", { min: 0 }); assertRange(from, to); const limit = takeIntegerOption(rest, "--limit", { min: 1 }); const cursor = takeOption(rest, "--cursor"); rejectArgs(rest, USAGE); - const page = queryLabObservations({ - subjectId, - layer, - suiteId, - scenarioId, - outcome, - executionMode, - from, - to, - }, cursor, limit, configDir); + const page = queryLabObservations({ subjectId, layer, suiteId, scenarioId, outcome, executionMode, from, to }, cursor, limit, configDir); printData(page, wantsJson, observationLines(page)); return; } case "events": { - const eventKind = takeEnumOption( - rest, - "--event-kind", - EVENT_KINDS, - "--event-kind must be a supported lab event kind", - ); + const eventKind = takeEnumOption(rest, "--event-kind", EVENT_KINDS, "--event-kind must be a supported lab event kind"); const subjectId = takeOption(rest, "--subject"); const from = takeIntegerOption(rest, "--from", { min: 0 }); const to = takeIntegerOption(rest, "--to", { min: 0 }); @@ -346,18 +313,8 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P return; } case "artifacts": { - const status = takeEnumOption( - rest, - "--status", - ARTIFACT_STATUSES, - "--status must be present, corrupt, or purged_unavailable", - ); - const artifactClass = takeEnumOption( - rest, - "--artifact-class", - ARTIFACT_CLASSES, - "--artifact-class must be a supported artifact class", - ); + const status = takeEnumOption(rest, "--status", ARTIFACT_STATUSES, "--status must be present, corrupt, or purged_unavailable"); + const artifactClass = takeEnumOption(rest, "--artifact-class", ARTIFACT_CLASSES, "--artifact-class must be a supported artifact class"); const limit = takeIntegerOption(rest, "--limit", { min: 1 }); const cursor = takeOption(rest, "--cursor"); rejectArgs(rest, USAGE); @@ -376,12 +333,7 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P return; } case "catalog": { - const layer = takeEnumOption( - rest, - "--layer", - EVIDENCE_LAYERS, - "--layer must be a supported evidence layer", - ); + const layer = takeEnumOption(rest, "--layer", EVIDENCE_LAYERS, "--layer must be a supported evidence layer"); const suiteId = takeOption(rest, "--suite"); rejectArgs(rest, USAGE); const scenarios = queryLabCatalogEntries({ layer, suiteId }); @@ -398,19 +350,22 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P return; } case "enable": { + const enableProtocol = takeFlag(automationRest, "--protocol"); const enableLive = takeFlag(automationRest, "--live"); rejectArgs(automationRest, USAGE); + const selectionExplicit = enableProtocol || enableLive; const policy = { ...loadLabAutomationPolicy(configDir), enabled: true, layers: { - protocolConformance: true, + protocolConformance: selectionExplicit ? enableProtocol : true, liveRouteCompatibility: enableLive, taskEffectiveness: false, }, taskEffectivenessBackgroundEnabled: false, }; saveLabAutomationPolicy(policy, configDir); + reconcileLabAutomationQueue(configDir); startLabAutomationScheduler(configDir); const status = buildLabAutomationStatus(configDir); printData(status, wantsJson, automationStatusLines(status)); @@ -420,6 +375,7 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P rejectArgs(automationRest, USAGE); const policy = { ...loadLabAutomationPolicy(configDir), enabled: false }; saveLabAutomationPolicy(policy, configDir); + reconcileLabAutomationQueue(configDir); stopLabAutomationScheduler(); const status = buildLabAutomationStatus(configDir); printData(status, wantsJson, automationStatusLines(status)); @@ -449,12 +405,21 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P const modelId = takeOption(rest, "--model"); rejectArgs(rest, USAGE); if (!layer || !scenarioId) throw new CliUsageError("--layer and --scenario are required", USAGE); + const configSnapshot = readConfigDiagnostics().config; + if (layer === "live_route_compatibility") { + const loadConfig = () => configSnapshot; + setLabAutomationDispatchDeps({ + configDir, + loadConfig, + routeExecutor: createProductionLabRouteExecutor({ configDir, loadConfig }), + }); + } const planned = planManualLabRun({ evidenceLayer: layer, scenarioId, providerName, modelId, - config: readConfigDiagnostics().config, + config: configSnapshot, configDir, }); const record = await enqueueManualLabRun(planned, configDir); From 6706ac6e6d685de0c8639c0fc8f8cb80e292e6d1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:37:29 +0200 Subject: [PATCH 21/67] fix(lab): harden automation review bounds --- src/lab/automation/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lab/automation/constants.ts b/src/lab/automation/constants.ts index e9d1540403..960d467913 100644 --- a/src/lab/automation/constants.ts +++ b/src/lab/automation/constants.ts @@ -11,6 +11,7 @@ export const LAB_AUTOMATION_HARD_MAX = Object.freeze({ maxLiveRequestsPerHour: 30, failureCooldownMs: 24 * 60 * 60 * 1000, blockedCooldownMs: 7 * 24 * 60 * 60 * 1000, + terminalRunRetentionMs: 7 * 24 * 60 * 60 * 1000, maxQueuedRuns: 256, maxPersistedRuns: 512, maxAutomationRoutes: 64, From a697421af579b5dcfec204e31d3085285dd7f63f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:37:58 +0200 Subject: [PATCH 22/67] fix(lab): reject zero concurrency limits --- src/lab/automation/policy.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/lab/automation/policy.ts b/src/lab/automation/policy.ts index c487f0714b..0c44426131 100644 --- a/src/lab/automation/policy.ts +++ b/src/lab/automation/policy.ts @@ -49,6 +49,14 @@ function assertNonNegativeInt(value: unknown, field: string, max: number): numbe return value; } +function assertPositiveInt(value: unknown, field: string, max: number): number { + const parsed = assertNonNegativeInt(value, field, max); + if (parsed === 0) { + throw new LabAutomationError(`policy field ${field} must be at least 1`, "invalid_policy"); + } + return parsed; +} + /** Validate and normalize automation policy with a closed schema and hard upper bounds. */ export function normalizeLabAutomationPolicyV1(raw: unknown): LabAutomationPolicyV1 { if (!raw || typeof raw !== "object" || Array.isArray(raw)) { @@ -83,17 +91,17 @@ export function normalizeLabAutomationPolicyV1(raw: unknown): LabAutomationPolic "refreshBeforeStaleMs", LAB_AUTOMATION_HARD_MAX.refreshBeforeStaleMs, ), - maxConcurrentRuns: assertNonNegativeInt( + maxConcurrentRuns: assertPositiveInt( obj.maxConcurrentRuns, "maxConcurrentRuns", LAB_AUTOMATION_HARD_MAX.maxConcurrentRuns, ), - maxConcurrentLiveRuns: assertNonNegativeInt( + maxConcurrentLiveRuns: assertPositiveInt( obj.maxConcurrentLiveRuns, "maxConcurrentLiveRuns", LAB_AUTOMATION_HARD_MAX.maxConcurrentLiveRuns, ), - maxConcurrentRunsPerRoute: assertNonNegativeInt( + maxConcurrentRunsPerRoute: assertPositiveInt( obj.maxConcurrentRunsPerRoute, "maxConcurrentRunsPerRoute", LAB_AUTOMATION_HARD_MAX.maxConcurrentRunsPerRoute, From ab7e689c1adcdcec2b3f0a38a1bb13d1d6e12500 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:38:23 +0200 Subject: [PATCH 23/67] fix(lab): bound cooldown persistence --- src/lab/automation/cooldown.ts | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/lab/automation/cooldown.ts b/src/lab/automation/cooldown.ts index 78bc1745a0..7bc4197514 100644 --- a/src/lab/automation/cooldown.ts +++ b/src/lab/automation/cooldown.ts @@ -1,3 +1,4 @@ +import { LAB_AUTOMATION_HARD_MAX } from "./constants"; import type { LabAutomationPolicyV1, LabAutomationStateV1 } from "./types"; export function cooldownActive(state: LabAutomationStateV1, key: string, now: number): boolean { @@ -5,18 +6,30 @@ export function cooldownActive(state: LabAutomationStateV1, key: string, now: nu return typeof until === "number" && until > now; } +function activeCooldowns(state: LabAutomationStateV1, now: number): Record { + const next: Record = {}; + for (const [key, until] of Object.entries(state.cooldownUntilByKey)) { + if (until > now) next[key] = until; + } + return next; +} + +export function cooldownCapacityExhausted(state: LabAutomationStateV1, now: number): boolean { + return Object.keys(activeCooldowns(state, now)).length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns; +} + export function setCooldown( state: LabAutomationStateV1, key: string, untilMs: number, + now: number, ): LabAutomationStateV1 { - return { - ...state, - cooldownUntilByKey: { - ...state.cooldownUntilByKey, - [key]: untilMs, - }, - }; + const next = activeCooldowns(state, now); + if (!(key in next) && Object.keys(next).length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { + return { ...state, cooldownUntilByKey: next }; + } + next[key] = untilMs; + return { ...state, cooldownUntilByKey: next }; } export function clearCooldown(state: LabAutomationStateV1, key: string): LabAutomationStateV1 { From b8d9f876c3b50eeb9c9caa88aa05fe72a86d5d6a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:39:01 +0200 Subject: [PATCH 24/67] fix(lab): retain rolling budget evidence --- src/lab/automation/queue.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/lab/automation/queue.ts b/src/lab/automation/queue.ts index 7d8f559189..7e3e91da70 100644 --- a/src/lab/automation/queue.ts +++ b/src/lab/automation/queue.ts @@ -20,12 +20,19 @@ function isTerminal(run: LabAutomationRunRecordV1): boolean { return TERMINAL_STATES.has(run.state); } -function evictOldestTerminal(runs: LabAutomationRunRecordV1[]): boolean { +function isRollingBudgetEvidence(run: LabAutomationRunRecordV1, now: number): boolean { + if (typeof run.startedAt !== "number") return false; + const cutoff = now - LAB_AUTOMATION_HARD_MAX.budgetWindowMs; + return run.startedAt > cutoff && run.startedAt <= now; +} + +/** Evict only disposable terminal history; trailing-window attempts are budget authority. */ +function evictOldestTerminal(runs: LabAutomationRunRecordV1[], now: number): boolean { let oldestIndex = -1; let oldestTime = Number.POSITIVE_INFINITY; for (let index = 0; index < runs.length; index += 1) { const run = runs[index]!; - if (!isTerminal(run)) continue; + if (!isTerminal(run) || isRollingBudgetEvidence(run, now)) continue; const terminalAt = run.completedAt ?? run.updatedAt; if (terminalAt < oldestTime) { oldestTime = terminalAt; @@ -70,7 +77,7 @@ export function enqueuePlannedRuns( if (existingActiveKeys.has(plan.runKey)) continue; if (queuedCount >= LAB_AUTOMATION_HARD_MAX.maxQueuedRuns) break; while (runs.length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { - if (!evictOldestTerminal(runs)) return { ...state, runs }; + if (!evictOldestTerminal(runs, now)) return { ...state, runs }; } runs.push({ runId: randomUUID(), @@ -162,14 +169,17 @@ export function selectDispatchableRuns( } export function trimTerminalRuns(state: LabAutomationStateV1, now: number): LabAutomationStateV1 { - const keepMs = 7 * 24 * 60 * 60 * 1000; + const keepMs = LAB_AUTOMATION_HARD_MAX.terminalRunRetentionMs; const runs = state.runs.filter((row) => { if (!isTerminal(row)) return true; + // Retention is longer than the rolling budget window, but keep the authority check explicit + // so a future retention change cannot erase attempts that still bound provider traffic. + if (isRollingBudgetEvidence(row, now)) return true; const completedAt = row.completedAt ?? row.updatedAt; return now - completedAt < keepMs; }); while (runs.length > LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { - if (!evictOldestTerminal(runs)) break; + if (!evictOldestTerminal(runs, now)) break; } return { ...state, runs }; } From 2eea0a6c043705736a8bb6ef64f165fce36eeb62 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:39:23 +0200 Subject: [PATCH 25/67] fix(lab): reject stale run cursors --- src/lab/automation/runs-query.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lab/automation/runs-query.ts b/src/lab/automation/runs-query.ts index 85fb43338d..597ca6e39e 100644 --- a/src/lab/automation/runs-query.ts +++ b/src/lab/automation/runs-query.ts @@ -1,5 +1,6 @@ import type { LabAutomationRunRecordV1, LabAutomationStateV1 } from "./types"; import { LAB_AUTOMATION_HARD_MAX } from "./constants"; +import { LabAutomationError } from "./types"; export interface LabAutomationRunsPage { items: LabAutomationRunRecordV1[]; @@ -20,7 +21,8 @@ export function listLabAutomationRuns( let start = 0; if (cursor) { const index = sorted.findIndex((row) => row.runId === cursor); - start = index >= 0 ? index + 1 : 0; + if (index < 0) throw new LabAutomationError("invalid or expired run cursor", "invalid_cursor"); + start = index + 1; } const slice = sorted.slice(start, start + capped); const hasMore = start + capped < sorted.length; From 7ae55c68663d791bd7cfe757931870386388e63d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:41:01 +0200 Subject: [PATCH 26/67] fix(lab): harden OAuth live-route dispatch --- src/lib/lab-live-route-production.ts | 30 ++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/lib/lab-live-route-production.ts b/src/lib/lab-live-route-production.ts index 1a4fa18f7b..5d31dbac6a 100644 --- a/src/lib/lab-live-route-production.ts +++ b/src/lib/lab-live-route-production.ts @@ -8,7 +8,11 @@ * @internal host integration only */ import { resolveEnvValue } from "../config"; -import { getValidAccessTokenSnapshot } from "../oauth"; +import { + getValidAccessTokenSnapshot, + OAuthLoginRequiredError, + UnsupportedOAuthProviderError, +} from "../oauth"; import type { OcxConfig } from "../types"; import { createCredentialLease } from "../lab/live/credential-lease"; import { @@ -50,11 +54,25 @@ async function buildLabProviderAuthHeaders( }; if (provider.authMode === "oauth") { + let protocol: string; + try { + protocol = new URL(routeContext.baseUrl).protocol; + } catch { + throw new TransportError("auth_blocked", "invalid OAuth provider destination"); + } + if (protocol !== "https:") { + throw new TransportError("auth_blocked", "OAuth lab probes require HTTPS"); + } try { const snapshot = await getValidAccessTokenSnapshot(routeContext.providerId); headers.Authorization = `Bearer ${snapshot.accessToken}`; - } catch { - throw new TransportError("auth_blocked", "oauth unavailable"); + } catch (error) { + if (error instanceof OAuthLoginRequiredError || error instanceof UnsupportedOAuthProviderError) { + throw new TransportError("auth_blocked", "oauth unavailable"); + } + // Non-terminal refresh failures (network, lock contention, provider transient errors) + // are harness/infrastructure failures rather than evidence that credentials are invalid. + throw new TransportError("harness_failure", "oauth refresh unavailable"); } } else { const apiKey = resolveEnvValue(provider.apiKey)?.trim(); @@ -82,6 +100,9 @@ export function createProductionLabRouteExecutor( deps: ProductionLabRouteExecutorDeps, ): TrustedLabRouteExecutor { return createHostIssuedLabRouteExecutor(async (input) => { + if (!input.initiatingRequest) { + throw new TransportError("harness_failure", "live scenario missing initiating request"); + } const config = deps.loadConfig(); const lease = createCredentialLease({ destination: input.destination, @@ -97,9 +118,6 @@ export function createProductionLabRouteExecutor( limits: input.limits, transportId: "lab-automation", }); - if (!input.initiatingRequest) { - throw new TransportError("harness_failure", "live scenario missing initiating request"); - } const path = liveUpstreamRequestPath(input.routeContext.upstreamProtocol); const response = await transport.request({ method: "POST", From 68c938f85de691b805c92236b09287e19c69ab86 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:42:45 +0200 Subject: [PATCH 27/67] fix(lab): fence automation state ownership --- src/lab/automation/persistence.ts | 107 +++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 17 deletions(-) diff --git a/src/lab/automation/persistence.ts b/src/lab/automation/persistence.ts index b29aac2025..b0cbd77a95 100644 --- a/src/lab/automation/persistence.ts +++ b/src/lab/automation/persistence.ts @@ -1,10 +1,10 @@ +import { randomUUID } from "node:crypto"; import { closeSync, existsSync, openSync, readFileSync, renameSync, - statSync, unlinkSync, writeFileSync, } from "node:fs"; @@ -60,6 +60,7 @@ const RUN_KEYS = new Set([ "modelId", ]); const RUN_STATES = new Set(["queued", "running", "completed", "blocked", "failed", "cancelled", "abandoned"]); +const TERMINAL_RUN_STATES = new Set(["completed", "blocked", "failed", "cancelled", "abandoned"]); const RUN_REASONS = new Set([ "automation_disabled", "layer_disabled", @@ -75,9 +76,13 @@ const RUN_REASONS = new Set([ "task_background_disabled", ]); const STATE_LOCK_WAIT_MS = 5_000; -const STATE_LOCK_STALE_MS = 30_000; const LOCK_SLEEP = new Int32Array(new SharedArrayBuffer(4)); +interface StateLockMeta { + pid: number; + token: string; +} + function assertClosedKeys( value: Record, allowed: ReadonlySet, @@ -145,34 +150,75 @@ function stateLockPath(configDir?: string): string { return `${labAutomationStatePath(configDir)}.lock`; } +function readStateLockMeta(lockPath: string): StateLockMeta | null { + try { + const parsed = JSON.parse(readFileSync(lockPath, "utf8")) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; + const row = parsed as Record; + if (!Number.isSafeInteger(row.pid) || (row.pid as number) <= 0) return null; + if (typeof row.token !== "string" || row.token.length < 16 || row.token.length > 128) return null; + return { pid: row.pid as number, token: row.token }; + } catch { + return null; + } +} + +function pidDefinitelyDead(pid: number): boolean { + try { + process.kill(pid, 0); + return false; + } catch (error) { + const code = error && typeof error === "object" && "code" in error + ? String((error as { code?: unknown }).code) + : undefined; + return code === "ESRCH"; + } +} + +function releaseStateLock(lockPath: string, token: string): void { + try { + if (readStateLockMeta(lockPath)?.token === token) unlinkSync(lockPath); + } catch { + /* already released/cleaned */ + } +} + +function reclaimDeadStateLock(lockPath: string): boolean { + const observed = readStateLockMeta(lockPath); + if (!observed || !pidDefinitelyDead(observed.pid)) return false; + // Re-check ownership immediately before deletion. A live/paused process is never reclaimed; + // token-checked release also prevents an old owner from deleting a successor's lock. + const current = readStateLockMeta(lockPath); + if (!current || current.pid !== observed.pid || current.token !== observed.token) return false; + try { + unlinkSync(lockPath); + return true; + } catch { + return false; + } +} + function acquireStateLock(configDir?: string): () => void { ensureLabDirs(configDir); const lockPath = stateLockPath(configDir); const deadline = Date.now() + STATE_LOCK_WAIT_MS; while (true) { + const token = randomUUID(); try { const fd = openSync(lockPath, "wx", 0o600); try { - writeFileSync(fd, `${process.pid}:${Date.now()}`, { encoding: "utf8" }); + const meta: StateLockMeta = { pid: process.pid, token }; + writeFileSync(fd, JSON.stringify(meta), { encoding: "utf8" }); } finally { closeSync(fd); } - return () => { - try { unlinkSync(lockPath); } catch { /* already released/cleaned */ } - }; + return () => releaseStateLock(lockPath, token); } catch (error) { const code = error && typeof error === "object" && "code" in error ? String((error as { code?: unknown }).code) : undefined; if (code !== "EEXIST") throw new LabAutomationError("automation state lock failed", "state_lock_failed"); - try { - if (Date.now() - statSync(lockPath).mtimeMs > STATE_LOCK_STALE_MS) { - unlinkSync(lockPath); - continue; - } - } catch { - continue; - } + if (reclaimDeadStateLock(lockPath)) continue; if (Date.now() >= deadline) { throw new LabAutomationError("automation state is busy", "state_lock_busy"); } @@ -195,7 +241,7 @@ export function saveLabAutomationPolicy(policy: LabAutomationPolicyV1, configDir atomicWriteJson(labAutomationPolicyPath(configDir), normalized); } -function normalizeRoutes(raw: unknown): LabAutomationRoutesV1 { +export function normalizeLabAutomationRoutesV1(raw: unknown): LabAutomationRoutesV1 { if (!raw || typeof raw !== "object" || Array.isArray(raw)) { throw new LabAutomationError("automation routes must be an object", "invalid_routes"); } @@ -231,12 +277,12 @@ export function loadLabAutomationRoutes(configDir?: string): LabAutomationRoutes ensureLabDirs(configDir); const raw = readJsonFile(labAutomationRoutesPath(configDir), "invalid_routes"); if (raw === undefined) return defaultLabAutomationRoutesV1(); - return normalizeRoutes(raw); + return normalizeLabAutomationRoutesV1(raw); } export function saveLabAutomationRoutes(routes: LabAutomationRoutesV1, configDir?: string): void { ensureLabDirs(configDir); - atomicWriteJson(labAutomationRoutesPath(configDir), normalizeRoutes(routes)); + atomicWriteJson(labAutomationRoutesPath(configDir), normalizeLabAutomationRoutesV1(routes)); } function optionalTimestamp(row: Record, key: "startedAt" | "completedAt", index: number): number | undefined { @@ -283,9 +329,12 @@ function normalizeRunRecord(raw: unknown, index: number): LabAutomationRunRecord const completedAt = optionalTimestamp(row, "completedAt", index); if (updatedAt < createdAt) throw new LabAutomationError(`updatedAt precedes createdAt at ${index}`, "invalid_state"); if (startedAt !== undefined && startedAt < createdAt) throw new LabAutomationError(`startedAt precedes createdAt at ${index}`, "invalid_state"); + if (completedAt !== undefined && completedAt < createdAt) throw new LabAutomationError(`completedAt precedes createdAt at ${index}`, "invalid_state"); if (completedAt !== undefined && startedAt !== undefined && completedAt < startedAt) { throw new LabAutomationError(`completedAt precedes startedAt at ${index}`, "invalid_state"); } + if (startedAt !== undefined && updatedAt < startedAt) throw new LabAutomationError(`updatedAt precedes startedAt at ${index}`, "invalid_state"); + if (completedAt !== undefined && updatedAt < completedAt) throw new LabAutomationError(`updatedAt precedes completedAt at ${index}`, "invalid_state"); const terminalCode = row.terminalCode === undefined ? undefined : assertBoundedString(row.terminalCode, `terminalCode at ${index}`, "invalid_state", 256); @@ -295,6 +344,17 @@ function normalizeRunRecord(raw: unknown, index: number): LabAutomationRunRecord const modelId = row.modelId === undefined ? undefined : assertBoundedString(row.modelId, `modelId at ${index}`, "invalid_state", 512); + + if (state === "queued" && (startedAt !== undefined || completedAt !== undefined || terminalCode !== undefined)) { + throw new LabAutomationError(`queued run has lifecycle terminal fields at ${index}`, "invalid_state"); + } + if (state === "running" && (completedAt !== undefined || terminalCode !== undefined)) { + throw new LabAutomationError(`running run has terminal fields at ${index}`, "invalid_state"); + } + if (TERMINAL_RUN_STATES.has(state) && completedAt === undefined) { + throw new LabAutomationError(`terminal run missing completedAt at ${index}`, "invalid_state"); + } + return { runId, runKey, @@ -321,6 +381,18 @@ function normalizeRunRecord(raw: unknown, index: number): LabAutomationRunRecord }; } +function assertStateRunInvariants(runs: LabAutomationRunRecordV1[]): void { + const runIds = new Set(); + const activeRunKeys = new Set(); + for (const run of runs) { + if (runIds.has(run.runId)) throw new LabAutomationError(`duplicate runId ${run.runId}`, "invalid_state"); + runIds.add(run.runId); + if (run.state !== "queued" && run.state !== "running") continue; + if (activeRunKeys.has(run.runKey)) throw new LabAutomationError(`duplicate active runKey ${run.runKey}`, "invalid_state"); + activeRunKeys.add(run.runKey); + } +} + function normalizeState(raw: unknown): LabAutomationStateV1 { if (!raw || typeof raw !== "object" || Array.isArray(raw)) { throw new LabAutomationError("automation state must be an object", "invalid_state"); @@ -333,6 +405,7 @@ function normalizeState(raw: unknown): LabAutomationStateV1 { throw new LabAutomationError("too many persisted runs", "invalid_state"); } const runs = obj.runs.map((row, index) => normalizeRunRecord(row, index)); + assertStateRunInvariants(runs); const cooldownUntilByKey: Record = {}; if (!obj.cooldownUntilByKey || typeof obj.cooldownUntilByKey !== "object" || Array.isArray(obj.cooldownUntilByKey)) { throw new LabAutomationError("invalid cooldownUntilByKey", "invalid_state"); From 32878f5f3bdd0b5d2def7883ff1bb0ca08ea2b5f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:44:25 +0200 Subject: [PATCH 28/67] fix(lab): isolate scheduler ownership and cancellation --- src/lab/automation/orchestrator.ts | 108 +++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 27 deletions(-) diff --git a/src/lab/automation/orchestrator.ts b/src/lab/automation/orchestrator.ts index 67204049f3..c1f82e6286 100644 --- a/src/lab/automation/orchestrator.ts +++ b/src/lab/automation/orchestrator.ts @@ -36,19 +36,34 @@ import { LabAutomationError } from "./types"; import { LAB_AUTOMATION_HARD_MAX } from "./constants"; import { cancelQueuedRun } from "./queue"; -let schedulerTimer: ReturnType | null = null; -let tickInProgress = false; +const schedulerTimers = new Map>(); +const ticksInProgress = new Set(); let shutdownRequested = false; -let dispatchDeps: AutomationDispatchDeps = {}; +const dispatchDepsByConfigDir = new Map(); const inFlightControllers = new Map(); const cancellingRunIds = new Set(); +function configKey(configDir?: string): string { + return configDir ?? ""; +} + +function dispatchDepsFor(configDir?: string): AutomationDispatchDeps { + return dispatchDepsByConfigDir.get(configKey(configDir)) + ?? dispatchDepsByConfigDir.get("") + ?? {}; +} + export function setLabAutomationDispatchDeps(deps: AutomationDispatchDeps): void { - dispatchDeps = deps; + const key = configKey(deps.configDir); + if (Object.keys(deps).length === 0) { + dispatchDepsByConfigDir.delete(key); + return; + } + dispatchDepsByConfigDir.set(key, deps); } -export function isLabAutomationSchedulerRunning(): boolean { - return schedulerTimer !== null; +export function isLabAutomationSchedulerRunning(configDir?: string): boolean { + return schedulerTimers.has(configKey(configDir)); } export function requestLabAutomationShutdown(): void { @@ -77,7 +92,7 @@ export function buildLabAutomationStatus(configDir?: string): LabAutomationStatu remainingRunBudget: runBudgetRemaining(policy, state, now), remainingLiveRequestBudget: liveRequestBudgetRemaining(policy, state, now), }, - schedulerRunning: isLabAutomationSchedulerRunning(), + schedulerRunning: isLabAutomationSchedulerRunning(configDir), }; } @@ -131,9 +146,10 @@ export function reconcileLabAutomationQueue(configDir?: string): void { })); } -function loadPlannerConfig(): import("../../types").OcxConfig | undefined { +function loadPlannerConfig(configDir?: string): import("../../types").OcxConfig | undefined { + const runtimeDeps = dispatchDepsFor(configDir); try { - return dispatchDeps.loadConfig?.() ?? readConfigDiagnostics().config; + return runtimeDeps.loadConfig?.() ?? readConfigDiagnostics().config; } catch { return undefined; } @@ -209,6 +225,14 @@ function finalizeRun( let next = state; if (cancelled || (error instanceof LabAutomationError && error.code === "cancelled")) { next = transitionRun(next, run.runId, "cancelled", completedAt, "cancelled"); + if (run.trigger === "scheduled") { + next = setCooldown( + next, + run.runKey, + cooldownForFailure(policy, "cancelled", completedAt), + completedAt, + ); + } return { state: trimTerminalRuns(next, completedAt), value: undefined }; } if (result) { @@ -220,20 +244,26 @@ function finalizeRun( next, run.runKey, cooldownForFailure(policy, result.cooldownCode, completedAt), + completedAt, ); } return { state: trimTerminalRuns(next, completedAt), value: undefined }; } const terminalCode = error instanceof LabAutomationError ? error.code : "dispatch_failure"; next = transitionRun(next, run.runId, "failed", completedAt, terminalCode); - next = setCooldown(next, run.runKey, cooldownForFailure(policy, "harness_failure", completedAt)); + next = setCooldown( + next, + run.runKey, + cooldownForFailure(policy, "harness_failure", completedAt), + completedAt, + ); return { state: trimTerminalRuns(next, completedAt), value: undefined }; }); } async function runDispatchBatch( configDir?: string, - options: { manualRunId?: string } = {}, + options: { manualRunId?: string; abortSignal?: AbortSignal } = {}, ): Promise { if (shutdownRequested) return; const initialPolicy = loadLabAutomationPolicy(configDir); @@ -242,7 +272,7 @@ async function runDispatchBatch( reconcileLabAutomationQueue(configDir); return; } - const config = loadPlannerConfig(); + const config = loadPlannerConfig(configDir); const snapshot = loadLabAutomationState(configDir); const planned = initialPolicy.enabled && options.manualRunId === undefined ? planLabAutomationRuns({ @@ -255,6 +285,8 @@ async function runDispatchBatch( }) : []; let enqueuePlan = true; + // Dispatch is intentionally sequential inside one tick. The concurrency fields still bound + // atomic selection against already-running work and future parallel workers/processes. const maxDispatches = options.manualRunId ? 1 : Math.max(1, initialPolicy.maxConcurrentRuns); for (let dispatched = 0; dispatched < maxDispatches; dispatched += 1) { if (shutdownRequested) break; @@ -272,14 +304,18 @@ async function runDispatchBatch( enqueuePlan = false; if (!run) break; const controller = new AbortController(); + const abortFromCaller = () => controller.abort(options.abortSignal?.reason ?? new Error("cancelled")); + if (options.abortSignal?.aborted) abortFromCaller(); + else options.abortSignal?.addEventListener("abort", abortFromCaller, { once: true }); inFlightControllers.set(run.runId, controller); let result: DispatchResult | null = null; let error: unknown; try { + const runtimeDeps = dispatchDepsFor(configDir); const deps: AutomationDispatchDeps = { - ...dispatchDeps, + ...runtimeDeps, configDir, - loadConfig: dispatchDeps.loadConfig ?? (() => readConfigDiagnostics().config), + loadConfig: runtimeDeps.loadConfig ?? (() => readConfigDiagnostics().config), abortSignal: controller.signal, enforceRunIdentity: true, }; @@ -287,6 +323,7 @@ async function runDispatchBatch( } catch (caught) { error = caught; } finally { + options.abortSignal?.removeEventListener("abort", abortFromCaller); const cancelled = cancellingRunIds.has(run.runId) || controller.signal.aborted; finalizeRun(run, policy, configDir, result, error, cancelled); inFlightControllers.delete(run.runId); @@ -298,17 +335,19 @@ async function runDispatchBatch( /** Single bounded scheduler tick — plan, enqueue, dispatch within concurrency limits. */ export async function runLabAutomationTick(configDir?: string): Promise { - if (tickInProgress) return; - tickInProgress = true; + const key = configKey(configDir); + if (ticksInProgress.has(key)) return; + ticksInProgress.add(key); try { await runDispatchBatch(configDir); } finally { - tickInProgress = false; + ticksInProgress.delete(key); } } export function startLabAutomationScheduler(configDir?: string): void { - if (schedulerTimer) return; + const key = configKey(configDir); + if (schedulerTimers.has(key)) return; shutdownRequested = false; const policy = loadLabAutomationPolicy(configDir); const routes = loadLabAutomationRoutes(configDir); @@ -318,24 +357,32 @@ export function startLabAutomationScheduler(configDir?: string): void { next = reconcileQueuedState(next, policy, routes, now); return { state: trimTerminalRuns(next, now), value: undefined }; }); - schedulerTimer = setInterval(() => { + const timer = setInterval(() => { void runLabAutomationTick(configDir); }, LAB_AUTOMATION_HARD_MAX.schedulerTickMs); - schedulerTimer.unref?.(); + timer.unref?.(); + schedulerTimers.set(key, timer); } /** Stop periodic scheduling only. Process shutdown is a separate explicit signal. */ -export function stopLabAutomationScheduler(): void { - if (schedulerTimer) { - clearInterval(schedulerTimer); - schedulerTimer = null; +export function stopLabAutomationScheduler(configDir?: string): void { + if (configDir !== undefined) { + const key = configKey(configDir); + const timer = schedulerTimers.get(key); + if (timer) clearInterval(timer); + schedulerTimers.delete(key); + return; } + for (const timer of schedulerTimers.values()) clearInterval(timer); + schedulerTimers.clear(); } /** Test-only reset of scheduler globals between isolated automation tests. */ export function resetLabAutomationSchedulerStateForTests(): void { + stopLabAutomationScheduler(); shutdownRequested = false; - tickInProgress = false; + ticksInProgress.clear(); + dispatchDepsByConfigDir.clear(); inFlightControllers.clear(); cancellingRunIds.clear(); } @@ -343,6 +390,7 @@ export function resetLabAutomationSchedulerStateForTests(): void { export async function enqueueManualLabRun( planned: import("./types").PlannedLabRunV1, configDir?: string, + abortSignal?: AbortSignal, ): Promise { const now = Date.now(); const created = mutateLabAutomationState(configDir, (state) => { @@ -354,7 +402,7 @@ export async function enqueueManualLabRun( }); if (!created) return null; // Manual execution is independent of automation enablement/layer toggles. - await runDispatchBatch(configDir, { manualRunId: created.runId }); + await runDispatchBatch(configDir, { manualRunId: created.runId, abortSignal }); return loadLabAutomationState(configDir).runs.find((row) => row.runId === created.runId) ?? created; } @@ -365,9 +413,15 @@ export function cancelLabAutomationRun(runId: string, configDir?: string): boole controller.abort(new Error("cancelled")); return true; } + const policy = loadLabAutomationPolicy(configDir); + const now = Date.now(); return mutateLabAutomationState(configDir, (state) => { const run = state.runs.find((row) => row.runId === runId); if (!run || run.state !== "queued") return { state, value: false }; - return { state: cancelQueuedRun(state, runId, Date.now()), value: true }; + let next = cancelQueuedRun(state, runId, now); + if (run.trigger === "scheduled") { + next = setCooldown(next, run.runKey, cooldownForFailure(policy, "cancelled", now), now); + } + return { state: next, value: true }; }); } From 271eabe96ef49ce95ba6445aff4e73b04bc12462 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:45:52 +0200 Subject: [PATCH 29/67] fix(lab): make freshness exact and bounded --- src/lab/automation/planner.ts | 122 ++++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 44 deletions(-) diff --git a/src/lab/automation/planner.ts b/src/lab/automation/planner.ts index f110f13125..212f6917d3 100644 --- a/src/lab/automation/planner.ts +++ b/src/lab/automation/planner.ts @@ -29,10 +29,15 @@ import { liveExecutionContractDigest, protocolExecutionContractDigest, } from "./run-key"; -import { cooldownActive } from "./cooldown"; +import { cooldownActive, cooldownCapacityExhausted } from "./cooldown"; import { isLiveRequestBudgetExhausted, isRunBudgetExhausted } from "./budgets"; import { LAB_AUTOMATION_HARD_MAX } from "./constants"; +const FRESHNESS_QUERY_PAGE_SIZE = 100; + +type FreshnessIndex = Map; +type FreshnessCache = Map; + export interface PlannerInput { policy: LabAutomationPolicyV1; routes: LabAutomationRoutesV1; @@ -53,12 +58,6 @@ interface EvidenceIdentity { scenarioManifestDigest: string; } -function effectiveMaxAgeMs(suiteMax: number | null, scenarioMax: number | null): number | null { - if (suiteMax === null) return scenarioMax; - if (scenarioMax === null) return suiteMax; - return Math.min(suiteMax, scenarioMax); -} - function freshnessReason( latestCompletedAt: number | undefined, maxAgeMs: number | null, @@ -78,36 +77,71 @@ function activeRunForKey(state: LabAutomationStateV1, runKey: string): LabAutoma return state.runs.find((row) => row.runKey === runKey && (row.state === "queued" || row.state === "running")); } -function latestMatchingObservationCompletedAt(identity: EvidenceIdentity, configDir?: string): number | undefined { +function freshnessIdentityKey(identity: Omit): string { + return [ + identity.suiteId, + identity.suiteVersion, + identity.suiteManifestDigest, + identity.scenarioId, + identity.scenarioVersion, + identity.scenarioManifestDigest, + ].join("\u0000"); +} + +function freshnessIndexForSubject( + layer: EvidenceLayer, + subjectId: string, + cache: FreshnessCache, + configDir?: string, +): FreshnessIndex { + const cacheKey = `${layer}\u0000${subjectId}`; + const cached = cache.get(cacheKey); + if (cached) return cached; + + const index: FreshnessIndex = new Map(); + cache.set(cacheKey, index); try { - const page = queryLabObservations( - { - layer: identity.layer, - subjectId: identity.subjectId, - suiteId: identity.suiteId, - scenarioId: identity.scenarioId, - }, - undefined, - LAB_AUTOMATION_HARD_MAX.maxPersistedRuns, - configDir, - ); - let latest: number | undefined; - for (const row of page.items) { - if (row.excluded) continue; - if (row.suiteVersion !== identity.suiteVersion) continue; - if (row.suiteManifestDigest !== identity.suiteManifestDigest) continue; - if (row.scenarioVersion !== identity.scenarioVersion) continue; - if (row.scenarioManifestDigest !== identity.scenarioManifestDigest) continue; - if (latest === undefined || row.completedAt > latest) latest = row.completedAt; - } - return latest; + let cursor: string | undefined; + do { + const page = queryLabObservations( + { layer, subjectId }, + cursor, + FRESHNESS_QUERY_PAGE_SIZE, + configDir, + ); + for (const row of page.items) { + if (row.excluded) continue; + const key = freshnessIdentityKey({ + suiteId: row.suiteId, + suiteVersion: row.suiteVersion, + suiteManifestDigest: row.suiteManifestDigest, + scenarioId: row.scenarioId, + scenarioVersion: row.scenarioVersion, + scenarioManifestDigest: row.scenarioManifestDigest, + }); + // Query order is newest first, so first exact identity is authoritative for freshness. + if (!index.has(key)) index.set(key, row.completedAt); + } + cursor = page.hasMore ? page.nextCursor : undefined; + } while (cursor); + return index; } catch (error) { - if (error instanceof LabProjectionUnavailableError) return undefined; + if (error instanceof LabProjectionUnavailableError) return index; throw error; } } -function planProtocolScenarios(input: PlannerInput): PlannedLabRunV1[] { +function latestMatchingObservationCompletedAt( + identity: EvidenceIdentity, + cache: FreshnessCache, + configDir?: string, +): number | undefined { + return freshnessIndexForSubject(identity.layer, identity.subjectId, cache, configDir).get( + freshnessIdentityKey(identity), + ); +} + +function planProtocolScenarios(input: PlannerInput, freshnessCache: FreshnessCache): PlannedLabRunV1[] { if (!input.policy.enabled || !input.policy.layers.protocolConformance) return []; const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, CL01_SUITES); @@ -149,11 +183,10 @@ function planProtocolScenarios(input: PlannerInput): PlannedLabRunV1[] { scenarioId: caseRecord.id, scenarioVersion, scenarioManifestDigest: scenarioDigest, - }, input.configDir); - const maxAge = effectiveMaxAgeMs( - authority.manifestDefaults.freshness.maxAgeMs, - authority.manifestDefaults.freshness.maxAgeMs, - ); + }, freshnessCache, input.configDir); + // CL-00 freezes one freshness default into both suite and scenario manifests; there is no + // independent case-level override in the current authority schema. + const maxAge = authority.manifestDefaults.freshness.maxAgeMs; const freshness = freshnessReason(latestCompletedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); if (freshness === "fresh") continue; planned.push({ @@ -174,7 +207,7 @@ function planProtocolScenarios(input: PlannerInput): PlannedLabRunV1[] { return planned; } -function planLiveScenarios(input: PlannerInput): PlannedLabRunV1[] { +function planLiveScenarios(input: PlannerInput, freshnessCache: FreshnessCache): PlannedLabRunV1[] { if (!input.policy.enabled || !input.policy.layers.liveRouteCompatibility) return []; if (!input.config) return []; if (isLiveRequestBudgetExhausted(input.policy, input.state, input.now)) return []; @@ -226,11 +259,8 @@ function planLiveScenarios(input: PlannerInput): PlannedLabRunV1[] { scenarioId: caseRecord.id, scenarioVersion, scenarioManifestDigest: scenarioDigest, - }, input.configDir); - const maxAge = effectiveMaxAgeMs( - authority.manifestDefaults.freshness.maxAgeMs, - authority.manifestDefaults.freshness.maxAgeMs, - ); + }, freshnessCache, input.configDir); + const maxAge = authority.manifestDefaults.freshness.maxAgeMs; const freshness = freshnessReason(latestCompletedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); if (freshness === "fresh") continue; planned.push({ @@ -258,8 +288,12 @@ function planLiveScenarios(input: PlannerInput): PlannedLabRunV1[] { export function planLabAutomationRuns(input: PlannerInput): PlannedLabRunV1[] { if (!input.policy.enabled) return []; if (isRunBudgetExhausted(input.policy, input.state, input.now)) return []; - const protocol = planProtocolScenarios(input); - const live = planLiveScenarios(input); + // Cooldown persistence is bounded. If every slot contains an active backoff, fail closed rather + // than enqueueing work whose retry suppression could not be recorded. + if (cooldownCapacityExhausted(input.state, input.now)) return []; + const freshnessCache: FreshnessCache = new Map(); + const protocol = planProtocolScenarios(input, freshnessCache); + const live = planLiveScenarios(input, freshnessCache); const merged = [...protocol, ...live]; merged.sort((a, b) => { if (a.priority !== b.priority) return a.priority - b.priority; From 4f1297885e8983e0d7498f12a23448e2efa60996 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:46:43 +0200 Subject: [PATCH 30/67] fix(lab): harden automation management API --- .../management/lab-automation-routes.ts | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/src/server/management/lab-automation-routes.ts b/src/server/management/lab-automation-routes.ts index dbb0584399..eb0eea8cce 100644 --- a/src/server/management/lab-automation-routes.ts +++ b/src/server/management/lab-automation-routes.ts @@ -23,6 +23,7 @@ import { planManualLabRun } from "../../lab/automation/planner"; import { loadLabAutomationPolicy, loadLabAutomationState, + normalizeLabAutomationRoutesV1, saveLabAutomationPolicy, saveLabAutomationRoutes, } from "../../lab/automation/persistence"; @@ -66,7 +67,7 @@ function applySchedulerPolicy(policy: LabAutomationPolicyV1, configDir?: string) if (policy.enabled) { startLabAutomationScheduler(configDir); } else { - stopLabAutomationScheduler(); + stopLabAutomationScheduler(configDir); } } @@ -84,19 +85,31 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise const limit = parseLimit(url.searchParams.get("limit"), ctx); if (limit instanceof Response) return limit; const cursor = url.searchParams.get("cursor")?.trim() || undefined; - const status = buildLabAutomationStatus(configDir); - const runsPage = listLabAutomationRuns(loadLabAutomationState(configDir), limit, cursor); - return jsonResponse({ - runs: runsPage.items, - hasMore: runsPage.hasMore, - ...(runsPage.nextCursor ? { nextCursor: runsPage.nextCursor } : {}), - counters: status.counters, - }, 200, req, config); + try { + const status = buildLabAutomationStatus(configDir); + const runsPage = listLabAutomationRuns(loadLabAutomationState(configDir), limit, cursor); + return jsonResponse({ + runs: runsPage.items, + hasMore: runsPage.hasMore, + ...(runsPage.nextCursor ? { nextCursor: runsPage.nextCursor } : {}), + counters: status.counters, + }, 200, req, config); + } catch (error) { + if (error instanceof LabAutomationError && error.code === "invalid_cursor") { + return automationErrorResponse(error.code, error.message, 400, ctx); + } + throw error; + } } const cancelMatch = url.pathname.match(/^\/api\/lab\/automation\/runs\/([^/]+)\/cancel$/); if (cancelMatch && req.method === "POST") { - const runId = decodeURIComponent(cancelMatch[1]!); + let runId: string; + try { + runId = decodeURIComponent(cancelMatch[1]!); + } catch { + return automationErrorResponse("invalid_run_id", "run id is not valid percent-encoding", 400, ctx); + } const cancelled = cancelLabAutomationRun(runId, configDir); if (!cancelled) return automationErrorResponse("not_found", "unknown or non-cancellable run", 404, ctx); return jsonResponse({ cancelled: true, runId }, 200, req, config); @@ -131,9 +144,12 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise config: ocxConfig, configDir, }); - const record = await enqueueManualLabRun(planned, configDir); + // This endpoint is intentionally synchronous: completion is the acknowledgement boundary. + // Existing harness execution limits provide the route-level deadline, and disconnects cancel + // through Request.signal instead of leaving orphaned provider work running. + const record = await enqueueManualLabRun(planned, configDir, req.signal); if (!record) return automationErrorResponse("enqueue_failed", "manual run could not be enqueued", 500, ctx); - return jsonResponse({ run: record, trigger: "manual" }, 202, req, config); + return jsonResponse({ run: record, trigger: "manual" }, 200, req, config); } catch (error) { rethrowManagementBodyTooLarge(error); if (error instanceof LabAutomationError) { @@ -150,6 +166,7 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise // Only policy/routes are authoritative. Other top-level properties are ignored rather than // treated as injectable process-local capabilities (for example a fake routeExecutor). let policy = loadLabAutomationPolicy(configDir); + let routes: LabAutomationRoutesV1 | undefined; if (body.policy !== undefined) { if (!isPlainRecord(body.policy)) return automationErrorResponse("invalid_policy", "policy must be an object", 400, ctx); const incoming = body.policy as Record; @@ -163,19 +180,16 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise ...incoming, layers: mergedLayers, }); - saveLabAutomationPolicy(policy, configDir); } if (body.routes !== undefined) { if (!isPlainRecord(body.routes)) return automationErrorResponse("invalid_routes", "routes must be an object", 400, ctx); - const routesPayload = body.routes as Record; - if (routesPayload.schemaVersion !== 1 || !Array.isArray(routesPayload.routes)) { - return automationErrorResponse("invalid_routes", "routes must include schemaVersion 1 and routes array", 400, ctx); - } - saveLabAutomationRoutes({ - schemaVersion: 1, - routes: routesPayload.routes as LabAutomationRoutesV1["routes"], - }, configDir); + routes = normalizeLabAutomationRoutesV1(body.routes); } + + // Validate the complete requested update before persisting either file so an invalid routes + // payload cannot leave a valid policy half-applied. + if (body.routes !== undefined && routes) saveLabAutomationRoutes(routes, configDir); + if (body.policy !== undefined) saveLabAutomationPolicy(policy, configDir); applySchedulerPolicy(policy, configDir); return jsonResponse(buildLabAutomationStatus(configDir), 200, req, config); } catch (error) { From e250560b03f7b9dc39eef43fed3b33cc8fa85d0b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:48:07 +0200 Subject: [PATCH 31/67] fix(lab): preserve CLI automation intent --- src/cli/lab.ts | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/cli/lab.ts b/src/cli/lab.ts index 44dd85b5f0..e29c797f1e 100644 --- a/src/cli/lab.ts +++ b/src/cli/lab.ts @@ -59,7 +59,7 @@ import { } from "../lab/automation/persistence"; import { planManualLabRun } from "../lab/automation/planner"; import { listLabAutomationRuns } from "../lab/automation/runs-query"; -import type { LabAutomationLayer } from "../lab/automation/types"; +import { LabAutomationError, type LabAutomationLayer } from "../lab/automation/types"; import { createProductionLabRouteExecutor } from "../lib/lab-live-route-production"; const USAGE = `Usage: @@ -97,6 +97,7 @@ function labErrorMessage(err: unknown): string { if (err instanceof LabProjectionUnavailableError) return "lab projection is not available"; if (err instanceof LabProjectionIncompatibleError) return "lab projection schema or spec version is incompatible"; if (err instanceof InvalidCursorError) return "invalid cursor"; + if (err instanceof LabAutomationError && err.code === "invalid_cursor") return "invalid cursor"; return "lab read failed"; } @@ -189,13 +190,18 @@ function catalogLines(scenarios: ReturnType): str function automationStatusLines(status: ReturnType): string[] { return [ `Automation enabled: ${status.policy.enabled}`, - `Scheduler running: ${status.schedulerRunning}`, + `Scheduler intent: ${status.policy.enabled ? "enabled" : "disabled"}`, `Layers: protocol=${status.policy.layers.protocolConformance} live=${status.policy.layers.liveRouteCompatibility} task=${status.policy.layers.taskEffectiveness}`, `Queued: ${status.counters.queued} | Running: ${status.counters.running}`, `Run budget remaining: ${status.counters.remainingRunBudget} | Live request budget: ${status.counters.remainingLiveRequestBudget}`, ]; } +function automationCliStatus(status: ReturnType) { + const { schedulerRunning: _processLocalSchedulerState, ...stable } = status; + return { ...stable, schedulerIntentEnabled: status.policy.enabled }; +} + function runListLines(page: ReturnType): string[] { const lines = page.items.map((row) => `${row.runId} ${row.state} ${row.evidenceLayer} ${row.scenarioId} trigger=${row.trigger}`, @@ -346,7 +352,7 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P case "status": { rejectArgs(automationRest, USAGE); const status = buildLabAutomationStatus(configDir); - printData(status, wantsJson, automationStatusLines(status)); + printData(automationCliStatus(status), wantsJson, automationStatusLines(status)); return; } case "enable": { @@ -354,12 +360,13 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P const enableLive = takeFlag(automationRest, "--live"); rejectArgs(automationRest, USAGE); const selectionExplicit = enableProtocol || enableLive; + const current = loadLabAutomationPolicy(configDir); const policy = { - ...loadLabAutomationPolicy(configDir), + ...current, enabled: true, layers: { - protocolConformance: selectionExplicit ? enableProtocol : true, - liveRouteCompatibility: enableLive, + protocolConformance: selectionExplicit ? enableProtocol : current.layers.protocolConformance, + liveRouteCompatibility: selectionExplicit ? enableLive : current.layers.liveRouteCompatibility, taskEffectiveness: false, }, taskEffectivenessBackgroundEnabled: false, @@ -368,7 +375,7 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P reconcileLabAutomationQueue(configDir); startLabAutomationScheduler(configDir); const status = buildLabAutomationStatus(configDir); - printData(status, wantsJson, automationStatusLines(status)); + printData(automationCliStatus(status), wantsJson, automationStatusLines(status)); return; } case "disable": { @@ -376,9 +383,9 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P const policy = { ...loadLabAutomationPolicy(configDir), enabled: false }; saveLabAutomationPolicy(policy, configDir); reconcileLabAutomationQueue(configDir); - stopLabAutomationScheduler(); + stopLabAutomationScheduler(configDir); const status = buildLabAutomationStatus(configDir); - printData(status, wantsJson, automationStatusLines(status)); + printData(automationCliStatus(status), wantsJson, automationStatusLines(status)); return; } case "runs": { From f35510414089f2b701c56a872b9ebf54b1b3e88f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:50:49 +0200 Subject: [PATCH 32/67] test(lab): cover CodeRabbit automation regressions --- ...-automation-coderabbit-regressions.test.ts | 325 ++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 tests/lab-automation-coderabbit-regressions.test.ts diff --git a/tests/lab-automation-coderabbit-regressions.test.ts b/tests/lab-automation-coderabbit-regressions.test.ts new file mode 100644 index 0000000000..e087f361e9 --- /dev/null +++ b/tests/lab-automation-coderabbit-regressions.test.ts @@ -0,0 +1,325 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { OcxConfig } from "../src/types"; +import { defaultLabAutomationPolicyV1, normalizeLabAutomationPolicyV1 } from "../src/lab/automation/policy"; +import { + defaultLabAutomationStateV1, + loadLabAutomationPolicy, + loadLabAutomationState, + saveLabAutomationPolicy, + saveLabAutomationState, +} from "../src/lab/automation/persistence"; +import { LAB_AUTOMATION_HARD_MAX } from "../src/lab/automation/constants"; +import { enqueuePlannedRuns } from "../src/lab/automation/queue"; +import { runBudgetRemaining } from "../src/lab/automation/budgets"; +import { setCooldown } from "../src/lab/automation/cooldown"; +import { listLabAutomationRuns } from "../src/lab/automation/runs-query"; +import { + cancelLabAutomationRun, + requestLabAutomationShutdown, + resetLabAutomationSchedulerStateForTests, + stopLabAutomationScheduler, +} from "../src/lab/automation/orchestrator"; +import type { LabAutomationRunRecordV1, PlannedLabRunV1 } from "../src/lab/automation/types"; +import { LabAutomationError } from "../src/lab/automation/types"; +import { labAutomationStatePath } from "../src/lab/paths"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import { handleLabCommand } from "../src/cli/lab"; +import { createProductionLabRouteExecutor } from "../src/lib/lab-live-route-production"; + +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-coderabbit-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + process.env.OPENCODEX_HOME = dir; + return dir; +} + +function emptyConfig(): OcxConfig { + return { providers: {} } as OcxConfig; +} + +function queuedRun(overrides: Partial = {}): LabAutomationRunRecordV1 { + const now = 1_000_000; + return { + runId: "run-1", + runKey: "key-1", + state: "queued", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "suite-digest", + scenarioId: "responses-core.protocol.request-shape", + scenarioVersion: "1", + scenarioManifestDigest: "scenario-digest", + subjectId: "subject-1", + reason: "missing", + priority: 0, + eligibleAt: now, + trigger: "scheduled", + createdAt: now, + updatedAt: now, + ...overrides, + }; +} + +function plan(overrides: Partial = {}): PlannedLabRunV1 { + return { + runKey: "new-key", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "suite-digest", + scenarioId: "responses-core.protocol.request-shape", + scenarioVersion: "1", + scenarioManifestDigest: "scenario-digest", + subjectId: "subject-new", + reason: "missing", + priority: 0, + eligibleAt: 1_000_000, + ...overrides, + }; +} + +afterEach(() => { + requestLabAutomationShutdown(); + stopLabAutomationScheduler(); + resetLabAutomationSchedulerStateForTests(); + for (const dir of HOMES.splice(0)) rmSync(dir, { recursive: true, force: true }); + delete process.env.OPENCODEX_HOME; +}); + +describe("CL-08 CodeRabbit regressions", () => { + test("zero concurrency policy values are rejected", () => { + const base = defaultLabAutomationPolicyV1(); + for (const field of ["maxConcurrentRuns", "maxConcurrentLiveRuns", "maxConcurrentRunsPerRoute"] as const) { + expect(() => normalizeLabAutomationPolicyV1({ ...base, [field]: 0 })).toThrow(LabAutomationError); + } + }); + + test("recent terminal attempts cannot be evicted out of the rolling budget window", () => { + const now = 2_000_000; + const runs = Array.from({ length: LAB_AUTOMATION_HARD_MAX.maxPersistedRuns }, (_, index) => queuedRun({ + runId: `done-${index}`, + runKey: `done-key-${index}`, + state: "completed", + createdAt: now - 500, + updatedAt: now - 100, + startedAt: now - 200, + completedAt: now - 100, + terminalCode: undefined, + })); + const state = { ...defaultLabAutomationStateV1(now), runs }; + const next = enqueuePlannedRuns(state, [plan({ eligibleAt: now })], "scheduled", now); + expect(next.runs).toHaveLength(LAB_AUTOMATION_HARD_MAX.maxPersistedRuns); + expect(next.runs.some((row) => row.runKey === "new-key")).toBe(false); + const policy = { ...defaultLabAutomationPolicyV1(), maxRunsPerHour: LAB_AUTOMATION_HARD_MAX.maxRunsPerHour }; + expect(runBudgetRemaining(policy, next, now)).toBe(0); + }); + + test("setting cooldown prunes expired entries and never exceeds persistence capacity", () => { + const now = 5_000; + const expired = Object.fromEntries( + Array.from({ length: LAB_AUTOMATION_HARD_MAX.maxPersistedRuns }, (_, index) => [`expired-${index}`, now - 1]), + ); + const pruned = setCooldown( + { ...defaultLabAutomationStateV1(now), cooldownUntilByKey: expired }, + "fresh-key", + now + 1_000, + now, + ); + expect(pruned.cooldownUntilByKey).toEqual({ "fresh-key": now + 1_000 }); + + const active = Object.fromEntries( + Array.from({ length: LAB_AUTOMATION_HARD_MAX.maxPersistedRuns }, (_, index) => [`active-${index}`, now + 10_000]), + ); + const saturated = setCooldown( + { ...defaultLabAutomationStateV1(now), cooldownUntilByKey: active }, + "overflow", + now + 20_000, + now, + ); + expect(Object.keys(saturated.cooldownUntilByKey)).toHaveLength(LAB_AUTOMATION_HARD_MAX.maxPersistedRuns); + expect(saturated.cooldownUntilByKey.overflow).toBeUndefined(); + }); + + test("persisted state rejects duplicate identities and impossible lifecycle fields", () => { + const home = tempHome(); + const writeState = (runs: unknown[]) => { + writeFileSync(labAutomationStatePath(home), JSON.stringify({ + schemaVersion: 1, + runs, + budgetWindowStartedAt: 1, + runsThisHour: 0, + liveRequestsThisHour: 0, + cooldownUntilByKey: {}, + }), "utf8"); + }; + + writeState([queuedRun(), queuedRun({ runKey: "other-key" })]); + expect(() => loadLabAutomationState(home)).toThrow(LabAutomationError); + + writeState([queuedRun(), queuedRun({ runId: "run-2" })]); + expect(() => loadLabAutomationState(home)).toThrow(LabAutomationError); + + writeState([queuedRun({ completedAt: 1_000_001 })]); + expect(() => loadLabAutomationState(home)).toThrow(LabAutomationError); + }); + + test("unknown run cursor is an explicit invalid_cursor error", () => { + expect(() => listLabAutomationRuns(defaultLabAutomationStateV1(), 10, "evicted-run")) + .toThrow(LabAutomationError); + try { + listLabAutomationRuns(defaultLabAutomationStateV1(), 10, "evicted-run"); + } catch (error) { + expect((error as LabAutomationError).code).toBe("invalid_cursor"); + } + }); + + test("operator cancellation gives scheduled work a positive backoff", () => { + const home = tempHome(); + const policy = { + ...defaultLabAutomationPolicyV1(), + enabled: true, + layers: { protocolConformance: true, liveRouteCompatibility: false, taskEffectiveness: false }, + failureCooldownMs: 5_000, + }; + saveLabAutomationPolicy(policy, home); + saveLabAutomationState({ ...defaultLabAutomationStateV1(), runs: [queuedRun()] }, home); + expect(cancelLabAutomationRun("run-1", home)).toBe(true); + const state = loadLabAutomationState(home); + expect(state.runs[0]?.state).toBe("cancelled"); + expect(state.cooldownUntilByKey["key-1"]).toBeGreaterThan(Date.now()); + }); + + test("management cancel rejects malformed percent encoding", async () => { + tempHome(); + const req = new ManagementRequest("http://127.0.0.1/api/lab/automation/runs/%E0%A4%A/cancel", { method: "POST" }); + const res = await handleManagementAPI(req, new URL(req.url), emptyConfig()); + expect(res).not.toBeNull(); + expect(res!.status).toBe(400); + expect((await res!.json()).error.code).toBe("invalid_run_id"); + }); + + test("management run listing rejects invalid limits and stale cursors", async () => { + tempHome(); + for (const suffix of ["?limit=0", "?limit=101", "?cursor=evicted"]) { + const req = new ManagementRequest(`http://127.0.0.1/api/lab/automation/runs${suffix}`); + const res = await handleManagementAPI(req, new URL(req.url), emptyConfig()); + expect(res).not.toBeNull(); + expect(res!.status).toBe(400); + } + }); + + test("invalid routes cannot partially persist an otherwise valid policy update", async () => { + const home = tempHome(); + expect(loadLabAutomationPolicy(home).enabled).toBe(false); + const req = new ManagementRequest("http://127.0.0.1/api/lab/automation", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + policy: { enabled: true }, + routes: { schemaVersion: 1, routes: [{ providerName: "fixture-provider" }] }, + }), + }); + const res = await handleManagementAPI(req, new URL(req.url), emptyConfig()); + expect(res).not.toBeNull(); + expect(res!.status).toBe(400); + expect(loadLabAutomationPolicy(home).enabled).toBe(false); + }); + + test("manual management body rejects unknown capability injection keys", async () => { + tempHome(); + const req = new ManagementRequest("http://127.0.0.1/api/lab/automation/run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + evidenceLayer: "protocol_conformance", + scenarioId: "responses-core.protocol.request-shape", + routeExecutor: { execute: "forged" }, + }), + }); + const res = await handleManagementAPI(req, new URL(req.url), emptyConfig()); + expect(res).not.toBeNull(); + expect(res!.status).toBe(400); + }); + + test("CLI re-enable without layer flags preserves prior selections", async () => { + const home = tempHome(); + saveLabAutomationPolicy({ + ...defaultLabAutomationPolicyV1(), + enabled: false, + layers: { protocolConformance: false, liveRouteCompatibility: true, taskEffectiveness: false }, + }, home); + expect(await handleLabCommand(["automation", "enable", "--json"], { configDir: home })).toBe(0); + const policy = loadLabAutomationPolicy(home); + expect(policy.enabled).toBe(true); + expect(policy.layers.protocolConformance).toBe(false); + expect(policy.layers.liveRouteCompatibility).toBe(true); + }); + + test("OAuth live probes reject cleartext destinations before transport", async () => { + const config = { + providers: { + xai: { + adapter: "openai-responses", + baseUrl: "http://example.com/v1", + authMode: "oauth", + models: ["grok"], + }, + }, + } as OcxConfig; + const executor = createProductionLabRouteExecutor({ loadConfig: () => config }); + await expect(executor.execute({ + routeContext: { + providerId: "xai", + providerInstanceKey: "fixture", + clientModelId: "grok", + upstreamModelId: "grok", + effectiveAdapter: "openai-responses", + inboundProtocol: "responses", + upstreamProtocol: "responses", + surface: "responses", + baseUrl: "http://example.com/v1", + opencodexCompatibilityVersion: "f".repeat(64), + behaviorValues: {}, + }, + destination: { + scheme: "http", + host: "example.com", + port: 80, + basePath: "/v1", + sniHost: "example.com", + addresses: [{ address: "93.184.216.34", family: 4 }], + privateNetwork: false, + fingerprint: "destination", + }, + routeSubject: {} as never, + scenarioId: "responses-core.live.basic-turn", + initiatingRequest: "{}", + limits: { + totalTimeoutMs: 1_000, + connectTimeoutMs: 500, + firstByteTimeoutMs: 500, + inactivityTimeoutMs: 500, + maxRequests: 1, + maxInputBytes: 1024, + maxOutputBytes: 1024, + maxOutputTokens: 128, + maxToolCalls: 0, + maxMemoryBytes: 1024 * 1024, + maxChildProcesses: 0, + maxArtifacts: 0, + perArtifactBytes: 0, + aggregateArtifactBytes: 0, + }, + signal: new AbortController().signal, + environment: {}, + })).rejects.toThrow("OAuth lab probes require HTTPS"); + }); +}); From 1ae9b4a3f724b28e6413cc261d693f18c2261aba Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:51:41 +0200 Subject: [PATCH 33/67] fix(lab): prune cooldowns during recovery --- src/lab/automation/recovery.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lab/automation/recovery.ts b/src/lab/automation/recovery.ts index 1408585611..ca122edf0f 100644 --- a/src/lab/automation/recovery.ts +++ b/src/lab/automation/recovery.ts @@ -17,6 +17,7 @@ export function recoverLabAutomationState( next, run.runKey, now + Math.min(policy.failureCooldownMs, LAB_AUTOMATION_HARD_MAX.failureCooldownMs), + now, ); } return next; From c441591e418716adc2c00aacc9dbdd68df64557f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:53:10 +0200 Subject: [PATCH 34/67] test(lab): make rolling budget regression deterministic --- tests/lab-automation-review-regressions.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lab-automation-review-regressions.test.ts b/tests/lab-automation-review-regressions.test.ts index 8f8c630d81..52075bff9b 100644 --- a/tests/lab-automation-review-regressions.test.ts +++ b/tests/lab-automation-review-regressions.test.ts @@ -412,7 +412,7 @@ describe("CL-08 independent review regressions", () => { runsThisHour: 2, }; const rolled = rollBudgetWindow(state, now); - expect(runBudgetRemaining(policy, rolled)).toBe(4); + expect(runBudgetRemaining(policy, rolled, now)).toBe(4); }); test("harness failures receive the stronger blocked cooldown and successful runs clear stale cooldown", async () => { From 7fbc18f97740e994188d05ac1e8fc53769d836dd Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:59:46 +0200 Subject: [PATCH 35/67] docs(lab): align CL-08 headings and budget reason --- .../_plan/260807_compatibility_lab/008_cl08_automation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md b/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md index 971b941be9..2c27e41040 100644 --- a/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md +++ b/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md @@ -440,7 +440,7 @@ It must never produce a route compatibility failure. Differentiate at minimum: -### Attributable compatibility result +## Attributable compatibility result Ordinary freshness rules apply. @@ -681,7 +681,7 @@ missing already_queued already_running cooldown -budget_exhausted +budget_blocked route_ineligible scenario_inapplicable task_background_disabled @@ -877,7 +877,7 @@ CL-08 must not implement: Recommended implementation sequence: -### CL-08.0 — Audit and contract +## CL-08.0 — Audit and contract - inspect current `dev`; - audit existing timers/background services/config persistence/API lifecycle; From e04dce6a91f759cda19fd180d5c2daf27449dcb1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:03:30 +0200 Subject: [PATCH 36/67] test(lab): strengthen automation authority regressions --- tests/lab-automation.test.ts | 53 +++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/tests/lab-automation.test.ts b/tests/lab-automation.test.ts index d86ace4c7b..ad63896582 100644 --- a/tests/lab-automation.test.ts +++ b/tests/lab-automation.test.ts @@ -19,6 +19,7 @@ import { recoverLabAutomationState } from "../src/lab/automation/recovery"; import { buildLabAutomationStatus, cancelLabAutomationRun, + enqueueManualLabRun, runLabAutomationTick, setLabAutomationDispatchDeps, stopLabAutomationScheduler, @@ -335,18 +336,21 @@ describe("CL-08 lab automation", () => { }); test("task background execution stays disabled by default", () => { - const policy = { - ...enabledProtocolPolicy(), - layers: { protocolConformance: false, liveRouteCompatibility: false, taskEffectiveness: true }, - taskEffectivenessBackgroundEnabled: false, - }; - const planned = planLabAutomationRuns({ - policy, - routes: { schemaVersion: 1, routes: [] }, - state: loadLabAutomationState(tempHome()), - now: Date.now(), + withHome((home) => { + const policy = { + ...enabledProtocolPolicy(), + layers: { protocolConformance: false, liveRouteCompatibility: false, taskEffectiveness: true }, + taskEffectivenessBackgroundEnabled: false, + }; + const planned = planLabAutomationRuns({ + policy, + routes: { schemaVersion: 1, routes: [] }, + state: loadLabAutomationState(home), + now: Date.now(), + configDir: home, + }); + expect(planned).toEqual([]); }); - expect(planned).toEqual([]); }); test("GET lab automation API does not start scheduler", async () => { @@ -354,7 +358,8 @@ describe("CL-08 lab automation", () => { const config = { providers: {} } as OcxConfig; const req = new ManagementRequest("http://127.0.0.1/api/lab/automation"); const res = await handleManagementAPI(req, new URL(req.url), config); - expect(res.status).toBe(200); + expect(res).not.toBeNull(); + expect(res?.status).toBe(200); const status = buildLabAutomationStatus(home); expect(status.schedulerRunning).toBe(false); }); @@ -552,8 +557,16 @@ describe("CL-08 lab automation", () => { test("management PUT cannot inject trusted executor authority", async () => { await withHome(async (home) => { + prepareLiveRouteHome(home); const config = fixtureProviderConfig(); - const fakeExecutor = { execute: async () => passObservation(), enforcedBoundaries: [] }; + let fakeInvokes = 0; + const fakeExecutor = { + execute: async () => { + fakeInvokes += 1; + return passObservation(); + }, + enforcedBoundaries: [], + }; const req = new ManagementRequest("http://127.0.0.1/api/lab/automation", { method: "PUT", headers: { "content-type": "application/json" }, @@ -563,10 +576,24 @@ describe("CL-08 lab automation", () => { }), }); const res = await handleManagementAPI(req, new URL(req.url), config); + expect(res).not.toBeNull(); expect(res?.status).toBe(200); const production = createProductionLabRouteExecutor({ loadConfig: () => config }); expect(isTrustedLabRouteExecutor(production)).toBe(true); expect(isTrustedLabRouteExecutor(fakeExecutor)).toBe(false); + + const plan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId: "responses-core.live.basic-turn", + providerName: "fixture-provider", + modelId: "fixture-model", + config, + configDir: home, + }); + const record = await enqueueManualLabRun(plan, home); + expect(fakeInvokes).toBe(0); + expect(record?.state).toBe("blocked"); + expect(record?.terminalCode).toBe("route_ineligible"); }); }); From 36bcc80f834ef78dc4eb11d2c6dbc384e7c42837 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:06:15 +0200 Subject: [PATCH 37/67] test(lab): cover automation management pagination and cancel routes --- tests/lab-automation-management-http.test.ts | 121 +++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tests/lab-automation-management-http.test.ts diff --git a/tests/lab-automation-management-http.test.ts b/tests/lab-automation-management-http.test.ts new file mode 100644 index 0000000000..1ff19010fb --- /dev/null +++ b/tests/lab-automation-management-http.test.ts @@ -0,0 +1,121 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { OcxConfig } from "../src/types"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import { + defaultLabAutomationStateV1, + saveLabAutomationState, +} from "../src/lab/automation/persistence"; +import { + requestLabAutomationShutdown, + resetLabAutomationSchedulerStateForTests, + stopLabAutomationScheduler, +} from "../src/lab/automation/orchestrator"; +import type { LabAutomationRunRecordV1 } from "../src/lab/automation/types"; + +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-http-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + process.env.OPENCODEX_HOME = dir; + return dir; +} + +function config(): OcxConfig { + return { providers: {} } as OcxConfig; +} + +function queuedRun(runId: string, runKey: string, createdAt: number): LabAutomationRunRecordV1 { + return { + runId, + runKey, + state: "queued", + evidenceLayer: "protocol_conformance", + suiteId: "responses-core", + suiteVersion: "1", + suiteManifestDigest: "suite-digest", + scenarioId: `responses-core.protocol.${runId}`, + scenarioVersion: "1", + scenarioManifestDigest: `scenario-digest-${runId}`, + subjectId: `subject-${runId}`, + reason: "missing", + priority: 0, + eligibleAt: createdAt, + trigger: "scheduled", + createdAt, + updatedAt: createdAt, + }; +} + +afterEach(() => { + requestLabAutomationShutdown(); + stopLabAutomationScheduler(); + resetLabAutomationSchedulerStateForTests(); + for (const dir of HOMES.splice(0)) rmSync(dir, { recursive: true, force: true }); + delete process.env.OPENCODEX_HOME; +}); + +describe("CL-08 automation management HTTP", () => { + test("cancel returns 200 for a queued run and 404 for an unknown run", async () => { + const home = tempHome(); + const now = Date.now(); + saveLabAutomationState({ + ...defaultLabAutomationStateV1(now), + runs: [queuedRun("queued-1", "key-1", now)], + }, home); + + const cancel = new ManagementRequest("http://127.0.0.1/api/lab/automation/runs/queued-1/cancel", { + method: "POST", + }); + const cancelRes = await handleManagementAPI(cancel, new URL(cancel.url), config()); + expect(cancelRes).not.toBeNull(); + expect(cancelRes?.status).toBe(200); + expect(await cancelRes?.json()).toMatchObject({ cancelled: true, runId: "queued-1" }); + + const unknown = new ManagementRequest("http://127.0.0.1/api/lab/automation/runs/missing/cancel", { + method: "POST", + }); + const unknownRes = await handleManagementAPI(unknown, new URL(unknown.url), config()); + expect(unknownRes).not.toBeNull(); + expect(unknownRes?.status).toBe(404); + expect((await unknownRes?.json())?.error?.code).toBe("not_found"); + }); + + test("run pagination returns a stable nextCursor", async () => { + const home = tempHome(); + const now = Date.now(); + saveLabAutomationState({ + ...defaultLabAutomationStateV1(now), + runs: [ + queuedRun("newer", "key-newer", now), + queuedRun("older", "key-older", now - 1), + ], + }, home); + + const first = new ManagementRequest("http://127.0.0.1/api/lab/automation/runs?limit=1"); + const firstRes = await handleManagementAPI(first, new URL(first.url), config()); + expect(firstRes).not.toBeNull(); + expect(firstRes?.status).toBe(200); + const firstBody = await firstRes?.json(); + expect(firstBody?.runs).toHaveLength(1); + expect(firstBody?.runs[0]?.runId).toBe("newer"); + expect(firstBody?.hasMore).toBe(true); + expect(firstBody?.nextCursor).toBe("newer"); + + const second = new ManagementRequest( + `http://127.0.0.1/api/lab/automation/runs?limit=1&cursor=${encodeURIComponent(firstBody.nextCursor)}`, + ); + const secondRes = await handleManagementAPI(second, new URL(second.url), config()); + expect(secondRes).not.toBeNull(); + expect(secondRes?.status).toBe(200); + const secondBody = await secondRes?.json(); + expect(secondBody?.runs).toHaveLength(1); + expect(secondBody?.runs[0]?.runId).toBe("older"); + expect(secondBody?.hasMore).toBe(false); + }); +}); From f13489a9fb979ba2e2957060c20f266859aa5ed9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:17:55 +0200 Subject: [PATCH 38/67] test(lab): assert automation HTTP error codes --- tests/lab-automation-coderabbit-regressions.test.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/lab-automation-coderabbit-regressions.test.ts b/tests/lab-automation-coderabbit-regressions.test.ts index e087f361e9..b227533b19 100644 --- a/tests/lab-automation-coderabbit-regressions.test.ts +++ b/tests/lab-automation-coderabbit-regressions.test.ts @@ -208,12 +208,19 @@ describe("CL-08 CodeRabbit regressions", () => { test("management run listing rejects invalid limits and stale cursors", async () => { tempHome(); - for (const suffix of ["?limit=0", "?limit=101", "?cursor=evicted"]) { + for (const suffix of ["?limit=0", "?limit=101"]) { const req = new ManagementRequest(`http://127.0.0.1/api/lab/automation/runs${suffix}`); const res = await handleManagementAPI(req, new URL(req.url), emptyConfig()); expect(res).not.toBeNull(); expect(res!.status).toBe(400); + expect((await res!.json()).error.code).toBe("invalid_limit"); } + + const staleReq = new ManagementRequest("http://127.0.0.1/api/lab/automation/runs?cursor=evicted"); + const staleRes = await handleManagementAPI(staleReq, new URL(staleReq.url), emptyConfig()); + expect(staleRes).not.toBeNull(); + expect(staleRes!.status).toBe(400); + expect((await staleRes!.json()).error.code).toBe("invalid_cursor"); }); test("invalid routes cannot partially persist an otherwise valid policy update", async () => { @@ -247,6 +254,7 @@ describe("CL-08 CodeRabbit regressions", () => { const res = await handleManagementAPI(req, new URL(req.url), emptyConfig()); expect(res).not.toBeNull(); expect(res!.status).toBe(400); + expect((await res!.json()).error.code).toBe("invalid_body"); }); test("CLI re-enable without layer flags preserves prior selections", async () => { From 4c4f8cf1dfaba849e9d19232d791a99079f5e9e5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:22:39 +0200 Subject: [PATCH 39/67] fix(lab): guarantee cancellation backoff --- src/lab/automation/orchestrator.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lab/automation/orchestrator.ts b/src/lab/automation/orchestrator.ts index c1f82e6286..fa69ea063e 100644 --- a/src/lab/automation/orchestrator.ts +++ b/src/lab/automation/orchestrator.ts @@ -53,6 +53,10 @@ function dispatchDepsFor(configDir?: string): AutomationDispatchDeps { ?? {}; } +function cancellationCooldownUntil(policy: LabAutomationPolicyV1, now: number): number { + return now + Math.max(policy.failureCooldownMs, LAB_AUTOMATION_HARD_MAX.schedulerTickMs); +} + export function setLabAutomationDispatchDeps(deps: AutomationDispatchDeps): void { const key = configKey(deps.configDir); if (Object.keys(deps).length === 0) { @@ -229,7 +233,7 @@ function finalizeRun( next = setCooldown( next, run.runKey, - cooldownForFailure(policy, "cancelled", completedAt), + cancellationCooldownUntil(policy, completedAt), completedAt, ); } @@ -420,7 +424,7 @@ export function cancelLabAutomationRun(runId: string, configDir?: string): boole if (!run || run.state !== "queued") return { state, value: false }; let next = cancelQueuedRun(state, runId, now); if (run.trigger === "scheduled") { - next = setCooldown(next, run.runKey, cooldownForFailure(policy, "cancelled", now), now); + next = setCooldown(next, run.runKey, cancellationCooldownUntil(policy, now), now); } return { state: next, value: true }; }); From 2390c203c340d4adb7eb55631594997ec724b492 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:23:39 +0200 Subject: [PATCH 40/67] test(lab): prove cancellation backoff with zero failure cooldown --- tests/lab-automation-coderabbit-regressions.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/lab-automation-coderabbit-regressions.test.ts b/tests/lab-automation-coderabbit-regressions.test.ts index b227533b19..d92e05c4aa 100644 --- a/tests/lab-automation-coderabbit-regressions.test.ts +++ b/tests/lab-automation-coderabbit-regressions.test.ts @@ -181,20 +181,21 @@ describe("CL-08 CodeRabbit regressions", () => { } }); - test("operator cancellation gives scheduled work a positive backoff", () => { + test("operator cancellation gives scheduled work a positive backoff even when failure cooldown is zero", () => { const home = tempHome(); const policy = { ...defaultLabAutomationPolicyV1(), enabled: true, layers: { protocolConformance: true, liveRouteCompatibility: false, taskEffectiveness: false }, - failureCooldownMs: 5_000, + failureCooldownMs: 0, }; saveLabAutomationPolicy(policy, home); saveLabAutomationState({ ...defaultLabAutomationStateV1(), runs: [queuedRun()] }, home); + const before = Date.now(); expect(cancelLabAutomationRun("run-1", home)).toBe(true); const state = loadLabAutomationState(home); expect(state.runs[0]?.state).toBe("cancelled"); - expect(state.cooldownUntilByKey["key-1"]).toBeGreaterThan(Date.now()); + expect(state.cooldownUntilByKey["key-1"]).toBeGreaterThanOrEqual(before + LAB_AUTOMATION_HARD_MAX.schedulerTickMs); }); test("management cancel rejects malformed percent encoding", async () => { From 27e7394b1623e67b4742ce0e4f4c0463ef10583e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:34:31 +0200 Subject: [PATCH 41/67] fix(lab): reserve cooldown capacity before dispatch --- src/lab/automation/cooldown.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/lab/automation/cooldown.ts b/src/lab/automation/cooldown.ts index 7bc4197514..15bef95f42 100644 --- a/src/lab/automation/cooldown.ts +++ b/src/lab/automation/cooldown.ts @@ -14,8 +14,27 @@ function activeCooldowns(state: LabAutomationStateV1, now: number): Record { + const reserved = new Set(Object.keys(activeCooldowns(state, now))); + for (const run of state.runs) { + if (run.trigger === "scheduled" && run.state === "running") reserved.add(run.runKey); + } + return reserved; +} + +/** True when another scheduled run could no longer reserve a durable failure cooldown. */ export function cooldownCapacityExhausted(state: LabAutomationStateV1, now: number): boolean { - return Object.keys(activeCooldowns(state, now)).length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns; + return reservedCooldownKeys(state, now).size >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns; +} + +/** Reserve capacity implicitly through the running scheduled record before dispatch. */ +export function canReserveScheduledCooldown( + state: LabAutomationStateV1, + runKey: string, + now: number, +): boolean { + const reserved = reservedCooldownKeys(state, now); + return reserved.has(runKey) || reserved.size < LAB_AUTOMATION_HARD_MAX.maxPersistedRuns; } export function setCooldown( From 7c09d512b4a24b6093e49b2a5149ffc69d9ef269 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:35:38 +0200 Subject: [PATCH 42/67] fix(lab): fence dispatch deps and cooldown reservations --- src/lab/automation/orchestrator.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lab/automation/orchestrator.ts b/src/lab/automation/orchestrator.ts index fa69ea063e..734c47ece4 100644 --- a/src/lab/automation/orchestrator.ts +++ b/src/lab/automation/orchestrator.ts @@ -15,7 +15,12 @@ import { isRunBudgetExhausted, isLiveRequestBudgetExhausted, } from "./budgets"; -import { clearCooldown, cooldownForFailure, setCooldown } from "./cooldown"; +import { + canReserveScheduledCooldown, + clearCooldown, + cooldownForFailure, + setCooldown, +} from "./cooldown"; import { enqueuePlannedRuns, selectDispatchableRuns, @@ -48,9 +53,7 @@ function configKey(configDir?: string): string { } function dispatchDepsFor(configDir?: string): AutomationDispatchDeps { - return dispatchDepsByConfigDir.get(configKey(configDir)) - ?? dispatchDepsByConfigDir.get("") - ?? {}; + return dispatchDepsByConfigDir.get(configKey(configDir)) ?? {}; } function cancellationCooldownUntil(policy: LabAutomationPolicyV1, now: number): number { @@ -199,6 +202,9 @@ function claimNextRun( }; const dispatchable = selectDispatchableRuns(policy, state, now, (run) => { if (!predicate(run)) return false; + if (run.trigger === "scheduled" && !canReserveScheduledCooldown(state, run.runKey, now)) { + return false; + } if (run.evidenceLayer === "live_route_compatibility" && isLiveRequestBudgetExhausted(policy, state, now)) { return false; } From 568f670c0c9d5d31d4a7b7e7bbc8850bdddf62b9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:40:39 +0200 Subject: [PATCH 43/67] fix(lab): publish state locks atomically --- src/lab/automation/persistence.ts | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/lab/automation/persistence.ts b/src/lab/automation/persistence.ts index b0cbd77a95..ef9ddb399b 100644 --- a/src/lab/automation/persistence.ts +++ b/src/lab/automation/persistence.ts @@ -2,6 +2,8 @@ import { randomUUID } from "node:crypto"; import { closeSync, existsSync, + fsyncSync, + linkSync, openSync, readFileSync, renameSync, @@ -198,25 +200,45 @@ function reclaimDeadStateLock(lockPath: string): boolean { } } +function cleanupPrivateLockFile(path: string): void { + try { + unlinkSync(path); + } catch { + /* absent or already cleaned */ + } +} + function acquireStateLock(configDir?: string): () => void { ensureLabDirs(configDir); const lockPath = stateLockPath(configDir); const deadline = Date.now() + STATE_LOCK_WAIT_MS; while (true) { const token = randomUUID(); + const privatePath = `${lockPath}.${process.pid}.${token}.tmp`; + let publicationAttempted = false; try { - const fd = openSync(lockPath, "wx", 0o600); + const fd = openSync(privatePath, "wx", 0o600); try { const meta: StateLockMeta = { pid: process.pid, token }; writeFileSync(fd, JSON.stringify(meta), { encoding: "utf8" }); + fsyncSync(fd); } finally { closeSync(fd); } + + // Publish only a fully written metadata inode. A hard-link create is atomic and fails + // with EEXIST when another owner already published the canonical lock path. + publicationAttempted = true; + linkSync(privatePath, lockPath); + cleanupPrivateLockFile(privatePath); return () => releaseStateLock(lockPath, token); } catch (error) { + cleanupPrivateLockFile(privatePath); const code = error && typeof error === "object" && "code" in error ? String((error as { code?: unknown }).code) : undefined; + // A UUID-named private file collision is not ownership contention; retry with a new token. + if (code === "EEXIST" && !publicationAttempted) continue; if (code !== "EEXIST") throw new LabAutomationError("automation state lock failed", "state_lock_failed"); if (reclaimDeadStateLock(lockPath)) continue; if (Date.now() >= deadline) { From 7cdb374ffd6fe5bb5ac7fb75f8073327063cf3bd Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:41:03 +0200 Subject: [PATCH 44/67] fix(lab): add bounded exact freshness query --- src/lab/query/freshness.ts | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/lab/query/freshness.ts diff --git a/src/lab/query/freshness.ts b/src/lab/query/freshness.ts new file mode 100644 index 0000000000..d638436bb2 --- /dev/null +++ b/src/lab/query/freshness.ts @@ -0,0 +1,53 @@ +import type { EvidenceLayer } from "../constants"; +import { closeLabReadConnection, openLabReadConnection } from "./connection"; + +export interface ExactObservationIdentity { + subjectId: string; + layer: EvidenceLayer; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + scenarioId: string; + scenarioVersion: string; + scenarioManifestDigest: string; +} + +/** Return the newest non-excluded observation for one exact evidence identity. */ +export function queryLatestExactObservationCompletedAt( + identity: ExactObservationIdentity, + configDir?: string, +): number | undefined { + const conn = openLabReadConnection(configDir); + try { + const row = conn.db + .query( + `SELECT o.completed_at + FROM observations o + JOIN events e ON e.event_id = o.event_id + WHERE o.subject_id = ? + AND o.evidence_layer = ? + AND o.suite_id = ? + AND o.suite_version = ? + AND o.suite_manifest_digest = ? + AND o.scenario_id = ? + AND o.scenario_version = ? + AND o.scenario_manifest_digest = ? + AND e.excluded = 0 + ORDER BY o.completed_at DESC, o.event_id DESC + LIMIT 1`, + ) + .get( + identity.subjectId, + identity.layer, + identity.suiteId, + identity.suiteVersion, + identity.suiteManifestDigest, + identity.scenarioId, + identity.scenarioVersion, + identity.scenarioManifestDigest, + ) as { completed_at: number } | null; + return row ? Number(row.completed_at) : undefined; + } finally { + closeLabReadConnection(conn); + } +} From b7990aea1f1318e398a17df8d486787b3f0cc87e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:25:45 +0200 Subject: [PATCH 45/67] test(lab): cover final CL-08 review blockers --- ...ation-final-coderabbit-regressions.test.ts | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 tests/lab-automation-final-coderabbit-regressions.test.ts diff --git a/tests/lab-automation-final-coderabbit-regressions.test.ts b/tests/lab-automation-final-coderabbit-regressions.test.ts new file mode 100644 index 0000000000..8c5bd41f13 --- /dev/null +++ b/tests/lab-automation-final-coderabbit-regressions.test.ts @@ -0,0 +1,234 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { OcxConfig } from "../src/types"; +import { LAB_SQLITE_DDL } from "../src/lab/projection/schema"; +import { defaultLabAutomationPolicyV1 } from "../src/lab/automation/policy"; +import { + defaultLabAutomationStateV1, + loadLabAutomationState, + saveLabAutomationPolicy, + saveLabAutomationRoutes, + saveLabAutomationState, +} from "../src/lab/automation/persistence"; +import { planLabAutomationRuns, planManualLabRun } from "../src/lab/automation/planner"; +import { + cancelLabAutomationRun, + requestLabAutomationShutdown, + resetLabAutomationSchedulerStateForTests, + runLabAutomationTick, + setLabAutomationDispatchDeps, + stopLabAutomationScheduler, +} from "../src/lab/automation/orchestrator"; +import { LAB_AUTOMATION_HARD_MAX } from "../src/lab/automation/constants"; +import type { LabAutomationRunRecordV1 } from "../src/lab/automation/types"; +import { createHostIssuedLabRouteExecutor } from "../src/lib/lab-live-host"; +import type { NormalizedObservation } from "../src/lab/conformance/types"; +import { readInstallationSalt } from "../src/lab/subject/installation-salt"; +import { + resetCompatibilityVersionCacheForTests, + setCompatibilityVersionOverrideForTests, +} from "../src/routing/compatibility/version"; + +const HOMES: string[] = []; +const COMPAT_VERSION = "9".repeat(64); + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-final-review-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + process.env.OPENCODEX_HOME = dir; + readInstallationSalt(dir); + setCompatibilityVersionOverrideForTests(COMPAT_VERSION); + return dir; +} + +function protocolPolicy() { + return { + ...defaultLabAutomationPolicyV1(), + enabled: true, + layers: { protocolConformance: true, liveRouteCompatibility: false, taskEffectiveness: false }, + failureCooldownMs: 0, + }; +} + +function livePolicy() { + return { + ...defaultLabAutomationPolicyV1(), + enabled: true, + layers: { protocolConformance: false, liveRouteCompatibility: true, taskEffectiveness: false }, + failureCooldownMs: 0, + }; +} + +function liveConfig(): OcxConfig { + return { + providers: { + "fixture-provider": { + adapter: "openai-responses", + baseUrl: "https://api.example.com/v1", + apiKey: "sk-fixture", + models: ["fixture-model"], + defaultModel: "fixture-model", + }, + }, + } as OcxConfig; +} + +function passObservation(): NormalizedObservation { + return { + client: { + request: { status: 200, headers: {}, json: {}, rawBytes: 0 }, + response: { + status: 200, + headers: {}, + json: { status: "completed", output: [{ type: "message", content: [{ type: "output_text", text: "OK" }] }] }, + events: [], + toolCalls: [], + mcpCalls: [], + terminal: "completed", + normalizedText: "OK", + }, + }, + upstream: { requests: [], responses: [] }, + process: { exitCode: null }, + verifiers: {}, + }; +} + +function activeCooldowns(count: number, now: number): Record { + return Object.fromEntries( + Array.from({ length: count }, (_, index) => [`occupied-${index}`, now + 60 * 60_000]), + ); +} + +function scheduledRecord(plan: ReturnType, runId: string, now: number): LabAutomationRunRecordV1 { + return { + ...plan, + runId, + state: "queued", + trigger: "scheduled", + createdAt: now, + updatedAt: now, + }; +} + +afterEach(() => { + requestLabAutomationShutdown(); + stopLabAutomationScheduler(); + resetLabAutomationSchedulerStateForTests(); + resetCompatibilityVersionCacheForTests(); + delete process.env.OPENCODEX_HOME; + for (const dir of HOMES.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("CL-08 final CodeRabbit regressions", () => { + test("freshness lookup is exact and index-backed instead of subject-wide pagination", () => { + const plannerSource = readFileSync(join(import.meta.dir, "../src/lab/automation/planner.ts"), "utf8"); + expect(plannerSource).not.toContain("FRESHNESS_QUERY_PAGE_SIZE"); + expect(plannerSource).toContain("queryLatestLabObservation"); + expect(LAB_SQLITE_DDL).toContain("idx_observations_exact_identity"); + }); + + test("queued cancellation remains a planner-honored backoff when cooldown storage is saturated", () => { + const home = tempHome(); + const policy = protocolPolicy(); + saveLabAutomationPolicy(policy, home); + const plan = planManualLabRun({ + evidenceLayer: "protocol_conformance", + scenarioId: "responses-core.protocol.request-shape", + configDir: home, + }); + const now = Date.now(); + saveLabAutomationState({ + ...defaultLabAutomationStateV1(now), + runs: [scheduledRecord(plan, "queued-cancel", now)], + cooldownUntilByKey: activeCooldowns(LAB_AUTOMATION_HARD_MAX.maxPersistedRuns, now), + }, home); + + expect(cancelLabAutomationRun("queued-cancel", home)).toBe(true); + let state = loadLabAutomationState(home); + expect(state.runs.find((row) => row.runId === "queued-cancel")?.state).toBe("cancelled"); + expect(state.cooldownUntilByKey[plan.runKey]).toBeUndefined(); + + delete state.cooldownUntilByKey["occupied-0"]; + saveLabAutomationState(state, home); + state = loadLabAutomationState(home); + const replanned = planLabAutomationRuns({ + policy, + routes: { schemaVersion: 1, routes: [] }, + state, + now: now + 1, + configDir: home, + }); + expect(replanned.some((row) => row.runKey === plan.runKey)).toBe(false); + }); + + test("in-flight cancellation remains a planner-honored backoff when cooldown storage saturates during dispatch", async () => { + const home = tempHome(); + const config = liveConfig(); + const policy = livePolicy(); + saveLabAutomationPolicy(policy, home); + saveLabAutomationRoutes({ + schemaVersion: 1, + routes: [{ providerName: "fixture-provider", modelId: "fixture-model" }], + }, home); + const plan = planManualLabRun({ + evidenceLayer: "live_route_compatibility", + scenarioId: "responses-core.live.basic-turn", + providerName: "fixture-provider", + modelId: "fixture-model", + config, + configDir: home, + }); + const now = Date.now(); + saveLabAutomationState({ + ...defaultLabAutomationStateV1(now), + runs: [scheduledRecord(plan, "inflight-cancel", now)], + cooldownUntilByKey: activeCooldowns(LAB_AUTOMATION_HARD_MAX.maxPersistedRuns - 1, now), + }, home); + + let startedResolve!: () => void; + const started = new Promise((resolve) => { startedResolve = resolve; }); + setLabAutomationDispatchDeps({ + configDir: home, + loadConfig: () => config, + resolve: async () => [{ address: "93.184.216.34", family: 4 as const }], + routeExecutor: createHostIssuedLabRouteExecutor(async (input) => { + startedResolve(); + if (!input.signal.aborted) { + await new Promise((resolve) => input.signal.addEventListener("abort", () => resolve(), { once: true })); + } + return passObservation(); + }), + }); + + const tick = runLabAutomationTick(home); + await started; + let state = loadLabAutomationState(home); + const running = state.runs.find((row) => row.runId === "inflight-cancel"); + expect(running?.state).toBe("running"); + state.cooldownUntilByKey["late-slot"] = now + 60 * 60_000; + saveLabAutomationState(state, home); + + expect(cancelLabAutomationRun("inflight-cancel", home)).toBe(true); + await tick; + state = loadLabAutomationState(home); + expect(state.runs.find((row) => row.runId === "inflight-cancel")?.state).toBe("cancelled"); + expect(state.cooldownUntilByKey[plan.runKey]).toBeUndefined(); + + delete state.cooldownUntilByKey["occupied-0"]; + saveLabAutomationState(state, home); + state = loadLabAutomationState(home); + const replanned = planLabAutomationRuns({ + policy, + routes: { schemaVersion: 1, routes: [{ providerName: "fixture-provider", modelId: "fixture-model" }] }, + state, + now: Date.now(), + config, + configDir: home, + }); + expect(replanned.some((row) => row.runKey === plan.runKey)).toBe(false); + }); +}); From 5cad25a3e40a4654c957ceff356d8bb6d70a5b83 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:41:01 +0200 Subject: [PATCH 46/67] fix(lab): close final CL-08 review blockers --- src/lab/automation/planner.ts | 97 +++++++++-------------------- src/lab/automation/queue.ts | 32 ++++++++-- src/lab/projection/schema.ts | 15 ++++- src/lab/query/latest-observation.ts | 59 ++++++++++++++++++ 4 files changed, 129 insertions(+), 74 deletions(-) create mode 100644 src/lab/query/latest-observation.ts diff --git a/src/lab/automation/planner.ts b/src/lab/automation/planner.ts index 212f6917d3..6d2db47c5d 100644 --- a/src/lab/automation/planner.ts +++ b/src/lab/automation/planner.ts @@ -10,7 +10,7 @@ import { buildAutomationLiveRouteContext } from "./route-context"; import { liveSuiteManifestDigestForCase } from "../live/suite-manifest"; import { suiteManifestDigestForCase } from "../conformance/suite-manifest"; import { isScenarioApplicable } from "../projection/verification"; -import { queryLabObservations } from "../query/queries"; +import { queryLatestLabObservation } from "../query/latest-observation"; import { LabProjectionUnavailableError } from "../query/errors"; import type { OcxConfig } from "../../types"; import { resolvePolicyCompatibilitySubjects } from "../../routing/compatibility/subject"; @@ -33,11 +33,6 @@ import { cooldownActive, cooldownCapacityExhausted } from "./cooldown"; import { isLiveRequestBudgetExhausted, isRunBudgetExhausted } from "./budgets"; import { LAB_AUTOMATION_HARD_MAX } from "./constants"; -const FRESHNESS_QUERY_PAGE_SIZE = 100; - -type FreshnessIndex = Map; -type FreshnessCache = Map; - export interface PlannerInput { policy: LabAutomationPolicyV1; routes: LabAutomationRoutesV1; @@ -77,71 +72,36 @@ function activeRunForKey(state: LabAutomationStateV1, runKey: string): LabAutoma return state.runs.find((row) => row.runKey === runKey && (row.state === "queued" || row.state === "running")); } -function freshnessIdentityKey(identity: Omit): string { - return [ - identity.suiteId, - identity.suiteVersion, - identity.suiteManifestDigest, - identity.scenarioId, - identity.scenarioVersion, - identity.scenarioManifestDigest, - ].join("\u0000"); +function cancellationBackoffActive( + state: LabAutomationStateV1, + runKey: string, + failureCooldownMs: number, + now: number, +): boolean { + const backoffMs = Math.max(failureCooldownMs, LAB_AUTOMATION_HARD_MAX.schedulerTickMs); + return state.runs.some((row) => + row.runKey === runKey + && row.trigger === "scheduled" + && row.state === "cancelled" + && row.terminalCode === "cancelled" + && typeof row.completedAt === "number" + && row.completedAt + backoffMs > now + ); } -function freshnessIndexForSubject( - layer: EvidenceLayer, - subjectId: string, - cache: FreshnessCache, +function latestMatchingObservationCompletedAt( + identity: EvidenceIdentity, configDir?: string, -): FreshnessIndex { - const cacheKey = `${layer}\u0000${subjectId}`; - const cached = cache.get(cacheKey); - if (cached) return cached; - - const index: FreshnessIndex = new Map(); - cache.set(cacheKey, index); +): number | undefined { try { - let cursor: string | undefined; - do { - const page = queryLabObservations( - { layer, subjectId }, - cursor, - FRESHNESS_QUERY_PAGE_SIZE, - configDir, - ); - for (const row of page.items) { - if (row.excluded) continue; - const key = freshnessIdentityKey({ - suiteId: row.suiteId, - suiteVersion: row.suiteVersion, - suiteManifestDigest: row.suiteManifestDigest, - scenarioId: row.scenarioId, - scenarioVersion: row.scenarioVersion, - scenarioManifestDigest: row.scenarioManifestDigest, - }); - // Query order is newest first, so first exact identity is authoritative for freshness. - if (!index.has(key)) index.set(key, row.completedAt); - } - cursor = page.hasMore ? page.nextCursor : undefined; - } while (cursor); - return index; + return queryLatestLabObservation(identity, configDir); } catch (error) { - if (error instanceof LabProjectionUnavailableError) return index; + if (error instanceof LabProjectionUnavailableError) return undefined; throw error; } } -function latestMatchingObservationCompletedAt( - identity: EvidenceIdentity, - cache: FreshnessCache, - configDir?: string, -): number | undefined { - return freshnessIndexForSubject(identity.layer, identity.subjectId, cache, configDir).get( - freshnessIdentityKey(identity), - ); -} - -function planProtocolScenarios(input: PlannerInput, freshnessCache: FreshnessCache): PlannedLabRunV1[] { +function planProtocolScenarios(input: PlannerInput): PlannedLabRunV1[] { if (!input.policy.enabled || !input.policy.layers.protocolConformance) return []; const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, CL01_SUITES); @@ -173,6 +133,7 @@ function planProtocolScenarios(input: PlannerInput, freshnessCache: FreshnessCac executionContractDigest: protocolExecutionContractDigest(), }); if (activeRunForKey(input.state, runKey)) continue; + if (cancellationBackoffActive(input.state, runKey, input.policy.failureCooldownMs, input.now)) continue; if (cooldownActive(input.state, runKey, input.now)) continue; const latestCompletedAt = latestMatchingObservationCompletedAt({ layer: "protocol_conformance", @@ -183,7 +144,7 @@ function planProtocolScenarios(input: PlannerInput, freshnessCache: FreshnessCac scenarioId: caseRecord.id, scenarioVersion, scenarioManifestDigest: scenarioDigest, - }, freshnessCache, input.configDir); + }, input.configDir); // CL-00 freezes one freshness default into both suite and scenario manifests; there is no // independent case-level override in the current authority schema. const maxAge = authority.manifestDefaults.freshness.maxAgeMs; @@ -207,7 +168,7 @@ function planProtocolScenarios(input: PlannerInput, freshnessCache: FreshnessCac return planned; } -function planLiveScenarios(input: PlannerInput, freshnessCache: FreshnessCache): PlannedLabRunV1[] { +function planLiveScenarios(input: PlannerInput): PlannedLabRunV1[] { if (!input.policy.enabled || !input.policy.layers.liveRouteCompatibility) return []; if (!input.config) return []; if (isLiveRequestBudgetExhausted(input.policy, input.state, input.now)) return []; @@ -249,6 +210,7 @@ function planLiveScenarios(input: PlannerInput, freshnessCache: FreshnessCache): executionContractDigest: liveExecutionContractDigest(), }); if (activeRunForKey(input.state, runKey)) continue; + if (cancellationBackoffActive(input.state, runKey, input.policy.failureCooldownMs, input.now)) continue; if (cooldownActive(input.state, runKey, input.now)) continue; const latestCompletedAt = latestMatchingObservationCompletedAt({ layer: "live_route_compatibility", @@ -259,7 +221,7 @@ function planLiveScenarios(input: PlannerInput, freshnessCache: FreshnessCache): scenarioId: caseRecord.id, scenarioVersion, scenarioManifestDigest: scenarioDigest, - }, freshnessCache, input.configDir); + }, input.configDir); const maxAge = authority.manifestDefaults.freshness.maxAgeMs; const freshness = freshnessReason(latestCompletedAt, maxAge, input.policy.refreshBeforeStaleMs, input.now); if (freshness === "fresh") continue; @@ -291,9 +253,8 @@ export function planLabAutomationRuns(input: PlannerInput): PlannedLabRunV1[] { // Cooldown persistence is bounded. If every slot contains an active backoff, fail closed rather // than enqueueing work whose retry suppression could not be recorded. if (cooldownCapacityExhausted(input.state, input.now)) return []; - const freshnessCache: FreshnessCache = new Map(); - const protocol = planProtocolScenarios(input, freshnessCache); - const live = planLiveScenarios(input, freshnessCache); + const protocol = planProtocolScenarios(input); + const live = planLiveScenarios(input); const merged = [...protocol, ...live]; merged.sort((a, b) => { if (a.priority !== b.priority) return a.priority - b.priority; diff --git a/src/lab/automation/queue.ts b/src/lab/automation/queue.ts index 7e3e91da70..524798ae79 100644 --- a/src/lab/automation/queue.ts +++ b/src/lab/automation/queue.ts @@ -26,13 +26,35 @@ function isRollingBudgetEvidence(run: LabAutomationRunRecordV1, now: number): bo return run.startedAt > cutoff && run.startedAt <= now; } -/** Evict only disposable terminal history; trailing-window attempts are budget authority. */ +function isCancellationBackoffEvidence(run: LabAutomationRunRecordV1, now: number): boolean { + if ( + run.trigger !== "scheduled" + || run.state !== "cancelled" + || run.terminalCode !== "cancelled" + || typeof run.completedAt !== "number" + ) { + return false; + } + const maxBackoffMs = Math.max( + LAB_AUTOMATION_HARD_MAX.failureCooldownMs, + LAB_AUTOMATION_HARD_MAX.schedulerTickMs, + ); + return run.completedAt + maxBackoffMs > now; +} + +/** Evict only disposable terminal history; budget and cancellation records remain authority. */ function evictOldestTerminal(runs: LabAutomationRunRecordV1[], now: number): boolean { let oldestIndex = -1; let oldestTime = Number.POSITIVE_INFINITY; for (let index = 0; index < runs.length; index += 1) { const run = runs[index]!; - if (!isTerminal(run) || isRollingBudgetEvidence(run, now)) continue; + if ( + !isTerminal(run) + || isRollingBudgetEvidence(run, now) + || isCancellationBackoffEvidence(run, now) + ) { + continue; + } const terminalAt = run.completedAt ?? run.updatedAt; if (terminalAt < oldestTime) { oldestTime = terminalAt; @@ -172,9 +194,9 @@ export function trimTerminalRuns(state: LabAutomationStateV1, now: number): LabA const keepMs = LAB_AUTOMATION_HARD_MAX.terminalRunRetentionMs; const runs = state.runs.filter((row) => { if (!isTerminal(row)) return true; - // Retention is longer than the rolling budget window, but keep the authority check explicit - // so a future retention change cannot erase attempts that still bound provider traffic. - if (isRollingBudgetEvidence(row, now)) return true; + // Retention exceeds both the rolling budget window and maximum cancellation backoff. + // Keep both authority checks explicit so future retention changes cannot erase them. + if (isRollingBudgetEvidence(row, now) || isCancellationBackoffEvidence(row, now)) return true; const completedAt = row.completedAt ?? row.updatedAt; return now - completedAt < keepMs; }); diff --git a/src/lab/projection/schema.ts b/src/lab/projection/schema.ts index a8519ad12e..01c612edec 100644 --- a/src/lab/projection/schema.ts +++ b/src/lab/projection/schema.ts @@ -2,7 +2,7 @@ * Disposable SQLite projection schema for Compatibility Lab (CL-02). * Not canonical storage — rebuildable from JSONL + content-addressed artifacts. */ -export const LAB_SQLITE_SCHEMA_VERSION = 2; +export const LAB_SQLITE_SCHEMA_VERSION = 3; export const LAB_SQLITE_DDL = ` CREATE TABLE IF NOT EXISTS schema_meta ( @@ -50,6 +50,19 @@ CREATE INDEX IF NOT EXISTS idx_observations_proj ON observations( subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest, completed_at ); +CREATE INDEX IF NOT EXISTS idx_observations_exact_identity ON observations( + evidence_layer, + subject_id, + suite_id, + suite_version, + suite_manifest_digest, + scenario_id, + scenario_version, + scenario_manifest_digest, + completed_at DESC, + event_id DESC +); + CREATE TABLE IF NOT EXISTS claims ( event_id TEXT PRIMARY KEY, subject_id TEXT NOT NULL, diff --git a/src/lab/query/latest-observation.ts b/src/lab/query/latest-observation.ts new file mode 100644 index 0000000000..2505dd76d2 --- /dev/null +++ b/src/lab/query/latest-observation.ts @@ -0,0 +1,59 @@ +import type { EvidenceLayer } from "../constants"; +import { closeLabReadConnection, openLabReadConnection } from "./connection"; + +export interface LatestLabObservationIdentity { + layer: EvidenceLayer; + subjectId: string; + suiteId: string; + suiteVersion: string; + suiteManifestDigest: string; + scenarioId: string; + scenarioVersion: string; + scenarioManifestDigest: string; +} + +/** + * Return the newest non-excluded observation for one exact evidence identity. + * + * The projection schema provides a composite index for every identity predicate + * and the descending freshness order, so planner work is independent of unrelated + * observation history retained for the same subject. + */ +export function queryLatestLabObservation( + identity: LatestLabObservationIdentity, + configDir?: string, +): number | undefined { + const conn = openLabReadConnection(configDir); + try { + const row = conn.db + .query( + `SELECT o.completed_at + FROM observations o + JOIN events e ON e.event_id = o.event_id + WHERE o.evidence_layer = ? + AND o.subject_id = ? + AND o.suite_id = ? + AND o.suite_version = ? + AND o.suite_manifest_digest = ? + AND o.scenario_id = ? + AND o.scenario_version = ? + AND o.scenario_manifest_digest = ? + AND e.excluded = 0 + ORDER BY o.completed_at DESC, o.event_id DESC + LIMIT 1`, + ) + .get( + identity.layer, + identity.subjectId, + identity.suiteId, + identity.suiteVersion, + identity.suiteManifestDigest, + identity.scenarioId, + identity.scenarioVersion, + identity.scenarioManifestDigest, + ) as { completed_at: number } | null; + return row ? Number(row.completed_at) : undefined; + } finally { + closeLabReadConnection(conn); + } +} From e9914571156120f830312c03080c096a4a78ac0a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:44:43 +0200 Subject: [PATCH 47/67] test(lab): prove bounded exact freshness lookup --- ...ation-final-coderabbit-regressions.test.ts | 60 ++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/lab-automation-final-coderabbit-regressions.test.ts b/tests/lab-automation-final-coderabbit-regressions.test.ts index 8c5bd41f13..a1b402ec46 100644 --- a/tests/lab-automation-final-coderabbit-regressions.test.ts +++ b/tests/lab-automation-final-coderabbit-regressions.test.ts @@ -1,9 +1,13 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; import { mkdirSync, readFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import type { OcxConfig } from "../src/types"; -import { LAB_SQLITE_DDL } from "../src/lab/projection/schema"; +import { LAB_PROJECTION_SPEC_VERSION } from "../src/lab/constants"; +import { ensureLabDirs, labSqlitePath } from "../src/lab/paths"; +import { LAB_SQLITE_DDL, LAB_SQLITE_SCHEMA_VERSION } from "../src/lab/projection/schema"; +import { queryLatestLabObservation } from "../src/lab/query/latest-observation"; import { defaultLabAutomationPolicyV1 } from "../src/lab/automation/policy"; import { defaultLabAutomationStateV1, @@ -114,6 +118,46 @@ function scheduledRecord(plan: ReturnType, runId: strin }; } +function buildFreshnessProjection(home: string): void { + ensureLabDirs(home); + const db = new Database(labSqlitePath(home), { create: true }); + try { + db.exec(LAB_SQLITE_DDL); + const insertMeta = db.query("INSERT INTO schema_meta(key, value) VALUES (?, ?)"); + insertMeta.run("schema_version", String(LAB_SQLITE_SCHEMA_VERSION)); + insertMeta.run("projection_spec_version", LAB_PROJECTION_SPEC_VERSION); + insertMeta.run("built_at_ms", "1"); + + const insertEvent = db.query( + "INSERT INTO events(event_id, event_kind, recorded_at, producer, producer_version, payload_json, excluded) VALUES (?, 'observation', ?, 'test', '1', '{}', 0)", + ); + const insertObservation = db.query( + `INSERT INTO observations( + event_id, subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest, + scenario_id, scenario_version, scenario_manifest_digest, outcome, completed_at, execution_mode + ) VALUES (?, 'subject', 'protocol_conformance', 'suite', '1', ?, ?, '1', ?, 'pass', ?, 'fixture')`, + ); + + db.transaction(() => { + insertEvent.run("target", 10); + insertObservation.run("target", "suite-digest", "target-scenario", "target-digest", 10); + for (let index = 0; index < 2_000; index += 1) { + const eventId = `irrelevant-${index}`; + insertEvent.run(eventId, 1_000 + index); + insertObservation.run( + eventId, + `irrelevant-suite-${index}`, + `irrelevant-scenario-${index}`, + `irrelevant-digest-${index}`, + 1_000 + index, + ); + } + })(); + } finally { + db.close(); + } +} + afterEach(() => { requestLabAutomationShutdown(); stopLabAutomationScheduler(); @@ -124,11 +168,23 @@ afterEach(() => { }); describe("CL-08 final CodeRabbit regressions", () => { - test("freshness lookup is exact and index-backed instead of subject-wide pagination", () => { + test("freshness lookup stays exact and bounded across large irrelevant history", () => { + const home = tempHome(); + buildFreshnessProjection(home); const plannerSource = readFileSync(join(import.meta.dir, "../src/lab/automation/planner.ts"), "utf8"); expect(plannerSource).not.toContain("FRESHNESS_QUERY_PAGE_SIZE"); expect(plannerSource).toContain("queryLatestLabObservation"); expect(LAB_SQLITE_DDL).toContain("idx_observations_exact_identity"); + expect(queryLatestLabObservation({ + layer: "protocol_conformance", + subjectId: "subject", + suiteId: "suite", + suiteVersion: "1", + suiteManifestDigest: "suite-digest", + scenarioId: "target-scenario", + scenarioVersion: "1", + scenarioManifestDigest: "target-digest", + }, home)).toBe(10); }); test("queued cancellation remains a planner-honored backoff when cooldown storage is saturated", () => { From e2bf73e6184dfaf7a979a907086bf44c9c414e10 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:17:08 +0200 Subject: [PATCH 48/67] test(lab): initialize persisted-state fixture dirs --- tests/lab-automation-coderabbit-regressions.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/lab-automation-coderabbit-regressions.test.ts b/tests/lab-automation-coderabbit-regressions.test.ts index d92e05c4aa..2a8cdc3586 100644 --- a/tests/lab-automation-coderabbit-regressions.test.ts +++ b/tests/lab-automation-coderabbit-regressions.test.ts @@ -24,7 +24,7 @@ import { } from "../src/lab/automation/orchestrator"; import type { LabAutomationRunRecordV1, PlannedLabRunV1 } from "../src/lab/automation/types"; import { LabAutomationError } from "../src/lab/automation/types"; -import { labAutomationStatePath } from "../src/lab/paths"; +import { ensureLabDirs, labAutomationStatePath } from "../src/lab/paths"; import { handleManagementAPI } from "../src/server/management-api"; import { ManagementRequest } from "./helpers/management-auth"; import { handleLabCommand } from "../src/cli/lab"; @@ -150,6 +150,7 @@ describe("CL-08 CodeRabbit regressions", () => { test("persisted state rejects duplicate identities and impossible lifecycle fields", () => { const home = tempHome(); + ensureLabDirs(home); const writeState = (runs: unknown[]) => { writeFileSync(labAutomationStatePath(home), JSON.stringify({ schemaVersion: 1, From 253a109a6b78e10107af5762166737b9cdc047b0 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:55:57 +0200 Subject: [PATCH 49/67] test(lab): reproduce cancellation history persistence overflow --- ...utomation-persisted-cap-regression.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/lab-automation-persisted-cap-regression.test.ts diff --git a/tests/lab-automation-persisted-cap-regression.test.ts b/tests/lab-automation-persisted-cap-regression.test.ts new file mode 100644 index 0000000000..3c1ead2034 --- /dev/null +++ b/tests/lab-automation-persisted-cap-regression.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import { LAB_AUTOMATION_HARD_MAX } from "../src/lab/automation/constants"; +import { defaultLabAutomationStateV1 } from "../src/lab/automation/persistence"; +import { trimTerminalRuns } from "../src/lab/automation/queue"; +import type { LabAutomationRunRecordV1 } from "../src/lab/automation/types"; + +function cancelledScheduledRun(index: number, now: number): LabAutomationRunRecordV1 { + return { + runId: `cancelled-${index}`, + runKey: `cancelled-key-${index}`, + state: "cancelled", + evidenceLayer: "protocol_conformance", + suiteId: "suite", + suiteVersion: "1", + suiteManifestDigest: "a".repeat(64), + scenarioId: `scenario-${index}`, + scenarioVersion: "1", + scenarioManifestDigest: "b".repeat(64), + subjectId: `subject-${index}`, + reason: "missing", + priority: 0, + eligibleAt: now - 1, + trigger: "scheduled", + createdAt: now - 1, + updatedAt: now, + completedAt: now, + terminalCode: "cancelled", + }; +} + +describe("CL-08 persisted run ceiling regressions", () => { + test("cancellation backoff history never makes trimTerminalRuns exceed maxPersistedRuns", () => { + const now = Date.now(); + const state = { + ...defaultLabAutomationStateV1(now), + runs: Array.from( + { length: LAB_AUTOMATION_HARD_MAX.maxPersistedRuns + 1 }, + (_, index) => cancelledScheduledRun(index, now), + ), + }; + + const trimmed = trimTerminalRuns(state, now); + + expect(trimmed.runs).toHaveLength(LAB_AUTOMATION_HARD_MAX.maxPersistedRuns); + }); +}); From f7598f088a6963e32bbe4856050697a072bd7f40 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:57:22 +0200 Subject: [PATCH 50/67] fix(lab): fail closed when cooldown persistence saturates --- src/lab/automation/cooldown.ts | 38 ++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/lab/automation/cooldown.ts b/src/lab/automation/cooldown.ts index 15bef95f42..f3ab677c83 100644 --- a/src/lab/automation/cooldown.ts +++ b/src/lab/automation/cooldown.ts @@ -1,8 +1,22 @@ import { LAB_AUTOMATION_HARD_MAX } from "./constants"; import type { LabAutomationPolicyV1, LabAutomationStateV1 } from "./types"; +/** + * A persisted, expiring tombstone used only when the bounded per-key cooldown map + * cannot accept another key. While active, scheduled planning fails closed for all + * run keys. Its expiry covers every cooldown that was collapsed into it. + */ +const COOLDOWN_SATURATION_KEY = "__ocx_cooldown_saturation__"; + +function saturationActive(cooldowns: Record, now: number): boolean { + const until = cooldowns[COOLDOWN_SATURATION_KEY]; + return typeof until === "number" && until > now; +} + export function cooldownActive(state: LabAutomationStateV1, key: string, now: number): boolean { - const until = state.cooldownUntilByKey[key]; + const cooldowns = activeCooldowns(state, now); + if (saturationActive(cooldowns, now)) return true; + const until = cooldowns[key]; return typeof until === "number" && until > now; } @@ -24,7 +38,10 @@ function reservedCooldownKeys(state: LabAutomationStateV1, now: number): Set= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns; + const cooldowns = activeCooldowns(state, now); + if (saturationActive(cooldowns, now)) return true; + return reservedCooldownKeys({ ...state, cooldownUntilByKey: cooldowns }, now).size + >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns; } /** Reserve capacity implicitly through the running scheduled record before dispatch. */ @@ -33,7 +50,9 @@ export function canReserveScheduledCooldown( runKey: string, now: number, ): boolean { - const reserved = reservedCooldownKeys(state, now); + const cooldowns = activeCooldowns(state, now); + if (saturationActive(cooldowns, now)) return false; + const reserved = reservedCooldownKeys({ ...state, cooldownUntilByKey: cooldowns }, now); return reserved.has(runKey) || reserved.size < LAB_AUTOMATION_HARD_MAX.maxPersistedRuns; } @@ -44,9 +63,20 @@ export function setCooldown( now: number, ): LabAutomationStateV1 { const next = activeCooldowns(state, now); - if (!(key in next) && Object.keys(next).length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { + const saturationUntil = next[COOLDOWN_SATURATION_KEY]; + if (typeof saturationUntil === "number") { + next[COOLDOWN_SATURATION_KEY] = Math.max(saturationUntil, untilMs); return { ...state, cooldownUntilByKey: next }; } + if (!(key in next) && Object.keys(next).length >= LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { + const longestExistingUntil = Math.max(now, ...Object.values(next)); + return { + ...state, + cooldownUntilByKey: { + [COOLDOWN_SATURATION_KEY]: Math.max(longestExistingUntil, untilMs), + }, + }; + } next[key] = untilMs; return { ...state, cooldownUntilByKey: next }; } From 7607cf6b35ce687b8241d53fc9996fde9c796bd2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:57:55 +0200 Subject: [PATCH 51/67] fix(lab): keep persisted run history within hard ceiling --- src/lab/automation/queue.ts | 52 +++++++++++++------------------------ 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/src/lab/automation/queue.ts b/src/lab/automation/queue.ts index 524798ae79..bade0fa324 100644 --- a/src/lab/automation/queue.ts +++ b/src/lab/automation/queue.ts @@ -1,10 +1,11 @@ import { randomUUID } from "node:crypto"; -import type { - LabAutomationPolicyV1, - LabAutomationRunRecordV1, - LabAutomationRunState, - LabAutomationStateV1, - PlannedLabRunV1, +import { + LabAutomationError, + type LabAutomationPolicyV1, + type LabAutomationRunRecordV1, + type LabAutomationRunState, + type LabAutomationStateV1, + type PlannedLabRunV1, } from "./types"; import { LAB_AUTOMATION_HARD_MAX } from "./constants"; @@ -26,35 +27,13 @@ function isRollingBudgetEvidence(run: LabAutomationRunRecordV1, now: number): bo return run.startedAt > cutoff && run.startedAt <= now; } -function isCancellationBackoffEvidence(run: LabAutomationRunRecordV1, now: number): boolean { - if ( - run.trigger !== "scheduled" - || run.state !== "cancelled" - || run.terminalCode !== "cancelled" - || typeof run.completedAt !== "number" - ) { - return false; - } - const maxBackoffMs = Math.max( - LAB_AUTOMATION_HARD_MAX.failureCooldownMs, - LAB_AUTOMATION_HARD_MAX.schedulerTickMs, - ); - return run.completedAt + maxBackoffMs > now; -} - -/** Evict only disposable terminal history; budget and cancellation records remain authority. */ +/** Evict only disposable terminal history; rolling budget records remain authority. */ function evictOldestTerminal(runs: LabAutomationRunRecordV1[], now: number): boolean { let oldestIndex = -1; let oldestTime = Number.POSITIVE_INFINITY; for (let index = 0; index < runs.length; index += 1) { const run = runs[index]!; - if ( - !isTerminal(run) - || isRollingBudgetEvidence(run, now) - || isCancellationBackoffEvidence(run, now) - ) { - continue; - } + if (!isTerminal(run) || isRollingBudgetEvidence(run, now)) continue; const terminalAt = run.completedAt ?? run.updatedAt; if (terminalAt < oldestTime) { oldestTime = terminalAt; @@ -194,14 +173,19 @@ export function trimTerminalRuns(state: LabAutomationStateV1, now: number): LabA const keepMs = LAB_AUTOMATION_HARD_MAX.terminalRunRetentionMs; const runs = state.runs.filter((row) => { if (!isTerminal(row)) return true; - // Retention exceeds both the rolling budget window and maximum cancellation backoff. - // Keep both authority checks explicit so future retention changes cannot erase them. - if (isRollingBudgetEvidence(row, now) || isCancellationBackoffEvidence(row, now)) return true; + // The retention window exceeds the rolling budget window. Keep budget evidence explicit + // so a future retention reduction cannot over-grant the hourly run budgets. + if (isRollingBudgetEvidence(row, now)) return true; const completedAt = row.completedAt ?? row.updatedAt; return now - completedAt < keepMs; }); while (runs.length > LAB_AUTOMATION_HARD_MAX.maxPersistedRuns) { - if (!evictOldestTerminal(runs, now)) break; + if (!evictOldestTerminal(runs, now)) { + throw new LabAutomationError( + "persisted run ceiling exceeded by non-disposable run records", + "invalid_state", + ); + } } return { ...state, runs }; } From 82f7e5790258c0fd3bdeb07269e724b108385a47 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:02:17 +0200 Subject: [PATCH 52/67] test(lab): carry parent-owned fabric isolation budgets to child --- src/lab/fabric/producer-protocol.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lab/fabric/producer-protocol.ts b/src/lab/fabric/producer-protocol.ts index ab20548467..496b5a1def 100644 --- a/src/lab/fabric/producer-protocol.ts +++ b/src/lab/fabric/producer-protocol.ts @@ -19,6 +19,9 @@ export interface ProducerParentRequest { executorModulePath?: string; scratchRoot: string; executorInput?: unknown; + /** Parent-owned budgets are passed through so synthetic harness timing can scale in tests. */ + totalTimeoutMs?: number; + inactivityTimeoutMs?: number; } export interface IsolatedProducerResult { From 382b72df3b522c56c9fb80756c99f26b8bdba359 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:02:51 +0200 Subject: [PATCH 53/67] test(lab): add guarded fabric isolation limit override --- src/lab/fabric/producer-isolate.ts | 31 ++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/lab/fabric/producer-isolate.ts b/src/lab/fabric/producer-isolate.ts index c808de0e12..6a2aa45617 100644 --- a/src/lab/fabric/producer-isolate.ts +++ b/src/lab/fabric/producer-isolate.ts @@ -14,6 +14,13 @@ import { FabricTaskError } from "./types"; const CHILD_ENTRY = join(dirname(fileURLToPath(import.meta.url)), "producer-child.ts"); +type FabricProducerIsolationLimits = { + totalTimeoutMs: number; + inactivityTimeoutMs: number; +}; + +let testIsolationLimits: FabricProducerIsolationLimits | undefined; + interface IsolateRequest { harnessKind?: FabricHarnessProducerKind; executorModulePath?: string; @@ -181,6 +188,8 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis harnessKind: request.harnessKind, executorModulePath: request.executorModulePath, scratchRoot: request.scratchRoot, + totalTimeoutMs: request.totalTimeoutMs, + inactivityTimeoutMs: request.inactivityTimeoutMs, executorInput: request.executorInput ? { routeContext: request.executorInput.routeContext, @@ -240,10 +249,28 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis }); } +/** Internal test seam. Production callers cannot arm it without the test-home guard. */ +export function setFabricProducerIsolationLimitsForTests(limits?: FabricProducerIsolationLimits): void { + if (process.env.OCX_TEST_HOME_GUARD !== "1") { + throw new Error("fabric isolation limits can only be overridden by the test harness"); + } + if (limits && ( + !Number.isFinite(limits.totalTimeoutMs) + || !Number.isFinite(limits.inactivityTimeoutMs) + || limits.totalTimeoutMs <= 0 + || limits.inactivityTimeoutMs <= 0 + || limits.inactivityTimeoutMs >= limits.totalTimeoutMs + )) { + throw new Error("invalid fabric test isolation limits"); + } + testIsolationLimits = limits ? { ...limits } : undefined; +} + /** Default isolation limits for fabric producer child processes. */ -export function fabricProducerIsolationLimits(): { totalTimeoutMs: number; inactivityTimeoutMs: number } { +export function fabricProducerIsolationLimits(): FabricProducerIsolationLimits { + if (testIsolationLimits) return { ...testIsolationLimits }; return { totalTimeoutMs: FABRIC_LIMITS.totalTimeoutMs, inactivityTimeoutMs: FABRIC_LIMITS.inactivityTimeoutMs, }; -} \ No newline at end of file +} From a8066646009cfaddc53a6ca526959215a2db4c64 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:03:15 +0200 Subject: [PATCH 54/67] test(lab): scale synthetic producer timing to parent budgets --- src/lab/fabric/producer-child.ts | 35 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/lab/fabric/producer-child.ts b/src/lab/fabric/producer-child.ts index 6a2f23349f..f42c4e3cd3 100644 --- a/src/lab/fabric/producer-child.ts +++ b/src/lab/fabric/producer-child.ts @@ -30,7 +30,12 @@ function wrongPatch(): SyntheticPatchV1 { }; } -async function runHarness(kind: FabricHarnessProducerKind, scratchRoot: string): Promise { +async function runHarness( + kind: FabricHarnessProducerKind, + scratchRoot: string, + totalTimeoutMs: number, + inactivityTimeoutMs: number, +): Promise { switch (kind) { case "deterministic_correct": return correctPatch(); @@ -42,27 +47,32 @@ async function runHarness(kind: FabricHarnessProducerKind, scratchRoot: string): } case "never_resolve": while (true) { - await Bun.sleep(1_000); + await Bun.sleep(Math.max(25, Math.min(1_000, Math.floor(inactivityTimeoutMs * 0.75)))); } case "mutate_after_delay": { - await Bun.sleep(FABRIC_LIMITS.totalTimeoutMs + 5_000); + await Bun.sleep(totalTimeoutMs + Math.max(250, inactivityTimeoutMs)); const { writeFileSync, mkdirSync } = await import("node:fs"); const { join, dirname } = await import("node:path"); mkdirSync(join(scratchRoot, dirname(SYNTHETIC_VALUE_PATH)), { recursive: true }); writeFileSync(join(scratchRoot, SYNTHETIC_VALUE_PATH), "late\n"); return correctPatch(); } - case "periodic_activity": - for (let i = 0; i < 20; i++) { + case "periodic_activity": { + const intervalMs = Math.max(10, Math.min(200, Math.floor(inactivityTimeoutMs * 0.4))); + const iterations = Math.max(3, Math.min(20, Math.floor((totalTimeoutMs * 0.7) / intervalMs))); + for (let i = 0; i < iterations; i++) { writeLine({ type: "activity" }); - await Bun.sleep(200); + await Bun.sleep(intervalMs); } return correctPatch(); - case "activity_until_total": + } + case "activity_until_total": { + const intervalMs = Math.max(10, Math.min(50, Math.floor(inactivityTimeoutMs * 0.25))); while (true) { writeLine({ type: "activity" }); - await Bun.sleep(50); + await Bun.sleep(intervalMs); } + } case "flood_stdout": while (true) { process.stdout.write("x".repeat(4096)); @@ -99,10 +109,17 @@ async function main(): Promise { return; } const req = JSON.parse(raw) as ProducerParentRequest; + const totalTimeoutMs = req.totalTimeoutMs ?? FABRIC_LIMITS.totalTimeoutMs; + const inactivityTimeoutMs = req.inactivityTimeoutMs ?? FABRIC_LIMITS.inactivityTimeoutMs; let patch: SyntheticPatchV1; const reportActivity = () => writeLine({ type: "activity" }); if (req.harnessKind) { - patch = await runHarness(req.harnessKind as FabricHarnessProducerKind, req.scratchRoot); + patch = await runHarness( + req.harnessKind as FabricHarnessProducerKind, + req.scratchRoot, + totalTimeoutMs, + inactivityTimeoutMs, + ); } else if (req.executorModulePath) { patch = await runExecutorModule(req.executorModulePath, req.executorInput, reportActivity); } else { From b2a909312847d1f51d5c5f99c198d6237860ab8e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:08:06 +0200 Subject: [PATCH 55/67] test(lab): shorten CL-07 isolation timing budgets --- tests/lab-fabric-task.test.ts | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/lab-fabric-task.test.ts b/tests/lab-fabric-task.test.ts index e63c19fa9d..b9b8c34422 100644 --- a/tests/lab-fabric-task.test.ts +++ b/tests/lab-fabric-task.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import * as nodeFs from "node:fs"; import { existsSync, @@ -54,6 +54,7 @@ import { ensureLabDirs, ensureRestrictedDir } from "../src/lab/paths"; import { verifyExactTreeDiffV1 } from "../src/lab/fabric/verifier"; import { parseSyntheticPatchV1 } from "../src/lab/fabric/patch"; import { FABRIC_LIMITS } from "../src/lab/fabric/constants"; +import { setFabricProducerIsolationLimitsForTests } from "../src/lab/fabric/producer-isolate"; import { taskSubjectApplicableToRequirements } from "../src/lab/projection/verification"; import { createHostIssuedFabricPatchExecutor } from "../src/lib/fabric-task-host"; import type { TrustedFabricPatchExecutor } from "../src/lab/fabric/types"; @@ -67,6 +68,10 @@ import { const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); const CREDENTIAL_CANARY = "credential-canary-abcdefghijklmnopqrstuvwxyz1234567890"; +const FAST_FABRIC_ISOLATION = Object.freeze({ + totalTimeoutMs: 2_000, + inactivityTimeoutMs: 750, +}); const HOMES: string[] = []; @@ -101,7 +106,7 @@ import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/la export async function execute(input: FabricPatchExecutorInput): Promise { for (let i = 0; i < 3; i++) { input.reportActivity(); - await Bun.sleep(Math.floor(${FABRIC_LIMITS.inactivityTimeoutMs} * 0.6)); + await Bun.sleep(Math.floor(${FAST_FABRIC_ISOLATION.inactivityTimeoutMs} * 0.4)); } return { schemaVersion: 1, @@ -112,7 +117,7 @@ export async function execute(input: FabricPatchExecutorInput): Promise { for (let i = 0; i < 3; i++) { input.reportActivity(); - await Bun.sleep(Math.floor(FABRIC_LIMITS.inactivityTimeoutMs * 0.6)); + await Bun.sleep(Math.floor(FAST_FABRIC_ISOLATION.inactivityTimeoutMs * 0.4)); } return correctSyntheticPatch(); }); @@ -127,7 +132,7 @@ import type { FabricPatchExecutorInput, SyntheticPatchV1 } from "${repoImport("s import { SYNTHETIC_AFTER_UTF8, SYNTHETIC_VALUE_PATH } from "${repoImport("src/lab/fabric/constants")}"; export async function execute(_input: FabricPatchExecutorInput): Promise { - await Bun.sleep(${FABRIC_LIMITS.inactivityTimeoutMs} + 50); + await Bun.sleep(${FAST_FABRIC_ISOLATION.inactivityTimeoutMs} + 50); return { schemaVersion: 1, operations: [{ op: "replace", path: SYNTHETIC_VALUE_PATH, contentUtf8: SYNTHETIC_AFTER_UTF8 }], @@ -135,7 +140,7 @@ export async function execute(_input: FabricPatchExecutorInput): Promise { - await Bun.sleep(FABRIC_LIMITS.inactivityTimeoutMs + 50); + await Bun.sleep(FAST_FABRIC_ISOLATION.inactivityTimeoutMs + 50); return correctSyntheticPatch(); }); } @@ -232,7 +237,12 @@ function linkDirectory(target: string, linkPath: string): boolean { const RESTRICTED_LINK_ERROR = /symbolic link|reparse-point substitution/i; +beforeEach(() => { + setFabricProducerIsolationLimitsForTests(FAST_FABRIC_ISOLATION); +}); + afterEach(() => { + setFabricProducerIsolationLimitsForTests(); for (const dir of HOMES.splice(0)) { try { rmSync(dir, { recursive: true, force: true }); @@ -525,7 +535,7 @@ describe("CL-07 task effectiveness producer", () => { expect(["blocked", "inconclusive"]).toContain(result.outcome.outcome); expect(["timeout", "inactivity_timeout", "harness_failure"]).toContain(result.outcome.failure?.code ?? ""); const scratchBase = join(home, "lab", "scratch"); - await Bun.sleep(6_000); + await Bun.sleep(FAST_FABRIC_ISOLATION.totalTimeoutMs + FAST_FABRIC_ISOLATION.inactivityTimeoutMs + 250); if (existsSync(scratchBase)) { expect(readdirSync(scratchBase).some((name) => name.startsWith("fabric-"))).toBe(false); } @@ -1071,4 +1081,4 @@ describe("CL-07 task effectiveness producer", () => { expect(text.includes("system prompt")).toBe(false); expect(text.includes(CREDENTIAL_CANARY)).toBe(false); }); -}); +}); \ No newline at end of file From 9e5b6e30414ced158605e1945aa5dc203a4bf082 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:17:39 +0200 Subject: [PATCH 56/67] test(lab): track current projection schema version --- tests/lab-read-surfaces.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/lab-read-surfaces.test.ts b/tests/lab-read-surfaces.test.ts index 87fedc3584..ec8fff2ac5 100644 --- a/tests/lab-read-surfaces.test.ts +++ b/tests/lab-read-surfaces.test.ts @@ -15,6 +15,7 @@ import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, LAB_PROJECTION_SPEC_VERSION, + LAB_SQLITE_SCHEMA_VERSION, } from "../src/lab"; import { createArtifactStore } from "../src/lab/artifacts/store"; import { resolveProtocolExecutionContext } from "../src/lab/conformance/executor"; @@ -148,7 +149,7 @@ describe("CL-04 query layer", () => { seedProjection(home); const status = queryLabStatus(home); expect(status.projectionAvailable).toBe(true); - expect(status.sqliteSchemaVersion).toBe(2); + expect(status.sqliteSchemaVersion).toBe(LAB_SQLITE_SCHEMA_VERSION); expect(status.projectionSpecVersion).toBe(LAB_PROJECTION_SPEC_VERSION); expect(status.verdictCount).toBeGreaterThan(0); @@ -472,4 +473,4 @@ describe("CL-04 privacy boundary", () => { expect(serializedEvent).not.toContain("C:\\Users"); }); }); -}); +}); \ No newline at end of file From e5332346943526b7b833e3fb202aecd2af11425e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:08:24 +0200 Subject: [PATCH 57/67] test(lab): assert fail-closed cooldown saturation --- tests/lab-automation-coderabbit-regressions.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/lab-automation-coderabbit-regressions.test.ts b/tests/lab-automation-coderabbit-regressions.test.ts index 2a8cdc3586..eec5e19028 100644 --- a/tests/lab-automation-coderabbit-regressions.test.ts +++ b/tests/lab-automation-coderabbit-regressions.test.ts @@ -14,7 +14,7 @@ import { import { LAB_AUTOMATION_HARD_MAX } from "../src/lab/automation/constants"; import { enqueuePlannedRuns } from "../src/lab/automation/queue"; import { runBudgetRemaining } from "../src/lab/automation/budgets"; -import { setCooldown } from "../src/lab/automation/cooldown"; +import { cooldownActive, setCooldown } from "../src/lab/automation/cooldown"; import { listLabAutomationRuns } from "../src/lab/automation/runs-query"; import { cancelLabAutomationRun, @@ -144,8 +144,12 @@ describe("CL-08 CodeRabbit regressions", () => { now + 20_000, now, ); - expect(Object.keys(saturated.cooldownUntilByKey)).toHaveLength(LAB_AUTOMATION_HARD_MAX.maxPersistedRuns); + expect(Object.keys(saturated.cooldownUntilByKey).length).toBeLessThanOrEqual(LAB_AUTOMATION_HARD_MAX.maxPersistedRuns); expect(saturated.cooldownUntilByKey.overflow).toBeUndefined(); + expect(cooldownActive(saturated, "active-0", now)).toBe(true); + expect(cooldownActive(saturated, "overflow", now)).toBe(true); + expect(cooldownActive(saturated, "unrelated-key", now)).toBe(true); + expect(cooldownActive(saturated, "unrelated-key", now + 20_001)).toBe(false); }); test("persisted state rejects duplicate identities and impossible lifecycle fields", () => { From dd04b8ac5d9cd6c5e0c5b6552219624615ad3817 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:47:39 +0200 Subject: [PATCH 58/67] test(lab): reproduce maintainer lifecycle blockers --- ...ab-automation-ingwannu-regressions.test.ts | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 tests/lab-automation-ingwannu-regressions.test.ts diff --git a/tests/lab-automation-ingwannu-regressions.test.ts b/tests/lab-automation-ingwannu-regressions.test.ts new file mode 100644 index 0000000000..6d123721df --- /dev/null +++ b/tests/lab-automation-ingwannu-regressions.test.ts @@ -0,0 +1,133 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdirSync, readFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { OcxConfig } from "../src/types"; +import { defaultLabAutomationPolicyV1 } from "../src/lab/automation/policy"; +import * as persistence from "../src/lab/automation/persistence"; +import { + isLabAutomationSchedulerRunning, + resetLabAutomationSchedulerStateForTests, + runLabAutomationTick, + setLabAutomationDispatchDeps, + startLabAutomationScheduler, +} from "../src/lab/automation/orchestrator"; + +const HOMES: string[] = []; + +function tempHome(): string { + const dir = join(tmpdir(), `ocx-lab-ingwannu-${process.pid}-${Math.random().toString(16).slice(2)}`); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + HOMES.push(dir); + return dir; +} + +function emptyConfig(): OcxConfig { + return { providers: {} } as OcxConfig; +} + +afterEach(() => { + resetLabAutomationSchedulerStateForTests(); + for (const dir of HOMES.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("CL-08 Ingwannu regressions", () => { + test("same-root successor survives predecessor runtime release", async () => { + const home = tempHome(); + let firstLoads = 0; + let secondLoads = 0; + + const releaseFirst = setLabAutomationDispatchDeps({ + configDir: home, + loadConfig: () => { + firstLoads += 1; + return emptyConfig(); + }, + }) as unknown; + expect(typeof releaseFirst).toBe("function"); + if (typeof releaseFirst !== "function") return; + + startLabAutomationScheduler(home); + expect(isLabAutomationSchedulerRunning(home)).toBe(true); + + const releaseSecond = setLabAutomationDispatchDeps({ + configDir: home, + loadConfig: () => { + secondLoads += 1; + return emptyConfig(); + }, + }) as unknown; + expect(typeof releaseSecond).toBe("function"); + if (typeof releaseSecond !== "function") return; + + startLabAutomationScheduler(home); + releaseFirst(); + expect(isLabAutomationSchedulerRunning(home)).toBe(true); + + await runLabAutomationTick(home); + expect(firstLoads).toBe(0); + expect(secondLoads).toBeGreaterThan(0); + + releaseSecond(); + expect(isLabAutomationSchedulerRunning(home)).toBe(false); + }); + + test("failed combined automation config commit leaves the prior generation effective", async () => { + const home = tempHome(); + const saveConfig = Reflect.get(persistence, "saveLabAutomationConfig") as unknown; + const loadConfig = Reflect.get(persistence, "loadLabAutomationConfig") as unknown; + const setCommitFault = Reflect.get(persistence, "setLabAutomationConfigCommitFaultForTests") as unknown; + + expect(typeof saveConfig).toBe("function"); + expect(typeof loadConfig).toBe("function"); + expect(typeof setCommitFault).toBe("function"); + if (typeof saveConfig !== "function" || typeof loadConfig !== "function" || typeof setCommitFault !== "function") return; + + const initialPolicy = { + ...defaultLabAutomationPolicyV1(), + enabled: true, + layers: { + protocolConformance: false, + liveRouteCompatibility: true, + taskEffectiveness: false, + }, + }; + const initialRoutes = persistence.defaultLabAutomationRoutesV1(); + saveConfig(initialPolicy, initialRoutes, home); + + const nextPolicy = { + ...initialPolicy, + failureCooldownMs: initialPolicy.failureCooldownMs + 1, + }; + const nextRoutes = { + schemaVersion: 1 as const, + routes: [{ providerName: "provider-new", modelId: "model-new" }], + }; + + setCommitFault("before_publish"); + expect(() => saveConfig(nextPolicy, nextRoutes, home)).toThrow(); + setCommitFault(null); + + const effective = loadConfig(home) as { + policy: typeof initialPolicy; + routes: typeof initialRoutes; + }; + expect(effective.policy).toEqual(initialPolicy); + expect(effective.routes).toEqual(initialRoutes); + + await runLabAutomationTick(home); + expect(persistence.loadLabAutomationState(home).runs).toHaveLength(0); + }); + + test("CL-08 plan has no trailing whitespace", () => { + const text = readFileSync( + join(import.meta.dir, "..", "devlog", "_plan", "260807_compatibility_lab", "008_cl08_automation.md"), + "utf8", + ); + const offenders = text + .split("\n") + .map((line, index) => ({ line, lineNumber: index + 1 })) + .filter(({ line }) => /[ \t]+$/.test(line)); + expect(offenders).toEqual([]); + }); +}); From 6220ea7d5c60081eb2f94cf07ab2e83744cb7089 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:52:07 +0200 Subject: [PATCH 59/67] fix(lab): scope runtime resources to server owners --- src/lib/server-resource-ownership.ts | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/lib/server-resource-ownership.ts diff --git a/src/lib/server-resource-ownership.ts b/src/lib/server-resource-ownership.ts new file mode 100644 index 0000000000..7a4fd799eb --- /dev/null +++ b/src/lib/server-resource-ownership.ts @@ -0,0 +1,71 @@ +type Cleanup = () => void; + +type ServerResourceOwner = { + token: symbol; + cleanups: Set; +}; + +export type ServerResourceOwnerLease = { + release(): void; +}; + +const owners: ServerResourceOwner[] = []; + +function currentOwner(): ServerResourceOwner | undefined { + return owners.at(-1); +} + +/** Acquire ownership for resources created while one server instance starts. */ +export function acquireServerResourceOwner(): ServerResourceOwnerLease { + const owner: ServerResourceOwner = { + token: Symbol("server-resource-owner"), + cleanups: new Set(), + }; + owners.push(owner); + + let released = false; + return { + release() { + if (released) return; + released = true; + const index = owners.findIndex(candidate => candidate.token === owner.token); + if (index !== -1) owners.splice(index, 1); + for (const cleanup of [...owner.cleanups]) { + try { + cleanup(); + } catch { + // Resource cleanup is best-effort and one failed hook must not retain siblings. + } + } + owner.cleanups.clear(); + }, + }; +} + +/** Attach cleanup to the server instance currently being started. */ +export function registerCurrentServerResourceCleanup(cleanup: Cleanup): () => void { + const owner = currentOwner(); + if (!owner) return () => {}; + owner.cleanups.add(cleanup); + let attached = true; + return () => { + if (!attached) return; + attached = false; + owner.cleanups.delete(cleanup); + }; +} + +/** Test-only reset for isolated lifecycle tests. */ +export function resetServerResourceOwnershipForTests(): void { + for (const owner of [...owners].reverse()) { + for (const cleanup of [...owner.cleanups]) { + try { + cleanup(); + } catch { + // Test reset must leave no retained owner even when a synthetic cleanup throws. + } + } + owner.cleanups.clear(); + } + owners.length = 0; +} From 1998e97f72d15c70c64cb12a71dd13632d48764f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:52:31 +0200 Subject: [PATCH 60/67] fix(lab): release server-owned runtime resources --- src/server/background-lifecycle.ts | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/server/background-lifecycle.ts b/src/server/background-lifecycle.ts index 4c1b95e059..5625d6b67b 100644 --- a/src/server/background-lifecycle.ts +++ b/src/server/background-lifecycle.ts @@ -14,6 +14,10 @@ import { cancelQueuedStorageWorkerSpawns, drainStorageWorkers, } from "../storage/worker-lifecycle"; +import { + acquireServerResourceOwner, + type ServerResourceOwnerLease, +} from "../lib/server-resource-ownership"; import { startMemoryWatchdog, type MemoryWatchdog, @@ -29,6 +33,7 @@ type ProcessLoops = { type LeaseOwner = { token: symbol; applyPolicy: PolicyApply; + resources: ServerResourceOwnerLease; }; export type ServerBackgroundLifecycleLease = { @@ -103,6 +108,7 @@ function removeOwner(owner: LeaseOwner): boolean { function releaseOwnerSynchronously(owner: LeaseOwner): "inactive" | "shared" | "last" { if (!removeOwner(owner)) return "inactive"; + owner.resources.release(); const nextOwner = owners.at(-1); if (nextOwner) { setLivePolicyOwner(nextOwner.applyPolicy); @@ -117,7 +123,8 @@ function releaseOwnerSynchronously(owner: LeaseOwner): "inactive" | "shared" | " * * The first live server starts the loops. Later servers only add a reference and * become the current policy sink; out-of-order release restores the newest - * remaining sink. The final release owns timer and Worker teardown. + * remaining sink. The final release owns timer and Worker teardown. Resources + * registered while this server starts are released with this exact lease. */ export function acquireServerBackgroundLifecycle( applyPolicy: PolicyApply, @@ -129,13 +136,19 @@ export function acquireServerBackgroundLifecycle( const owner: LeaseOwner = { token: Symbol("server-background-lifecycle"), applyPolicy, + resources: acquireServerResourceOwner(), }; - if (!processLoops) { - processLoops = startProcessLoops(applyPolicy); - } else { - setLivePolicyOwner(applyPolicy); + try { + if (!processLoops) { + processLoops = startProcessLoops(applyPolicy); + } else { + setLivePolicyOwner(applyPolicy); + } + owners.push(owner); + } catch (error) { + owner.resources.release(); + throw error; } - owners.push(owner); let releaseFlight: Promise | null = null; return { @@ -166,4 +179,4 @@ export function acquireServerBackgroundLifecycle( releaseFlight = Promise.resolve(); }, }; -} +} \ No newline at end of file From 07a15cce829434d5f3d4e3a64445d957f7267334 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:54:01 +0200 Subject: [PATCH 61/67] fix(lab): commit automation config atomically --- src/lab/automation/config-persistence.ts | 256 +++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 src/lab/automation/config-persistence.ts diff --git a/src/lab/automation/config-persistence.ts b/src/lab/automation/config-persistence.ts new file mode 100644 index 0000000000..e4dca2992f --- /dev/null +++ b/src/lab/automation/config-persistence.ts @@ -0,0 +1,256 @@ +import { randomUUID } from "node:crypto"; +import { + closeSync, + existsSync, + fsyncSync, + linkSync, + openSync, + readFileSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import { + ensureLabDirs, + labAutomationPolicyPath, +} from "../paths"; +import { normalizeLabAutomationPolicyV1 } from "./policy"; +import { + loadLabAutomationPolicy, + loadLabAutomationRoutes, + normalizeLabAutomationRoutesV1, +} from "./persistence"; +import type { LabAutomationPolicyV1, LabAutomationRoutesV1 } from "./types"; +import { LabAutomationError } from "./types"; + +const CONFIG_SCHEMA_VERSION = 1 as const; +const CONFIG_KEYS = new Set(["schemaVersion", "policy", "routes"]); +const CONFIG_LOCK_WAIT_MS = 5_000; +const LOCK_SLEEP = new Int32Array(new SharedArrayBuffer(4)); + +interface ConfigLockMeta { + pid: number; + token: string; +} + +export interface LabAutomationConfigV1 { + schemaVersion: typeof CONFIG_SCHEMA_VERSION; + policy: LabAutomationPolicyV1; + routes: LabAutomationRoutesV1; +} + +type ConfigCommitFault = "before_publish" | null; +let configCommitFaultForTests: ConfigCommitFault = null; + +function configPath(configDir?: string): string { + return join(dirname(labAutomationPolicyPath(configDir)), "automation-config.json"); +} + +function configLockPath(configDir?: string): string { + return `${configPath(configDir)}.lock`; +} + +function sleepLockRetry(): void { + Atomics.wait(LOCK_SLEEP, 0, 0, 10); +} + +function readLockMeta(lockPath: string): ConfigLockMeta | null { + try { + const parsed = JSON.parse(readFileSync(lockPath, "utf8")) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; + const row = parsed as Record; + if (!Number.isSafeInteger(row.pid) || (row.pid as number) <= 0) return null; + if (typeof row.token !== "string" || row.token.length < 16 || row.token.length > 128) return null; + return { pid: row.pid as number, token: row.token }; + } catch { + return null; + } +} + +function pidDefinitelyDead(pid: number): boolean { + try { + process.kill(pid, 0); + return false; + } catch (error) { + const code = error && typeof error === "object" && "code" in error + ? String((error as { code?: unknown }).code) + : undefined; + return code === "ESRCH"; + } +} + +function cleanupFile(path: string): void { + try { + unlinkSync(path); + } catch { + // Absent or already cleaned. + } +} + +function releaseLock(lockPath: string, token: string): void { + try { + if (readLockMeta(lockPath)?.token === token) unlinkSync(lockPath); + } catch { + // Already released or replaced. + } +} + +function reclaimDeadLock(lockPath: string): boolean { + const observed = readLockMeta(lockPath); + if (!observed || !pidDefinitelyDead(observed.pid)) return false; + const current = readLockMeta(lockPath); + if (!current || current.pid !== observed.pid || current.token !== observed.token) return false; + try { + unlinkSync(lockPath); + return true; + } catch { + return false; + } +} + +function acquireConfigLock(configDir?: string): () => void { + ensureLabDirs(configDir); + const lockPath = configLockPath(configDir); + const deadline = Date.now() + CONFIG_LOCK_WAIT_MS; + while (true) { + const token = randomUUID(); + const privatePath = `${lockPath}.${process.pid}.${token}.tmp`; + let publicationAttempted = false; + try { + const fd = openSync(privatePath, "wx", 0o600); + try { + writeFileSync(fd, JSON.stringify({ pid: process.pid, token }), { encoding: "utf8" }); + fsyncSync(fd); + } finally { + closeSync(fd); + } + publicationAttempted = true; + linkSync(privatePath, lockPath); + cleanupFile(privatePath); + return () => releaseLock(lockPath, token); + } catch (error) { + cleanupFile(privatePath); + const code = error && typeof error === "object" && "code" in error + ? String((error as { code?: unknown }).code) + : undefined; + if (code === "EEXIST" && !publicationAttempted) continue; + if (code !== "EEXIST") { + throw new LabAutomationError("automation config lock failed", "state_lock_failed"); + } + if (reclaimDeadLock(lockPath)) continue; + if (Date.now() >= deadline) { + throw new LabAutomationError("automation config is busy", "state_lock_busy"); + } + sleepLockRetry(); + } + } +} + +function withConfigLock(configDir: string | undefined, action: () => T): T { + const release = acquireConfigLock(configDir); + try { + return action(); + } finally { + release(); + } +} + +function normalizeConfig(raw: unknown): LabAutomationConfigV1 { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + throw new LabAutomationError("automation config must be an object", "invalid_state"); + } + const row = raw as Record; + for (const key of Object.keys(row)) { + if (!CONFIG_KEYS.has(key)) { + throw new LabAutomationError(`unknown automation config field ${key}`, "invalid_state"); + } + } + if (row.schemaVersion !== CONFIG_SCHEMA_VERSION) { + throw new LabAutomationError("unsupported automation config schemaVersion", "invalid_state"); + } + return Object.freeze({ + schemaVersion: CONFIG_SCHEMA_VERSION, + policy: normalizeLabAutomationPolicyV1(row.policy), + routes: normalizeLabAutomationRoutesV1(row.routes), + }); +} + +function loadConfigUnlocked(configDir?: string): LabAutomationConfigV1 { + ensureLabDirs(configDir); + const path = configPath(configDir); + if (!existsSync(path)) { + return Object.freeze({ + schemaVersion: CONFIG_SCHEMA_VERSION, + policy: loadLabAutomationPolicy(configDir), + routes: loadLabAutomationRoutes(configDir), + }); + } + try { + return normalizeConfig(JSON.parse(readFileSync(path, "utf8")) as unknown); + } catch (error) { + if (error instanceof LabAutomationError) throw error; + throw new LabAutomationError("invalid automation config JSON", "invalid_state"); + } +} + +function writeConfigUnlocked( + policy: LabAutomationPolicyV1, + routes: LabAutomationRoutesV1, + configDir?: string, +): LabAutomationConfigV1 { + ensureLabDirs(configDir); + const normalized = Object.freeze({ + schemaVersion: CONFIG_SCHEMA_VERSION, + policy: normalizeLabAutomationPolicyV1(policy), + routes: normalizeLabAutomationRoutesV1(routes), + }); + const path = configPath(configDir); + const tmp = join(dirname(path), `.automation-config.${process.pid}.${randomUUID()}.tmp`); + let fd: number | null = null; + try { + fd = openSync(tmp, "wx", 0o600); + writeFileSync(fd, JSON.stringify(normalized), { encoding: "utf8" }); + fsyncSync(fd); + closeSync(fd); + fd = null; + if (configCommitFaultForTests === "before_publish") { + throw new LabAutomationError("synthetic automation config commit failure", "invalid_state"); + } + renameSync(tmp, path); + return normalized; + } finally { + if (fd !== null) closeSync(fd); + cleanupFile(tmp); + } +} + +/** Load one coherent policy/routes generation. */ +export function loadLabAutomationConfig(configDir?: string): LabAutomationConfigV1 { + return withConfigLock(configDir, () => loadConfigUnlocked(configDir)); +} + +/** Atomically publish one coherent policy/routes generation. */ +export function saveLabAutomationConfig( + policy: LabAutomationPolicyV1, + routes: LabAutomationRoutesV1, + configDir?: string, +): LabAutomationConfigV1 { + return withConfigLock(configDir, () => writeConfigUnlocked(policy, routes, configDir)); +} + +/** Update only policy while preserving routes from the same committed generation. */ +export function saveLabAutomationPolicyConfig( + policy: LabAutomationPolicyV1, + configDir?: string, +): LabAutomationConfigV1 { + return withConfigLock(configDir, () => { + const current = loadConfigUnlocked(configDir); + return writeConfigUnlocked(policy, current.routes, configDir); + }); +} + +/** Test-only fault injection before the atomic rename publication point. */ +export function setLabAutomationConfigCommitFaultForTests(fault: ConfigCommitFault): void { + configCommitFaultForTests = fault; +} From 40b75393b6b987532c0f74e6f69018a16c49a212 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:54:59 +0200 Subject: [PATCH 62/67] fix(lab): publish policy and routes as one config --- .../management/lab-automation-routes.ts | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/server/management/lab-automation-routes.ts b/src/server/management/lab-automation-routes.ts index eb0eea8cce..dfaf0f4e90 100644 --- a/src/server/management/lab-automation-routes.ts +++ b/src/server/management/lab-automation-routes.ts @@ -21,15 +21,16 @@ import { } from "../../lab/automation/orchestrator"; import { planManualLabRun } from "../../lab/automation/planner"; import { - loadLabAutomationPolicy, loadLabAutomationState, normalizeLabAutomationRoutesV1, - saveLabAutomationPolicy, - saveLabAutomationRoutes, } from "../../lab/automation/persistence"; +import { + loadLabAutomationConfig, + saveLabAutomationConfig, +} from "../../lab/automation/config-persistence"; import { normalizeLabAutomationPolicyV1 } from "../../lab/automation/policy"; import { listLabAutomationRuns } from "../../lab/automation/runs-query"; -import type { LabAutomationLayer, LabAutomationPolicyV1, LabAutomationRoutesV1 } from "../../lab/automation/types"; +import type { LabAutomationLayer, LabAutomationPolicyV1 } from "../../lab/automation/types"; import { LabAutomationError } from "../../lab/automation/types"; import { jsonResponse } from "../auth-cors"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; @@ -165,8 +166,7 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise if (!isPlainRecord(body)) return automationErrorResponse("invalid_body", "body must be an object", 400, ctx); // Only policy/routes are authoritative. Other top-level properties are ignored rather than // treated as injectable process-local capabilities (for example a fake routeExecutor). - let policy = loadLabAutomationPolicy(configDir); - let routes: LabAutomationRoutesV1 | undefined; + let { policy, routes } = loadLabAutomationConfig(configDir); if (body.policy !== undefined) { if (!isPlainRecord(body.policy)) return automationErrorResponse("invalid_policy", "policy must be an object", 400, ctx); const incoming = body.policy as Record; @@ -186,10 +186,11 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise routes = normalizeLabAutomationRoutesV1(body.routes); } - // Validate the complete requested update before persisting either file so an invalid routes - // payload cannot leave a valid policy half-applied. - if (body.routes !== undefined && routes) saveLabAutomationRoutes(routes, configDir); - if (body.policy !== undefined) saveLabAutomationPolicy(policy, configDir); + // One atomic rename publishes policy and routes as a coherent generation. A failed write + // leaves the previous generation authoritative and scheduler reconciliation is not applied. + if (body.policy !== undefined || body.routes !== undefined) { + saveLabAutomationConfig(policy, routes, configDir); + } applySchedulerPolicy(policy, configDir); return jsonResponse(buildLabAutomationStatus(configDir), 200, req, config); } catch (error) { @@ -202,4 +203,4 @@ export async function handleLabAutomationRoutes(ctx: ManagementContext): Promise } return automationErrorResponse("not_found", "unknown resource", 404, ctx); -} +} \ No newline at end of file From ba393fd31b6936829d52ec34fa0b7cd4e961ac07 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:56:15 +0200 Subject: [PATCH 63/67] fix(lab): bind scheduler authority to runtime owner --- src/lab/automation/orchestrator.ts | 93 ++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 25 deletions(-) diff --git a/src/lab/automation/orchestrator.ts b/src/lab/automation/orchestrator.ts index 734c47ece4..d9ca492b5a 100644 --- a/src/lab/automation/orchestrator.ts +++ b/src/lab/automation/orchestrator.ts @@ -1,13 +1,13 @@ import { readConfigDiagnostics } from "../../config"; +import { registerCurrentServerResourceCleanup } from "../../lib/server-resource-ownership"; import { queryLabStatus } from "../query"; import { rebuildLabProjection } from "../projection/rebuild"; import { planLabAutomationRuns } from "./planner"; import { - loadLabAutomationPolicy, - loadLabAutomationRoutes, loadLabAutomationState, mutateLabAutomationState, } from "./persistence"; +import { loadLabAutomationConfig } from "./config-persistence"; import { rollBudgetWindow, runBudgetRemaining, @@ -41,10 +41,20 @@ import { LabAutomationError } from "./types"; import { LAB_AUTOMATION_HARD_MAX } from "./constants"; import { cancelQueuedRun } from "./queue"; -const schedulerTimers = new Map>(); +type SchedulerOwner = { + timer: ReturnType; + ownerToken: symbol; +}; + +type DispatchOwner = { + token: symbol; + deps: AutomationDispatchDeps; +}; + +const schedulerTimers = new Map(); const ticksInProgress = new Set(); let shutdownRequested = false; -const dispatchDepsByConfigDir = new Map(); +const dispatchDepsByConfigDir = new Map(); const inFlightControllers = new Map(); const cancellingRunIds = new Set(); @@ -53,20 +63,48 @@ function configKey(configDir?: string): string { } function dispatchDepsFor(configDir?: string): AutomationDispatchDeps { - return dispatchDepsByConfigDir.get(configKey(configDir)) ?? {}; + return dispatchDepsByConfigDir.get(configKey(configDir))?.deps ?? {}; } function cancellationCooldownUntil(policy: LabAutomationPolicyV1, now: number): number { return now + Math.max(policy.failureCooldownMs, LAB_AUTOMATION_HARD_MAX.schedulerTickMs); } -export function setLabAutomationDispatchDeps(deps: AutomationDispatchDeps): void { +/** + * Register process-local dispatch authority for one runtime owner. + * + * The returned receipt releases only the authority that this call installed. If a same-root + * successor has already replaced it, releasing the predecessor is a no-op for the successor. + */ +export function setLabAutomationDispatchDeps(deps: AutomationDispatchDeps): () => void { const key = configKey(deps.configDir); if (Object.keys(deps).length === 0) { dispatchDepsByConfigDir.delete(key); - return; + return () => {}; } - dispatchDepsByConfigDir.set(key, deps); + + const token = Symbol("lab-automation-runtime-owner"); + dispatchDepsByConfigDir.set(key, { token, deps }); + const existingScheduler = schedulerTimers.get(key); + if (existingScheduler) existingScheduler.ownerToken = token; + + let released = false; + let detachServerCleanup = () => {}; + const release = () => { + if (released) return; + released = true; + detachServerCleanup(); + const current = dispatchDepsByConfigDir.get(key); + if (current?.token !== token) return; + dispatchDepsByConfigDir.delete(key); + const scheduler = schedulerTimers.get(key); + if (scheduler?.ownerToken === token) { + clearInterval(scheduler.timer); + schedulerTimers.delete(key); + } + }; + detachServerCleanup = registerCurrentServerResourceCleanup(release); + return release; } export function isLabAutomationSchedulerRunning(configDir?: string): boolean { @@ -81,8 +119,7 @@ export function requestLabAutomationShutdown(): void { } export function buildLabAutomationStatus(configDir?: string): LabAutomationStatusV1 { - const policy = loadLabAutomationPolicy(configDir); - const routes = loadLabAutomationRoutes(configDir); + const { policy, routes } = loadLabAutomationConfig(configDir); const state = loadLabAutomationState(configDir); const now = Date.now(); const hourAgo = now - LAB_AUTOMATION_HARD_MAX.budgetWindowMs; @@ -144,8 +181,7 @@ function reconcileQueuedState( /** Apply current policy/route enrollment to already queued scheduled work without executing it. */ export function reconcileLabAutomationQueue(configDir?: string): void { - const policy = loadLabAutomationPolicy(configDir); - const routes = loadLabAutomationRoutes(configDir); + const { policy, routes } = loadLabAutomationConfig(configDir); const now = Date.now(); mutateLabAutomationState(configDir, (state) => ({ state: trimTerminalRuns(reconcileQueuedState(state, policy, routes, now), now), @@ -276,8 +312,9 @@ async function runDispatchBatch( options: { manualRunId?: string; abortSignal?: AbortSignal } = {}, ): Promise { if (shutdownRequested) return; - const initialPolicy = loadLabAutomationPolicy(configDir); - const initialRoutes = loadLabAutomationRoutes(configDir); + const initialConfig = loadLabAutomationConfig(configDir); + const initialPolicy = initialConfig.policy; + const initialRoutes = initialConfig.routes; if (initialPolicy.enabled && options.manualRunId === undefined && !ensureAutomationProjection(configDir)) { reconcileLabAutomationQueue(configDir); return; @@ -300,8 +337,7 @@ async function runDispatchBatch( const maxDispatches = options.manualRunId ? 1 : Math.max(1, initialPolicy.maxConcurrentRuns); for (let dispatched = 0; dispatched < maxDispatches; dispatched += 1) { if (shutdownRequested) break; - const policy = loadLabAutomationPolicy(configDir); - const routes = loadLabAutomationRoutes(configDir); + const { policy, routes } = loadLabAutomationConfig(configDir); const run = claimNextRun( policy, routes, @@ -357,10 +393,14 @@ export async function runLabAutomationTick(configDir?: string): Promise { export function startLabAutomationScheduler(configDir?: string): void { const key = configKey(configDir); - if (schedulerTimers.has(key)) return; + const currentOwner = dispatchDepsByConfigDir.get(key)?.token; + const existing = schedulerTimers.get(key); + if (existing) { + if (currentOwner) existing.ownerToken = currentOwner; + return; + } shutdownRequested = false; - const policy = loadLabAutomationPolicy(configDir); - const routes = loadLabAutomationRoutes(configDir); + const { policy, routes } = loadLabAutomationConfig(configDir); const now = Date.now(); mutateLabAutomationState(configDir, (state) => { let next = recoverLabAutomationState(policy, state, now); @@ -371,19 +411,22 @@ export function startLabAutomationScheduler(configDir?: string): void { void runLabAutomationTick(configDir); }, LAB_AUTOMATION_HARD_MAX.schedulerTickMs); timer.unref?.(); - schedulerTimers.set(key, timer); + schedulerTimers.set(key, { + timer, + ownerToken: currentOwner ?? Symbol("lab-automation-manual-scheduler"), + }); } /** Stop periodic scheduling only. Process shutdown is a separate explicit signal. */ export function stopLabAutomationScheduler(configDir?: string): void { if (configDir !== undefined) { const key = configKey(configDir); - const timer = schedulerTimers.get(key); - if (timer) clearInterval(timer); + const scheduler = schedulerTimers.get(key); + if (scheduler) clearInterval(scheduler.timer); schedulerTimers.delete(key); return; } - for (const timer of schedulerTimers.values()) clearInterval(timer); + for (const scheduler of schedulerTimers.values()) clearInterval(scheduler.timer); schedulerTimers.clear(); } @@ -423,7 +466,7 @@ export function cancelLabAutomationRun(runId: string, configDir?: string): boole controller.abort(new Error("cancelled")); return true; } - const policy = loadLabAutomationPolicy(configDir); + const policy = loadLabAutomationConfig(configDir).policy; const now = Date.now(); return mutateLabAutomationState(configDir, (state) => { const run = state.runs.find((row) => row.runId === runId); @@ -434,4 +477,4 @@ export function cancelLabAutomationRun(runId: string, configDir?: string): boole } return { state: next, value: true }; }); -} +} \ No newline at end of file From 8a37c28bab815e7e5f34a02d8011c51a955ea51b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:56:57 +0200 Subject: [PATCH 64/67] test(lab): exercise server-owned runtime cleanup --- ...ab-automation-ingwannu-regressions.test.ts | 58 +++++++++---------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/tests/lab-automation-ingwannu-regressions.test.ts b/tests/lab-automation-ingwannu-regressions.test.ts index 6d123721df..b6dc37ca9a 100644 --- a/tests/lab-automation-ingwannu-regressions.test.ts +++ b/tests/lab-automation-ingwannu-regressions.test.ts @@ -5,6 +5,11 @@ import { tmpdir } from "node:os"; import type { OcxConfig } from "../src/types"; import { defaultLabAutomationPolicyV1 } from "../src/lab/automation/policy"; import * as persistence from "../src/lab/automation/persistence"; +import { + loadLabAutomationConfig, + saveLabAutomationConfig, + setLabAutomationConfigCommitFaultForTests, +} from "../src/lab/automation/config-persistence"; import { isLabAutomationSchedulerRunning, resetLabAutomationSchedulerStateForTests, @@ -12,6 +17,10 @@ import { setLabAutomationDispatchDeps, startLabAutomationScheduler, } from "../src/lab/automation/orchestrator"; +import { + acquireServerResourceOwner, + resetServerResourceOwnershipForTests, +} from "../src/lib/server-resource-ownership"; const HOMES: string[] = []; @@ -27,62 +36,52 @@ function emptyConfig(): OcxConfig { } afterEach(() => { + setLabAutomationConfigCommitFaultForTests(null); + resetServerResourceOwnershipForTests(); resetLabAutomationSchedulerStateForTests(); for (const dir of HOMES.splice(0)) rmSync(dir, { recursive: true, force: true }); }); describe("CL-08 Ingwannu regressions", () => { - test("same-root successor survives predecessor runtime release", async () => { + test("same-root successor survives predecessor server-owner release", async () => { const home = tempHome(); let firstLoads = 0; let secondLoads = 0; - const releaseFirst = setLabAutomationDispatchDeps({ + const firstOwner = acquireServerResourceOwner(); + setLabAutomationDispatchDeps({ configDir: home, loadConfig: () => { firstLoads += 1; return emptyConfig(); }, - }) as unknown; - expect(typeof releaseFirst).toBe("function"); - if (typeof releaseFirst !== "function") return; - + }); startLabAutomationScheduler(home); expect(isLabAutomationSchedulerRunning(home)).toBe(true); - const releaseSecond = setLabAutomationDispatchDeps({ + const secondOwner = acquireServerResourceOwner(); + setLabAutomationDispatchDeps({ configDir: home, loadConfig: () => { secondLoads += 1; return emptyConfig(); }, - }) as unknown; - expect(typeof releaseSecond).toBe("function"); - if (typeof releaseSecond !== "function") return; - + }); startLabAutomationScheduler(home); - releaseFirst(); + + firstOwner.release(); expect(isLabAutomationSchedulerRunning(home)).toBe(true); await runLabAutomationTick(home); expect(firstLoads).toBe(0); expect(secondLoads).toBeGreaterThan(0); - releaseSecond(); + secondOwner.release(); expect(isLabAutomationSchedulerRunning(home)).toBe(false); }); test("failed combined automation config commit leaves the prior generation effective", async () => { const home = tempHome(); - const saveConfig = Reflect.get(persistence, "saveLabAutomationConfig") as unknown; - const loadConfig = Reflect.get(persistence, "loadLabAutomationConfig") as unknown; - const setCommitFault = Reflect.get(persistence, "setLabAutomationConfigCommitFaultForTests") as unknown; - - expect(typeof saveConfig).toBe("function"); - expect(typeof loadConfig).toBe("function"); - expect(typeof setCommitFault).toBe("function"); - if (typeof saveConfig !== "function" || typeof loadConfig !== "function" || typeof setCommitFault !== "function") return; - const initialPolicy = { ...defaultLabAutomationPolicyV1(), enabled: true, @@ -93,7 +92,7 @@ describe("CL-08 Ingwannu regressions", () => { }, }; const initialRoutes = persistence.defaultLabAutomationRoutesV1(); - saveConfig(initialPolicy, initialRoutes, home); + saveLabAutomationConfig(initialPolicy, initialRoutes, home); const nextPolicy = { ...initialPolicy, @@ -104,14 +103,11 @@ describe("CL-08 Ingwannu regressions", () => { routes: [{ providerName: "provider-new", modelId: "model-new" }], }; - setCommitFault("before_publish"); - expect(() => saveConfig(nextPolicy, nextRoutes, home)).toThrow(); - setCommitFault(null); + setLabAutomationConfigCommitFaultForTests("before_publish"); + expect(() => saveLabAutomationConfig(nextPolicy, nextRoutes, home)).toThrow(); + setLabAutomationConfigCommitFaultForTests(null); - const effective = loadConfig(home) as { - policy: typeof initialPolicy; - routes: typeof initialRoutes; - }; + const effective = loadLabAutomationConfig(home); expect(effective.policy).toEqual(initialPolicy); expect(effective.routes).toEqual(initialRoutes); @@ -130,4 +126,4 @@ describe("CL-08 Ingwannu regressions", () => { .filter(({ line }) => /[ \t]+$/.test(line)); expect(offenders).toEqual([]); }); -}); +}); \ No newline at end of file From 23c8f9b5681cbefa66651dca5a9e3a70a95383e8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:59:04 +0200 Subject: [PATCH 65/67] fix(lab): keep CLI policy writes on committed config --- src/cli/lab.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cli/lab.ts b/src/cli/lab.ts index e29c797f1e..fabb819007 100644 --- a/src/cli/lab.ts +++ b/src/cli/lab.ts @@ -52,11 +52,11 @@ import { startLabAutomationScheduler, stopLabAutomationScheduler, } from "../lab/automation/orchestrator"; +import { loadLabAutomationState } from "../lab/automation/persistence"; import { - loadLabAutomationPolicy, - saveLabAutomationPolicy, - loadLabAutomationState, -} from "../lab/automation/persistence"; + loadLabAutomationConfig, + saveLabAutomationPolicyConfig, +} from "../lab/automation/config-persistence"; import { planManualLabRun } from "../lab/automation/planner"; import { listLabAutomationRuns } from "../lab/automation/runs-query"; import { LabAutomationError, type LabAutomationLayer } from "../lab/automation/types"; @@ -360,7 +360,7 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P const enableLive = takeFlag(automationRest, "--live"); rejectArgs(automationRest, USAGE); const selectionExplicit = enableProtocol || enableLive; - const current = loadLabAutomationPolicy(configDir); + const current = loadLabAutomationConfig(configDir).policy; const policy = { ...current, enabled: true, @@ -371,7 +371,7 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P }, taskEffectivenessBackgroundEnabled: false, }; - saveLabAutomationPolicy(policy, configDir); + saveLabAutomationPolicyConfig(policy, configDir); reconcileLabAutomationQueue(configDir); startLabAutomationScheduler(configDir); const status = buildLabAutomationStatus(configDir); @@ -380,8 +380,8 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P } case "disable": { rejectArgs(automationRest, USAGE); - const policy = { ...loadLabAutomationPolicy(configDir), enabled: false }; - saveLabAutomationPolicy(policy, configDir); + const policy = { ...loadLabAutomationConfig(configDir).policy, enabled: false }; + saveLabAutomationPolicyConfig(policy, configDir); reconcileLabAutomationQueue(configDir); stopLabAutomationScheduler(configDir); const status = buildLabAutomationStatus(configDir); @@ -447,4 +447,4 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P }); } -export const LAB_USAGE = USAGE; +export const LAB_USAGE = USAGE; \ No newline at end of file From 1492c291602194e4898c5d17f3aeeb7fb994fca4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:05:56 +0200 Subject: [PATCH 66/67] chore(lab): remove CL-08 plan trailing whitespace --- .../260807_compatibility_lab/008_cl08_automation.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md b/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md index 2c27e41040..9003837d60 100644 --- a/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md +++ b/devlog/_plan/260807_compatibility_lab/008_cl08_automation.md @@ -2,10 +2,10 @@ ## Programme position -**Repository:** `lidge-jun/opencodex` -**Integration target:** `dev` -**Branch:** `feat/cl-08-lab-automation` -**Starting SHA:** `da8ebd3135553c1d4dd85c1f258e998a5de14f28` +**Repository:** `lidge-jun/opencodex` +**Integration target:** `dev` +**Branch:** `feat/cl-08-lab-automation` +**Starting SHA:** `da8ebd3135553c1d4dd85c1f258e998a5de14f28` **CL-07 merge prerequisite:** `02e62fc8c7354c544ef71f8bb3db5ebba42cb600` CL-07 is accepted and closed. @@ -1016,4 +1016,4 @@ Audit against repository state after CL-08 implementation on `feat/cl-08-lab-aut | Task-effectiveness background | DEFERRED | Separate opt-in remains disabled; unattended CL-07 not proven safe. | | UI | OUT OF SCOPE | Per plan §21. | -**Known limitation:** Automation trusted-route contract advertises `live_transport` only (`CL08_TRUSTED_LIVE_HARNESS_FEATURES`). Scenarios requiring additional harness features (inert tools, MCP stubs, etc.) are not planned or dispatched until a broader trusted contract is explicitly defined. +**Known limitation:** Automation trusted-route contract advertises `live_transport` only (`CL08_TRUSTED_LIVE_HARNESS_FEATURES`). Scenarios requiring additional harness features (inert tools, MCP stubs, etc.) are not planned or dispatched until a broader trusted contract is explicitly defined. \ No newline at end of file From bfaad5d01a975e8d48b9437bc0a0537077a04134 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:15:59 +0200 Subject: [PATCH 67/67] test(lab): read committed automation config generation --- tests/lab-automation-coderabbit-regressions.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/lab-automation-coderabbit-regressions.test.ts b/tests/lab-automation-coderabbit-regressions.test.ts index eec5e19028..fdabdd07c7 100644 --- a/tests/lab-automation-coderabbit-regressions.test.ts +++ b/tests/lab-automation-coderabbit-regressions.test.ts @@ -11,6 +11,7 @@ import { saveLabAutomationPolicy, saveLabAutomationState, } from "../src/lab/automation/persistence"; +import { loadLabAutomationConfig } from "../src/lab/automation/config-persistence"; import { LAB_AUTOMATION_HARD_MAX } from "../src/lab/automation/constants"; import { enqueuePlannedRuns } from "../src/lab/automation/queue"; import { runBudgetRemaining } from "../src/lab/automation/budgets"; @@ -271,7 +272,7 @@ describe("CL-08 CodeRabbit regressions", () => { layers: { protocolConformance: false, liveRouteCompatibility: true, taskEffectiveness: false }, }, home); expect(await handleLabCommand(["automation", "enable", "--json"], { configDir: home })).toBe(0); - const policy = loadLabAutomationPolicy(home); + const policy = loadLabAutomationConfig(home).policy; expect(policy.enabled).toBe(true); expect(policy.layers.protocolConformance).toBe(false); expect(policy.layers.liveRouteCompatibility).toBe(true); @@ -336,4 +337,4 @@ describe("CL-08 CodeRabbit regressions", () => { environment: {}, })).rejects.toThrow("OAuth lab probes require HTTPS"); }); -}); +}); \ No newline at end of file