From 7b56c6aa12455ec0ea79bda0644592f36dfb8feb Mon Sep 17 00:00:00 2001 From: "Claude (agent)" <238336761+hummbl-dev@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:18:22 -0400 Subject: [PATCH] feat: add governed sol route evaluation scaffold --- .../gpt-5.6-sol-route-pilot-v0.1.md | 178 +++ .../v0.1/PREREGISTRATION.md | 140 ++ .../v0.1/pilot-manifest.json | 305 +++++ ..._evaluation_manifest.v0.1.arm.invalid.json | 10 + ...aluation_manifest.v0.1.budget.invalid.json | 27 + ...evaluation_manifest.v0.1.hash.invalid.json | 17 + ...on_manifest.v0.1.invalidation.invalid.json | 13 + ...uation_manifest.v0.1.sequence.invalid.json | 18 + ...ation_receipt.v0.2.delegation.invalid.json | 10 + ...uation_receipt.v0.2.deviation.invalid.json | 8 + ...receipt.v0.2.numeric-overflow.invalid.json | 12 + ...aluation_receipt.v0.2.reroute.invalid.json | 8 + ...ceipt.v0.2.route-verification.invalid.json | 9 + ...uation_receipt.v0.2.timestamp.invalid.json | 13 + ...uation_receipt.v0.2.token-sum.invalid.json | 12 + ...eceipt.v0.2.unavailable-value.invalid.json | 12 + .../v0.1/attestation/prompt.md | 15 + .../v0.1/attestation/rubric.json | 21 + .../v0.1/attestation/task.json | 38 + .../v0.1/attestation/validate.py | 80 ++ .../v0.1/attestation/workspace/numbers.json | 9 + .../v0.1/attestation/workspace/slug.py | 4 + .../v0.1/attestation/workspace/status.txt | 1 + .../v0.1/decomposable-greenfield/prompt.md | 17 + .../v0.1/decomposable-greenfield/rubric.json | 37 + .../v0.1/decomposable-greenfield/task.json | 39 + .../v0.1/decomposable-greenfield/validate.py | 381 ++++++ .../decomposable-greenfield/workspace/SPEC.md | 45 + .../workspace/sample-events.jsonl | 3 + .../maintenance-negative-control/prompt.md | 12 + .../maintenance-negative-control/rubric.json | 27 + .../maintenance-negative-control/task.json | 35 + .../maintenance-negative-control/validate.py | 84 ++ .../workspace/README.md | 7 + .../workspace/reference-notes.txt | 3 + .../workspace/service-config.json | 10 + .../v0.1/tightly-coupled-debugging/prompt.md | 13 + .../tightly-coupled-debugging/rubric.json | 32 + .../v0.1/tightly-coupled-debugging/task.json | 34 + .../tightly-coupled-debugging/validate.py | 209 +++ .../workspace/README.md | 21 + .../workspace/scheduler.py | 26 + .../route_evaluation_manifest.v0.1.valid.json | 191 +++ ...ute_evaluation_receipt.v0.2.max.valid.json | 87 ++ ...e_evaluation_receipt.v0.2.ultra.valid.json | 186 +++ ...e_evaluation_receipt.v0.2.xhigh.valid.json | 87 ++ ...route_evaluation_manifest.v0.1.schema.json | 391 ++++++ .../route_evaluation_receipt.v0.2.schema.json | 633 +++++++++ tests/test_route_evaluation_manifest.py | 232 ++++ tests/test_route_evaluation_receipt.py | 400 ++++++ tests/test_route_evaluation_runner.py | 327 +++++ tests/test_route_pilot_packet.py | 395 ++++++ tools/run_route_evaluation.py | 1207 +++++++++++++++++ tools/validate_route_evaluation.py | 1064 +++++++++++++++ 54 files changed, 7195 insertions(+) create mode 100644 docs/evaluations/gpt-5.6-sol-route-pilot-v0.1.md create mode 100644 experiments/gpt-5.6-sol-route-pilot/v0.1/PREREGISTRATION.md create mode 100644 experiments/gpt-5.6-sol-route-pilot/v0.1/pilot-manifest.json create mode 100644 fixtures/invalid/route_evaluation_manifest.v0.1.arm.invalid.json create mode 100644 fixtures/invalid/route_evaluation_manifest.v0.1.budget.invalid.json create mode 100644 fixtures/invalid/route_evaluation_manifest.v0.1.hash.invalid.json create mode 100644 fixtures/invalid/route_evaluation_manifest.v0.1.invalidation.invalid.json create mode 100644 fixtures/invalid/route_evaluation_manifest.v0.1.sequence.invalid.json create mode 100644 fixtures/invalid/route_evaluation_receipt.v0.2.delegation.invalid.json create mode 100644 fixtures/invalid/route_evaluation_receipt.v0.2.deviation.invalid.json create mode 100644 fixtures/invalid/route_evaluation_receipt.v0.2.numeric-overflow.invalid.json create mode 100644 fixtures/invalid/route_evaluation_receipt.v0.2.reroute.invalid.json create mode 100644 fixtures/invalid/route_evaluation_receipt.v0.2.route-verification.invalid.json create mode 100644 fixtures/invalid/route_evaluation_receipt.v0.2.timestamp.invalid.json create mode 100644 fixtures/invalid/route_evaluation_receipt.v0.2.token-sum.invalid.json create mode 100644 fixtures/invalid/route_evaluation_receipt.v0.2.unavailable-value.invalid.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/prompt.md create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/rubric.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/task.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/validate.py create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/numbers.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/slug.py create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/status.txt create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/prompt.md create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/rubric.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/task.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/validate.py create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/workspace/SPEC.md create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/workspace/sample-events.jsonl create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/prompt.md create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/rubric.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/task.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/validate.py create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/README.md create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/reference-notes.txt create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/service-config.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/prompt.md create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/rubric.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/task.json create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/validate.py create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/workspace/README.md create mode 100644 fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/workspace/scheduler.py create mode 100644 fixtures/valid/route_evaluation_manifest.v0.1.valid.json create mode 100644 fixtures/valid/route_evaluation_receipt.v0.2.max.valid.json create mode 100644 fixtures/valid/route_evaluation_receipt.v0.2.ultra.valid.json create mode 100644 fixtures/valid/route_evaluation_receipt.v0.2.xhigh.valid.json create mode 100644 schemas/route_evaluation_manifest.v0.1.schema.json create mode 100644 schemas/route_evaluation_receipt.v0.2.schema.json create mode 100644 tests/test_route_evaluation_manifest.py create mode 100644 tests/test_route_evaluation_receipt.py create mode 100644 tests/test_route_evaluation_runner.py create mode 100644 tests/test_route_pilot_packet.py create mode 100644 tools/run_route_evaluation.py create mode 100644 tools/validate_route_evaluation.py diff --git a/docs/evaluations/gpt-5.6-sol-route-pilot-v0.1.md b/docs/evaluations/gpt-5.6-sol-route-pilot-v0.1.md new file mode 100644 index 0000000..90a99ae --- /dev/null +++ b/docs/evaluations/gpt-5.6-sol-route-pilot-v0.1.md @@ -0,0 +1,178 @@ +# GPT-5.6 Sol xhigh–Max–Ultra exploratory pilot v0.1 + +## Status + +Preregistered design. This document fixes the hypothesis, treatment, tasks, +execution order, resource envelope, invalidation rules, and decision thresholds +before any attestation or scored inference run. It is not evidence that any route +is superior. + +All validation is local. GitHub Actions must not be invoked while organizational +Actions minutes remain exhausted. + +## Frozen environment + +- Source repository: `hummbl-dev/model-routing-as-code` +- Source commit: `4e9580f816d0d7982e14149e82c40fe7b67670f2` +- Codex CLI: `0.144.1` +- Base model: `gpt-5.6-sol` +- Refreshed model-catalog SHA-256: + `3ade3703c3a258c2a7780c3e8314e52489baaa2c8634df2de795029f9364f3eb` +- GPT-5.6 Sol catalog-record SHA-256: + `ad2e4fc51d9e8c355ec5a3c94020525f01e3afc3476a6578f304bffad3363c04` +- Catalog configuration hash: `3000` +- Catalog multi-agent version: `v2` +- Supported route values observed in the catalog: `xhigh`, `max`, and `ultra` + +Catalog presence proves configuration availability, not successful execution. +The three-arm attestation gate below must pass before scored work. + +## Falsifiable hypothesis + +For a consequential decomposable task, Ultra will either discover a validated +critical defect or satisfy a critical rubric criterion missed by both xhigh and +Max, or reduce time to a no-worse validated deliverable by at least 20 percent. +It will not produce a stable material advantage on the maintenance negative +control, and Max will match or exceed it on the tightly coupled task unless +parallel discovery changes the validated outcome. + +Failure to observe these task-shape differences rejects the proposed routing +doctrine for this pilot. One exploratory pass cannot canonize a policy even when +the directional hypothesis is supported. + +## Experimental design + +| Element | Preregistered value | +| --- | --- | +| Independent variable | `model_reasoning_effort`: xhigh, max, or ultra; Ultra alone has multi-agent enabled | +| Primary outcomes | Frozen-rubric task success, critical failures, evidence completeness, and monotonic wall-clock time to validated output | +| Secondary outcomes | Repair cycles, unique defects, duplicate work, synthesis loss, interventions, runtime token counters when qualified, and cost only if authoritative telemetry exists | +| Control posture | xhigh is the maintenance baseline; Max is the tightly coupled counterfactual | +| Exploratory sample | Three tasks × three routes × one repetition = nine scored cells | +| Attestation sample | One unscored fixture × three routes = three cells | +| Resource regime | Route-native capacity within the common operational ceilings below | +| Policy effect | None; the pilot may only authorize, reject, or redesign a repeated confirmatory block | + +Each cell starts from an identical copied fixture under an isolated temporary +workspace. Raw JSONL, final workspaces, command output, and runtime metadata stay +outside Git. Published receipts contain only reviewed values and hashes. + +## Frozen tasks and order + +The task fixture packet contains one unscored attestation task and three scored +task shapes: + +1. `maintenance_negative_control`: a bounded deterministic maintenance defect + that should not need escalation beyond xhigh. +2. `tightly_coupled_debugging`: a correctness problem whose constraints interact + across the implementation. +3. `decomposable_greenfield`: a governed multi-artifact build with independent + evidence, implementation, validation, and synthesis boundaries. + +Scored execution order is a fixed Latin square: + +| Ordinal | Task | Route | +| ---: | --- | --- | +| 1 | maintenance negative control | xhigh | +| 2 | maintenance negative control | max | +| 3 | maintenance negative control | ultra | +| 4 | tightly coupled debugging | max | +| 5 | tightly coupled debugging | ultra | +| 6 | tightly coupled debugging | xhigh | +| 7 | decomposable greenfield | ultra | +| 8 | decomposable greenfield | xhigh | +| 9 | decomposable greenfield | max | + +No route may inspect another route's workspace, final response, receipt, or +validation output before all scored cells finish. + +## Hard resource envelope + +- Maximum inference invocations: 12 total—three attestation and nine scored. +- Attempts per cell: one. A retry requires invalidating the affected block and a + new preregistration revision. +- Maximum elapsed time per invocation: 600 seconds. +- Maximum aggregate harness elapsed time: 7,200 seconds. +- Network, sandbox, approvals, tools, prompt bytes, validation commands, and + source fixtures are identical across routes. Only the registered treatment + changes. +- Any usage-limit or rate-limit response stops the block. It is a confound, not a + model failure. + +The local subscription CLI exposes no enforceable USD or aggregate-token ceiling +and no authoritative cost field. Those budget values are explicitly unavailable. +The invocation and elapsed-time ceilings are enforceable operational limits; they +must not be relabeled as equal-dollar or equal-token budgets. + +## Attestation gate + +Before scored execution, every route must complete the same unscored fixture and +produce a valid private run bundle. + +Required evidence for each arm: + +- the requested model and effort; +- a runtime-observed effective model and effort from structured events or the + read-only Codex state record; +- no reroute; +- matching CLI and model-catalog identity; +- deterministic workspace validation; +- monotonic wall-clock duration; +- no usage, rate, timeout, sandbox, or protected-file confound. + +xhigh and Max must have multi-agent disabled and no child thread. Ultra must have +multi-agent enabled and expose at least one actual child execution or equivalent +structured delegation event on the legitimately decomposable attestation task. +Configured agent count alone is insufficient. + +The runtime `tokens_used` counter is retained with its source and semantics marked +unqualified until it reconciles with structured token events. It is not converted +to billed tokens or cost. If any arm fails attestation, no scored cell runs. + +## Validation and adjudication + +Automated task validators execute from frozen argv arrays without a shell. Their +results measure only the behaviors they cover. After all cells complete: + +1. Copy sanitized deliverables to route-blinded labels derived before review. +2. Two reviewers independently apply the frozen rubric without access to the + route-label mapping. +3. Resolve disagreements through a recorded consensus rule; preserve both initial + scores. +4. Unblind only after automated and human scores are immutable. +5. Record unique useful findings, duplicated investigation, rejected findings, + synthesis omissions, and human interventions without estimating unavailable + quantities. + +## Pilot decision rules + +The pilot is invalid if route attestation, task hashes, validation commands, +catalog identity, effective model/effort, common environment, or the one-attempt +rule drifts. + +Ultra has directional support on the decomposable task only if it: + +- uniquely changes a critical rubric outcome missed by both single-agent routes; + or +- produces a no-worse validated score at least 20 percent faster than the faster + valid single-agent route; + +and introduces no additional critical failure or material synthesis loss. + +A tie favors the cheaper route, but no economic winner can be declared when cost +or qualified token telemetry is unavailable. Max has directional support for the +tightly coupled class if it meets or exceeds the other routes without material +time, safety, or evidence disadvantage. xhigh remains the maintenance default +when escalation does not change the validated result. + +The pilot advances to confirmation only when all nine cells are valid, at least +one task meaningfully discriminates routes, and the observed task-shape pattern is +consistent with the hypothesis. Otherwise the decision is `KEEP_CURRENT`, +`REDESIGN`, or `INCONCLUSIVE`. + +## Confirmatory boundary + +Confirmation is a separate authorized block: at least two decomposable tasks, two +tightly coupled tasks, one negative control, three repetitions per route/task, +counterbalanced order, and a new spending authorization. This pilot cannot change +the route policy by itself. diff --git a/experiments/gpt-5.6-sol-route-pilot/v0.1/PREREGISTRATION.md b/experiments/gpt-5.6-sol-route-pilot/v0.1/PREREGISTRATION.md new file mode 100644 index 0000000..293a811 --- /dev/null +++ b/experiments/gpt-5.6-sol-route-pilot/v0.1/PREREGISTRATION.md @@ -0,0 +1,140 @@ +# GPT-5.6 Sol xhigh–Max–Ultra route pilot v0.1 + +## Status + +This packet was preregistered on 2026-07-11 before execution. No inference has +been run from this packet. The pilot is exploratory, uses one run per cell, and +cannot authorize a routing-policy change. A later confirmation experiment would +be required before any task-class-specific policy update; no policy update may be +made from this pilot alone. + +## Research question and hypothesis + +For frozen, consequential local tasks, does four-agent orchestration discover a +validated outcome improvement or reduce time to a validated deliverable enough to +justify its coordination and aggregate-use burden relative to the two single-agent +arms? + +The preregistered hypothesis is conditional: parallel orchestration may help the +decomposable greenfield task, should not materially help the maintenance negative +control, and may incur duplication or synthesis burden on tightly coupled debugging. +No arm is assumed superior before observation. + +## Frozen source and runtime + +- Repository: `https://github.com/hummbl-dev/model-routing-as-code.git` +- Merge commit: `4e9580f816d0d7982e14149e82c40fe7b67670f2` +- Source worktree: must be clean; any task, rubric, validator, runner, schema, or + catalog drift invalidates the affected block as specified by the manifest. +- Base model: `gpt-5.6-sol` +- Codex CLI: `/usr/local/bin/codex`, version `0.144.1`, SHA-256 + `c6eb747e4145ecb3bed2647dbd0f8464b190a5ccba964666ef7c98d4681a4a4c` +- Validator interpreter: `/usr/local/bin/python3`, Python `3.13.7` +- Catalog canonical SHA-256: + `3ade3703c3a258c2a7780c3e8314e52489baaa2c8634df2de795029f9364f3eb` +- `gpt-5.6-sol` catalog record SHA-256: + `ad2e4fc51d9e8c355ec5a3c94020525f01e3afc3476a6578f304bffad3363c04` +- Catalog metadata: `comp_hash=3000`, context window `372000`, multi-agent + version `v2`, API-supported settings `xhigh`, `max`, and `ultra`. + +The runner must ignore user configuration and rules, use copied fixture workspaces +under a temporary directory, keep route outputs isolated, prohibit GitHub Actions, +and expose the same offline tool and workspace-write envelope to every arm. + +## Attestation before scored cells + +One identical, route-neutral attestation is run once for each arm before any scored +cell. It contains three small independent checks: a text-state edit, a JSON +calculation, and a Python normalization repair. The prompt permits delegation only +when the configured route provides it. These three invocations are unscored and +exist only to attest effective route metadata, workspace writing, tool execution, +and local validation. + +An attestation failure stops the pilot before scored work. It is not retried or +silently converted into a scored invocation. + +## Scored tasks and frozen order + +The three task fixtures are synthetic and contain no credentials, private data, or +network dependency: + +1. `maintenance-negative-control`: a two-file support-address update with exact + scope preservation. +2. `tightly-coupled-debugging`: repair an event-driven non-preemptive scheduler + whose timing, priority, tie-break, and validation rules interact. +3. `decomposable-greenfield`: build a library, CLI, schema, and documentation for a + deterministic incident digest. + +The Latin-square-style order is frozen: + +| Sequence | Task | Arm | +| ---: | --- | --- | +| 1 | maintenance-negative-control | xhigh | +| 2 | maintenance-negative-control | max | +| 3 | maintenance-negative-control | ultra | +| 4 | tightly-coupled-debugging | max | +| 5 | tightly-coupled-debugging | ultra | +| 6 | tightly-coupled-debugging | xhigh | +| 7 | decomposable-greenfield | ultra | +| 8 | decomposable-greenfield | xhigh | +| 9 | decomposable-greenfield | max | + +There is one attempt and one repetition per cell. A failed, timed-out, rerouted, or +invalid cell remains failed or invalid; it is not retried under this protocol. + +## Resource ceiling and stopping rule + +- Total inference invocations: at most 12. +- Unscored route attestations: exactly 3 if all attestations are reached. +- Scored invocations: at most 9. +- Attempts per scored cell: 1. +- Scored-cell deadline: 600 seconds. +- Whole-pilot deadline: 7,200 seconds, including attestation and validation. +- Resource-budget regime: route-native capacity; aggregate and per-agent use are + recorded separately when exposed. + +Enforceable aggregate and per-agent USD and token ceilings are unavailable because +this Codex CLI surface exposes no enforceable interface for pre-binding any of those +values. No numeric estimate is substituted. The runner must stop immediately on a +usage limit or rate limit, record the remaining cells as not run, and report the +economic comparison as inconclusive. The same stop rule applies if any configured +ceiling unexpectedly binds or the effective route cannot be attested. + +## Validation and adjudication + +Each route receives identical prompt and workspace bytes for a task. Task-local +instructions, validators, rubrics, and frozen inputs are protected; modifying them +invalidates the cell even if a candidate changes tests to produce a passing result. +Only the explicit fixture-relative `workspace/...` editable paths may change. + +Automated local validation establishes the behavioral hard gate. The preregistered +rubric preserves component scores rather than collapsing correctness, scope, +documentation, duplication, or synthesis into an unexplained composite. Route +outputs remain hidden from later arms until all nine cells finish or the pilot +stops. Any human review must be blinded to route identity where practical. + +Primary observations are validated task success, critical correctness failures, +and elapsed time to the first validated deliverable. Secondary observations include +actual cost and token telemetry only if exposed, validation repair cycles, unique +useful findings, duplicate work, synthesis loss, human intervention, and scope or +safety failures. Unavailable measurements remain unavailable. + +## Raw-output and publication boundary + +Raw prompts, event streams, transcripts, diffs, and validation logs are written to +the external root selected through `ROUTE_PILOT_OUTPUT_ROOT`; raw outputs remain +outside the repository and are never published directly. The tracked repository may +receive only sanitized, content-reviewed receipts and hashes. Raw artifacts must be +content-reviewed before any later publication decision. + +## Invalidation and decision boundary + +The manifest lists machine-enforced invalidation conditions, including unverified +effective route, model rerouting, catalog or fixture drift, usage/rate limits, +budget binding, and validation-contract changes. A changed task, prompt, rubric, +validator, success criterion, runtime envelope, or evidence standard requires a new +protocol revision and rerunning the complete comparison block. + +This pilot can identify promising task-class hypotheses. It cannot establish that +one arm is universally faster, cheaper, safer, or more capable, and it cannot +support a policy update without repeated confirmation. diff --git a/experiments/gpt-5.6-sol-route-pilot/v0.1/pilot-manifest.json b/experiments/gpt-5.6-sol-route-pilot/v0.1/pilot-manifest.json new file mode 100644 index 0000000..d20547c --- /dev/null +++ b/experiments/gpt-5.6-sol-route-pilot/v0.1/pilot-manifest.json @@ -0,0 +1,305 @@ +{ + "schema_id": "route_evaluation_manifest.v0.1", + "schema_version": "0.1", + "pilot_id": "gpt-5.6-sol-route-pilot-v0.1", + "status": "preregistered_not_started", + "preregistered_at": "2026-07-11T15:52:29Z", + "protocol_ref": "experiments/gpt-5.6-sol-route-pilot/v0.1/PREREGISTRATION.md", + "base_model_id": "gpt-5.6-sol", + "source_repository": { + "url": "https://github.com/hummbl-dev/model-routing-as-code.git", + "commit_sha": "4e9580f816d0d7982e14149e82c40fe7b67670f2", + "clean_required": true + }, + "harness": { + "runner_path": "tools/run_route_evaluation.py", + "runner_sha256": "629ec7a46d35a078a302a04784310730bb37a5e2337488c45da94eec1e340639", + "validator_path": "tools/validate_route_evaluation.py", + "validator_sha256": "70cafdb981b8dfcf1ad91f56b3a52fc1a00509394fb78ed78aefae2715de669a", + "receipt_schema_path": "schemas/route_evaluation_receipt.v0.2.schema.json", + "receipt_schema_sha256": "1808e3d9387534da11a06646b34491c668b2f768b413e0d1c85c274fe40f1955", + "python_requirement": ">=3.11", + "python_executable_path": "/usr/local/bin/python3", + "python_version": "3.13.7", + "codex_executable_path": "/usr/local/bin/codex", + "codex_executable_sha256": "c6eb747e4145ecb3bed2647dbd0f8464b190a5ccba964666ef7c98d4681a4a4c" + }, + "catalog_attestation": { + "retrieved_at": "2026-07-11T15:04:04Z", + "catalog_sha256": "3ade3703c3a258c2a7780c3e8314e52489baaa2c8634df2de795029f9364f3eb", + "model_record_sha256": "ad2e4fc51d9e8c355ec5a3c94020525f01e3afc3476a6578f304bffad3363c04", + "comp_hash": "3000", + "cli_version": "0.144.1", + "context_window": 372000, + "multi_agent_version": "v2", + "supported_reasoning_settings": [ + "xhigh", + "max", + "ultra" + ] + }, + "frozen_envelope": { + "fixture_tree_hash_algorithm": "sha256(sorted(relative_posix + NUL + file_sha256 + LF))", + "network_policy": "offline_tools_only", + "sandbox_mode": "workspace-write", + "approval_policy": "never", + "user_config_policy": "ignored", + "github_actions_policy": "prohibited", + "source_access_policy": "fixture_only", + "task_change_policy": "invalidate_all_cells", + "route_output_isolation": true, + "common_cli_flags": [ + "--output-last-message", + "--strict-config", + "--ignore-user-config", + "--ignore-rules", + "--skip-git-repo-check", + "--sandbox=workspace-write", + "--config=approval_policy=never", + "--json", + "--color=never" + ] + }, + "attestation": { + "fixture_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation", + "tree_sha256": "50207c279c6e7a1d834eeec34cf2808325228a101bdf745c916f6a2b626e19c9", + "prompt_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/prompt.md", + "prompt_sha256": "1f1247c93aa61b17626862b956219e3873cf571675dff18fcd006f116b3936ee", + "validation_commands": [ + [ + "/usr/local/bin/python3", + "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/validate.py", + "--workspace", + "{workspace}" + ] + ], + "success_criteria": [ + "All three independent checks pass in one invocation.", + "Only files inside the copied workspace are modified.", + "No network access or third-party dependency is used." + ], + "routes": [ + "xhigh", + "max", + "ultra" + ], + "invocations_per_route": 1, + "scored": false, + "required_before_scored_cells": true + }, + "tasks": [ + { + "task_id": "maintenance-negative-control", + "task_class": "maintenance_negative_control", + "fixture_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control", + "tree_sha256": "fb63d972b3d07fcf96024f91f9d131eb033ea70392ece2ce596618e5ad23cbc5", + "prompt_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/prompt.md", + "prompt_sha256": "fa3f0a3ac88097df5a469810ee7c700273a90563136d2409b7d1a436ee6b42e0", + "rubric_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/rubric.json", + "rubric_sha256": "328368762be7911249148156a2b4050eff8317d1ffef8c04c10f36ccb0e1d3ae", + "validation_commands": [ + [ + "/usr/local/bin/python3", + "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/validate.py", + "--workspace", + "{workspace}" + ] + ], + "success_criteria": [ + "The support address is help@example.invalid in configuration and documentation.", + "All non-target configuration fields and reference notes remain byte-for-byte unchanged.", + "No file is added, removed, or changed outside the two allowed write paths." + ] + }, + { + "task_id": "tightly-coupled-debugging", + "task_class": "tightly_coupled", + "fixture_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging", + "tree_sha256": "4c73044ef66c5df79c3ced834357efe5a1438a71a6e7ee45623dbe53aeca8faf", + "prompt_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/prompt.md", + "prompt_sha256": "c5c4ee60a4eb826268f3989704e3a8667f8b68f0b50d3fdb0615750732959a86", + "rubric_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/rubric.json", + "rubric_sha256": "f4f8250331a8052326cf7cac2fd0b93f060571bdc339d1295f12a40e0e2d3bf5", + "validation_commands": [ + [ + "/usr/local/bin/python3", + "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/validate.py", + "--workspace", + "{workspace}" + ] + ], + "success_criteria": [ + "schedule_jobs implements the complete event, priority, tie-break, and worker assignment contract.", + "Invalid jobs and worker counts raise ValueError without mutating the caller's input.", + "README.md and the workspace file set remain unchanged." + ] + }, + { + "task_id": "decomposable-greenfield", + "task_class": "decomposable", + "fixture_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield", + "tree_sha256": "a5b1de2d492752c6864c9e296acbf924898f511cab525740934908c1edefaef9", + "prompt_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/prompt.md", + "prompt_sha256": "d65b461b3b50c85f1aabee83c171b97c57d4f4d926ca8eaf97c637b6a79852b7", + "rubric_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/rubric.json", + "rubric_sha256": "14d0ffb6319fe3a2740ba12946aebf557b1e010dea746d54bd52e2f96fa6108d", + "validation_commands": [ + [ + "/usr/local/bin/python3", + "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/validate.py", + "--workspace", + "{workspace}" + ] + ], + "success_criteria": [ + "The library validates, deduplicates, aggregates, orders, and renders events exactly as specified.", + "The CLI writes deterministic JSON and Markdown artifacts and rejects invalid input.", + "The output schema and usage documentation describe the implemented contract.", + "SPEC.md, sample-events.jsonl, and the workspace boundary remain unchanged." + ] + } + ], + "arms": [ + { + "arm_id": "xhigh", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "xhigh", + "orchestration": "single_agent", + "expected_agent_count": 1 + }, + { + "arm_id": "max", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "max", + "orchestration": "single_agent", + "expected_agent_count": 1 + }, + { + "arm_id": "ultra", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "ultra", + "orchestration": "automatic_delegation", + "expected_agent_count": 4 + } + ], + "execution_order": [ + { + "sequence": 1, + "cell_id": "maintenance-negative-control.xhigh.r1", + "task_id": "maintenance-negative-control", + "arm_id": "xhigh", + "repetition": 1 + }, + { + "sequence": 2, + "cell_id": "maintenance-negative-control.max.r1", + "task_id": "maintenance-negative-control", + "arm_id": "max", + "repetition": 1 + }, + { + "sequence": 3, + "cell_id": "maintenance-negative-control.ultra.r1", + "task_id": "maintenance-negative-control", + "arm_id": "ultra", + "repetition": 1 + }, + { + "sequence": 4, + "cell_id": "tightly-coupled-debugging.max.r1", + "task_id": "tightly-coupled-debugging", + "arm_id": "max", + "repetition": 1 + }, + { + "sequence": 5, + "cell_id": "tightly-coupled-debugging.ultra.r1", + "task_id": "tightly-coupled-debugging", + "arm_id": "ultra", + "repetition": 1 + }, + { + "sequence": 6, + "cell_id": "tightly-coupled-debugging.xhigh.r1", + "task_id": "tightly-coupled-debugging", + "arm_id": "xhigh", + "repetition": 1 + }, + { + "sequence": 7, + "cell_id": "decomposable-greenfield.ultra.r1", + "task_id": "decomposable-greenfield", + "arm_id": "ultra", + "repetition": 1 + }, + { + "sequence": 8, + "cell_id": "decomposable-greenfield.xhigh.r1", + "task_id": "decomposable-greenfield", + "arm_id": "xhigh", + "repetition": 1 + }, + { + "sequence": 9, + "cell_id": "decomposable-greenfield.max.r1", + "task_id": "decomposable-greenfield", + "arm_id": "max", + "repetition": 1 + } + ], + "budgets": { + "resource_budget_regime": "route_native_capacity", + "total_inference_invocations_max": 12, + "attestation_invocations": 3, + "scored_invocations": 9, + "attempts_per_cell": 1, + "cell_timeout_seconds": 600, + "pilot_timeout_seconds": 7200, + "usd_ceiling": { + "availability": "unavailable", + "reason": "Codex CLI 0.144.1 exposes no enforceable interface for pre-binding a USD ceiling." + }, + "token_ceiling": { + "availability": "unavailable", + "reason": "Codex CLI 0.144.1 exposes no enforceable interface for pre-binding a token ceiling." + }, + "per_agent_usd_ceiling": { + "availability": "unavailable", + "reason": "Codex CLI 0.144.1 exposes no enforceable interface for pre-binding a per-agent USD ceiling." + }, + "per_agent_token_ceiling": { + "availability": "unavailable", + "reason": "Codex CLI 0.144.1 exposes no enforceable interface for pre-binding a per-agent token ceiling." + }, + "stop_conditions": [ + "usage_limit", + "rate_limit", + "invocation_ceiling", + "cell_timeout", + "pilot_timeout", + "attestation_failure" + ] + }, + "raw_output_policy": { + "storage_root_template": "${ROUTE_PILOT_OUTPUT_ROOT}/gpt-5.6-sol-route-pilot-v0.1/{run_id}", + "tracked_in_git": false, + "publish_raw_outputs": false, + "content_review_required": true, + "published_receipts_sanitized": true, + "hash_algorithm": "sha256" + }, + "invalidation_conditions": [ + "effective-route-unverified", + "model-reroute", + "catalog-drift", + "fixture-drift", + "usage-limit", + "rate-limit", + "budget-bound", + "validation-contract-change", + "source-worktree-dirty", + "attestation-failure", + "route-output-leakage", + "timeout" + ] +} diff --git a/fixtures/invalid/route_evaluation_manifest.v0.1.arm.invalid.json b/fixtures/invalid/route_evaluation_manifest.v0.1.arm.invalid.json new file mode 100644 index 0000000..c395585 --- /dev/null +++ b/fixtures/invalid/route_evaluation_manifest.v0.1.arm.invalid.json @@ -0,0 +1,10 @@ +{ + "schema_id": "route_evaluation_manifest.v0.1", + "schema_version": "0.1", + "base_model_id": "gpt-5.6-sol", + "arms": [ + {"arm_id": "xhigh", "model_id": "gpt-5.6-sol", "reasoning_setting": "xhigh", "orchestration": "single_agent", "expected_agent_count": 1}, + {"arm_id": "max", "model_id": "gpt-5.6-sol", "reasoning_setting": "max", "orchestration": "single_agent", "expected_agent_count": 1}, + {"arm_id": "ultra", "model_id": "gpt-5.6-sol", "reasoning_setting": "max", "orchestration": "single_agent", "expected_agent_count": 1} + ] +} diff --git a/fixtures/invalid/route_evaluation_manifest.v0.1.budget.invalid.json b/fixtures/invalid/route_evaluation_manifest.v0.1.budget.invalid.json new file mode 100644 index 0000000..4cc9c38 --- /dev/null +++ b/fixtures/invalid/route_evaluation_manifest.v0.1.budget.invalid.json @@ -0,0 +1,27 @@ +{ + "schema_id": "route_evaluation_manifest.v0.1", + "schema_version": "0.1", + "attestation": { + "routes": ["xhigh", "max", "ultra"], + "invocations_per_route": 1 + }, + "execution_order": [ + {"sequence": 1}, {"sequence": 2}, {"sequence": 3}, + {"sequence": 4}, {"sequence": 5}, {"sequence": 6}, + {"sequence": 7}, {"sequence": 8}, {"sequence": 9} + ], + "budgets": { + "total_inference_invocations_max": 11, + "attestation_invocations": 3, + "scored_invocations": 9, + "attempts_per_cell": 1, + "cell_timeout_seconds": 600, + "pilot_timeout_seconds": 7200, + "resource_budget_regime": "route_native_capacity", + "usd_ceiling": {"availability": "unavailable", "reason": "Not exposed."}, + "token_ceiling": {"availability": "unavailable", "reason": "Not exposed."}, + "per_agent_usd_ceiling": {"availability": "unavailable", "reason": "Not exposed."}, + "per_agent_token_ceiling": {"availability": "unavailable", "reason": "Not exposed."}, + "stop_conditions": ["Stop at the ceiling."] + } +} diff --git a/fixtures/invalid/route_evaluation_manifest.v0.1.hash.invalid.json b/fixtures/invalid/route_evaluation_manifest.v0.1.hash.invalid.json new file mode 100644 index 0000000..1033ff0 --- /dev/null +++ b/fixtures/invalid/route_evaluation_manifest.v0.1.hash.invalid.json @@ -0,0 +1,17 @@ +{ + "schema_id": "route_evaluation_manifest.v0.1", + "schema_version": "0.1", + "harness": { + "runner_path": "tools/run_route_evaluation.py", + "runner_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "validator_path": "tools/validate_route_evaluation.py", + "validator_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "receipt_schema_path": "schemas/route_evaluation_receipt.v0.2.schema.json", + "receipt_schema_sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "python_requirement": ">=3.11", + "python_executable_path": "/usr/local/bin/python3", + "python_version": "3.13.7", + "codex_executable_path": "/usr/local/bin/codex", + "codex_executable_sha256": "c6eb747e4145ecb3bed2647dbd0f8464b190a5ccba964666ef7c98d4681a4a4c" + } +} diff --git a/fixtures/invalid/route_evaluation_manifest.v0.1.invalidation.invalid.json b/fixtures/invalid/route_evaluation_manifest.v0.1.invalidation.invalid.json new file mode 100644 index 0000000..f3b7825 --- /dev/null +++ b/fixtures/invalid/route_evaluation_manifest.v0.1.invalidation.invalid.json @@ -0,0 +1,13 @@ +{ + "schema_id": "route_evaluation_manifest.v0.1", + "schema_version": "0.1", + "invalidation_conditions": [ + "effective-route-unverified", + "catalog-drift", + "fixture-drift", + "usage-limit", + "rate-limit", + "budget-bound", + "validation-contract-change" + ] +} diff --git a/fixtures/invalid/route_evaluation_manifest.v0.1.sequence.invalid.json b/fixtures/invalid/route_evaluation_manifest.v0.1.sequence.invalid.json new file mode 100644 index 0000000..e9eed36 --- /dev/null +++ b/fixtures/invalid/route_evaluation_manifest.v0.1.sequence.invalid.json @@ -0,0 +1,18 @@ +{ + "schema_id": "route_evaluation_manifest.v0.1", + "schema_version": "0.1", + "tasks": [ + {"task_id": "one"}, + {"task_id": "two"}, + {"task_id": "three"} + ], + "arms": [ + {"arm_id": "xhigh"}, + {"arm_id": "max"}, + {"arm_id": "ultra"} + ], + "execution_order": [ + {"sequence": 1, "cell_id": "one.xhigh.r1", "task_id": "one", "arm_id": "xhigh", "repetition": 1}, + {"sequence": 1, "cell_id": "two.max.r1", "task_id": "two", "arm_id": "max", "repetition": 1} + ] +} diff --git a/fixtures/invalid/route_evaluation_receipt.v0.2.delegation.invalid.json b/fixtures/invalid/route_evaluation_receipt.v0.2.delegation.invalid.json new file mode 100644 index 0000000..9188e27 --- /dev/null +++ b/fixtures/invalid/route_evaluation_receipt.v0.2.delegation.invalid.json @@ -0,0 +1,10 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "cell": {"route": "ultra", "arm_id": "ultra"}, + "run": {"validity": "valid", "invalidation_reasons": []}, + "requested_configuration": {"reasoning_setting": "ultra", "multi_agent_enabled": true, "orchestration": "automatic_delegation", "expected_agent_count": 4}, + "verified_configuration": {"availability": "available", "model_id": "gpt-5.6-sol", "reasoning_setting": "ultra"}, + "delegation": {"availability": "available", "enabled": false, "observed_agent_count": 1, "max_concurrent_agents": 1, "measurement_source": "Persisted thread metadata."}, + "execution_units": [] +} diff --git a/fixtures/invalid/route_evaluation_receipt.v0.2.deviation.invalid.json b/fixtures/invalid/route_evaluation_receipt.v0.2.deviation.invalid.json new file mode 100644 index 0000000..0c4541d --- /dev/null +++ b/fixtures/invalid/route_evaluation_receipt.v0.2.deviation.invalid.json @@ -0,0 +1,8 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "run": {"validity": "valid", "invalidation_reasons": []}, + "deviations": [ + {"deviation_id": "fixture-drift", "severity": "material", "description": "Fixture changed.", "invalidates_run": true} + ] +} diff --git a/fixtures/invalid/route_evaluation_receipt.v0.2.numeric-overflow.invalid.json b/fixtures/invalid/route_evaluation_receipt.v0.2.numeric-overflow.invalid.json new file mode 100644 index 0000000..e145bc2 --- /dev/null +++ b/fixtures/invalid/route_evaluation_receipt.v0.2.numeric-overflow.invalid.json @@ -0,0 +1,12 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "telemetry": { + "aggregate_cost": {"availability": "available", "value": 1e400, "unit": "USD", "measurement_kind": "observed"}, + "input_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."}, + "token_aggregation_method": "unavailable" + } +} diff --git a/fixtures/invalid/route_evaluation_receipt.v0.2.reroute.invalid.json b/fixtures/invalid/route_evaluation_receipt.v0.2.reroute.invalid.json new file mode 100644 index 0000000..703982b --- /dev/null +++ b/fixtures/invalid/route_evaluation_receipt.v0.2.reroute.invalid.json @@ -0,0 +1,8 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "run": {"validity": "valid", "invalidation_reasons": []}, + "reroutes": [ + {"occurred_at": "2026-07-11T15:00:03Z", "from_model_id": "gpt-5.6-sol", "to_model_id": "gpt-5.6-terra", "reason": "Capacity reroute."} + ] +} diff --git a/fixtures/invalid/route_evaluation_receipt.v0.2.route-verification.invalid.json b/fixtures/invalid/route_evaluation_receipt.v0.2.route-verification.invalid.json new file mode 100644 index 0000000..ed9371e --- /dev/null +++ b/fixtures/invalid/route_evaluation_receipt.v0.2.route-verification.invalid.json @@ -0,0 +1,9 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "cell": {"route": "max", "arm_id": "max"}, + "run": {"validity": "valid", "invalidation_reasons": []}, + "requested_configuration": {"model_id": "gpt-5.6-sol", "reasoning_setting": "max"}, + "verified_configuration": {"availability": "available", "model_id": "gpt-5.6-sol", "reasoning_setting": "xhigh"}, + "reroutes": [] +} diff --git a/fixtures/invalid/route_evaluation_receipt.v0.2.timestamp.invalid.json b/fixtures/invalid/route_evaluation_receipt.v0.2.timestamp.invalid.json new file mode 100644 index 0000000..a8ce047 --- /dev/null +++ b/fixtures/invalid/route_evaluation_receipt.v0.2.timestamp.invalid.json @@ -0,0 +1,13 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "recorded_at": "2026-07-11T15:00:10Z", + "run": { + "started_at": "2026-07-11T15:00:05Z", + "completed_at": {"availability": "available", "value": "2026-07-11T15:00:04Z"}, + "status": "completed", + "validity": "invalid", + "invalidation_reasons": ["timestamp-order"], + "human_intervention_count": 0 + } +} diff --git a/fixtures/invalid/route_evaluation_receipt.v0.2.token-sum.invalid.json b/fixtures/invalid/route_evaluation_receipt.v0.2.token-sum.invalid.json new file mode 100644 index 0000000..94fdd18 --- /dev/null +++ b/fixtures/invalid/route_evaluation_receipt.v0.2.token-sum.invalid.json @@ -0,0 +1,12 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Not exposed."}, + "input_tokens": {"availability": "available", "value": 40, "unit": "tokens", "measurement_kind": "observed"}, + "output_tokens": {"availability": "available", "value": 20, "unit": "tokens", "measurement_kind": "observed"}, + "total_tokens": {"availability": "available", "value": 59, "unit": "tokens", "measurement_kind": "observed"}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."}, + "token_aggregation_method": "runtime_reported_aggregate" + } +} diff --git a/fixtures/invalid/route_evaluation_receipt.v0.2.unavailable-value.invalid.json b/fixtures/invalid/route_evaluation_receipt.v0.2.unavailable-value.invalid.json new file mode 100644 index 0000000..57b6d4e --- /dev/null +++ b/fixtures/invalid/route_evaluation_receipt.v0.2.unavailable-value.invalid.json @@ -0,0 +1,12 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Not exposed.", "value": 1.0}, + "input_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "Not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Not exposed."}, + "token_aggregation_method": "unavailable" + } +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/prompt.md b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/prompt.md new file mode 100644 index 0000000..26b661f --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/prompt.md @@ -0,0 +1,15 @@ +# Route-neutral workspace attestation + +Complete all three independent checks below. Delegation may be used only if the +configured route provides it; do not simulate or claim delegation otherwise. + +1. Change `status.txt` so its complete contents are `READY` followed by one newline. +2. In `numbers.json`, replace the null `sum` with the integer sum of `values` while + preserving the values. +3. Repair `normalize_slug` in `slug.py` so it trims surrounding whitespace, + lowercases text, collapses every run of whitespace to one hyphen, and returns an + empty string for whitespace-only input. + +Modify only files inside the provided workspace. Do not use network access or add +third-party dependencies. Run the supplied local validation command and report its +result concisely. diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/rubric.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/rubric.json new file mode 100644 index 0000000..f063ca0 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/rubric.json @@ -0,0 +1,21 @@ +{ + "schema_version": "route_pilot_rubric.v0.1", + "task_id": "route-attestation", + "scored": false, + "total_points": 0, + "pass_rule": "All three checks and the workspace-boundary check must pass.", + "checks": [ + { + "check_id": "status", + "required": true + }, + { + "check_id": "numbers", + "required": true + }, + { + "check_id": "slug", + "required": true + } + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/task.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/task.json new file mode 100644 index 0000000..d13e88f --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/task.json @@ -0,0 +1,38 @@ +{ + "schema_version": "route_pilot_task.v0.1", + "task_id": "route-attestation", + "task_class": "attestation", + "title": "Route-neutral workspace attestation", + "scored": false, + "deterministic": true, + "network_required": false, + "third_party_dependencies": false, + "path_scope": "fixture_relative", + "editable_paths": [ + "workspace/**" + ], + "protected_paths": [ + "prompt.md", + "rubric.json", + "task.json", + "validate.py" + ], + "independent_checks": [ + "Mark the status file READY.", + "Compute and record the integer sum in numbers.json.", + "Repair normalize_slug for trimmed lowercase hyphenated output." + ], + "validation_commands": [ + [ + "/usr/local/bin/python3", + "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/validate.py", + "--workspace", + "{workspace}" + ] + ], + "success_criteria": [ + "All three independent checks pass in one invocation.", + "Only files inside the copied workspace are modified.", + "No network access or third-party dependency is used." + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/validate.py b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/validate.py new file mode 100644 index 0000000..702b87a --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/validate.py @@ -0,0 +1,80 @@ +"""Validate the route-neutral attestation workspace using only the stdlib.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +EXPECTED_FILES = {"numbers.json", "slug.py", "status.txt"} + + +def _load_json(path: Path) -> Any: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def validate(workspace: Path) -> list[str]: + failures: list[str] = [] + if not workspace.is_dir(): + return ["workspace: directory does not exist"] + + observed = {item.name for item in workspace.iterdir()} + if observed != EXPECTED_FILES: + failures.append("workspace: file set changed") + if any(item.is_symlink() for item in workspace.rglob("*")): + failures.append("workspace: symbolic links are not allowed") + + status_path = workspace / "status.txt" + if ( + not status_path.is_file() + or status_path.read_text(encoding="utf-8") != "READY\n" + ): + failures.append("status: expected exact READY line") + + numbers_path = workspace / "numbers.json" + try: + numbers = _load_json(numbers_path) + except (OSError, ValueError) as error: + failures.append(f"numbers: invalid JSON ({type(error).__name__})") + else: + if numbers != {"sum": 42, "values": [3, 7, 11, 21]}: + failures.append("numbers: values must be preserved and sum must equal 42") + + slug_path = workspace / "slug.py" + namespace: dict[str, Any] = {"__name__": "candidate_slug"} + try: + source = slug_path.read_text(encoding="utf-8") + exec(compile(source, str(slug_path), "exec"), namespace) + normalize_slug = namespace["normalize_slug"] + cases = { + " Pilot Packet ": "pilot-packet", + "MANY\tspaces\nHERE": "many-spaces-here", + " ": "", + } + if any(normalize_slug(value) != expected for value, expected in cases.items()): + failures.append("slug: normalization behavior is incorrect") + except Exception as error: # noqa: BLE001 - candidate code is an evaluation input + failures.append(f"slug: could not execute candidate ({type(error).__name__})") + + return sorted(set(failures)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", required=True, type=Path) + args = parser.parse_args() + failures = validate(args.workspace) + print( + json.dumps( + {"failures": failures, "status": "pass" if not failures else "fail"}, + sort_keys=True, + ) + ) + return 0 if not failures else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/numbers.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/numbers.json new file mode 100644 index 0000000..3f943df --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/numbers.json @@ -0,0 +1,9 @@ +{ + "sum": null, + "values": [ + 3, + 7, + 11, + 21 + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/slug.py b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/slug.py new file mode 100644 index 0000000..cec1bd8 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/slug.py @@ -0,0 +1,4 @@ +def normalize_slug(value: str) -> str: + """Return a normalized slug.""" + + return value.lower().replace(" ", "-") diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/status.txt b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/status.txt new file mode 100644 index 0000000..622e73d --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/workspace/status.txt @@ -0,0 +1 @@ +PENDING diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/prompt.md b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/prompt.md new file mode 100644 index 0000000..1761608 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/prompt.md @@ -0,0 +1,17 @@ +# Decomposable greenfield task + +Build the stdlib-only incident digest described in `SPEC.md`. The work has four +separable artifacts that must agree: + +1. `incident_digest.py` implements `build_digest` and `render_markdown`. +2. `cli.py` provides the specified file-to-file command interface. +3. `output.schema.json` describes every generated JSON report. +4. `README.md` documents usage and invalid-input behavior. + +Treat `SPEC.md` and `sample-events.jsonl` as frozen inputs. Do not alter them. The +implementation must be deterministic and must reject malformed events rather than +silently dropping them. + +Modify only files inside the provided workspace. Do not use network access or add +third-party dependencies. Run the supplied local validation commands and report +their results concisely. diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/rubric.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/rubric.json new file mode 100644 index 0000000..af0d2f0 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/rubric.json @@ -0,0 +1,37 @@ +{ + "schema_version": "route_pilot_rubric.v0.1", + "task_id": "decomposable-greenfield", + "scored": true, + "total_points": 100, + "hard_gates": [ + "Local deterministic validation passes.", + "Frozen inputs and files outside the copied workspace are unchanged." + ], + "criteria": [ + { + "criterion_id": "library_contract", + "description": "Parsing, validation, deduplication, aggregation, sorting, and rendering match SPEC.md.", + "points": 45 + }, + { + "criterion_id": "cli_contract", + "description": "The CLI emits both deterministic artifacts and rejects invalid input.", + "points": 20 + }, + { + "criterion_id": "schema_contract", + "description": "The Draft 2020-12 output schema accurately constrains the report.", + "points": 15 + }, + { + "criterion_id": "documentation", + "description": "README usage and failure behavior are concise and accurate.", + "points": 10 + }, + { + "criterion_id": "scope_and_determinism", + "description": "The solution is stdlib-only, deterministic, and preserves frozen inputs.", + "points": 10 + } + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/task.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/task.json new file mode 100644 index 0000000..ec46879 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/task.json @@ -0,0 +1,39 @@ +{ + "schema_version": "route_pilot_task.v0.1", + "task_id": "decomposable-greenfield", + "task_class": "decomposable", + "title": "Build a deterministic incident digest library and CLI", + "scored": true, + "deterministic": true, + "network_required": false, + "third_party_dependencies": false, + "path_scope": "fixture_relative", + "editable_paths": [ + "workspace/README.md", + "workspace/cli.py", + "workspace/incident_digest.py", + "workspace/output.schema.json" + ], + "protected_paths": [ + "prompt.md", + "rubric.json", + "task.json", + "validate.py", + "workspace/SPEC.md", + "workspace/sample-events.jsonl" + ], + "validation_commands": [ + [ + "/usr/local/bin/python3", + "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/validate.py", + "--workspace", + "{workspace}" + ] + ], + "success_criteria": [ + "The library validates, deduplicates, aggregates, orders, and renders events exactly as specified.", + "The CLI writes deterministic JSON and Markdown artifacts and rejects invalid input.", + "The output schema and usage documentation describe the implemented contract.", + "SPEC.md, sample-events.jsonl, and the workspace boundary remain unchanged." + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/validate.py b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/validate.py new file mode 100644 index 0000000..1505884 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/validate.py @@ -0,0 +1,381 @@ +"""Behavioral validator for the decomposable incident-digest task.""" + +from __future__ import annotations + +import argparse +import ast +from datetime import datetime +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from typing import Any, Callable, Iterable + + +EXPECTED_FILES = { + "README.md", + "SPEC.md", + "cli.py", + "incident_digest.py", + "output.schema.json", + "sample-events.jsonl", +} +FROZEN_SPEC = (Path(__file__).parent / "workspace" / "SPEC.md").read_text( + encoding="utf-8" +) +FROZEN_SAMPLE = (Path(__file__).parent / "workspace" / "sample-events.jsonl").read_text( + encoding="utf-8" +) +EXPECTED_SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": False, + "properties": { + "latest_event_at": { + "oneOf": [ + { + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "type": "string", + }, + {"type": "null"}, + ] + }, + "schema_version": {"const": "incident-digest.v1"}, + "services": { + "items": { + "additionalProperties": False, + "properties": { + "critical": {"minimum": 0, "type": "integer"}, + "info": {"minimum": 0, "type": "integer"}, + "service": {"minLength": 1, "type": "string"}, + "total": {"minimum": 0, "type": "integer"}, + "warning": {"minimum": 0, "type": "integer"}, + }, + "required": ["service", "critical", "warning", "info", "total"], + "type": "object", + }, + "type": "array", + }, + "severity_counts": { + "additionalProperties": False, + "properties": { + "critical": {"minimum": 0, "type": "integer"}, + "info": {"minimum": 0, "type": "integer"}, + "warning": {"minimum": 0, "type": "integer"}, + }, + "required": ["critical", "warning", "info"], + "type": "object", + }, + "total_events": {"minimum": 0, "type": "integer"}, + }, + "required": [ + "schema_version", + "total_events", + "severity_counts", + "services", + "latest_event_at", + ], + "type": "object", +} + + +def _load_json(path: Path) -> Any: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def _validate_imports( + path: Path, *, allowed_local: frozenset[str] = frozenset() +) -> ast.Module: + source = path.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(path)) + for node in ast.walk(tree): + names: list[str] = [] + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + names = [node.module] + for name in names: + root = name.split(".", 1)[0] + if root not in sys.stdlib_module_names and root not in allowed_local: + raise ValueError(f"third-party import is not allowed: {name}") + return tree + + +def _load_library( + path: Path, +) -> tuple[Callable[[Iterable[str]], Any], Callable[[dict[str, Any]], Any]]: + tree = _validate_imports(path) + namespace: dict[str, Any] = {"__name__": "candidate_incident_digest"} + exec(compile(tree, str(path), "exec"), namespace) + build_digest = namespace.get("build_digest") + render_markdown = namespace.get("render_markdown") + if not callable(build_digest) or not callable(render_markdown): + raise ValueError("required library functions are missing") + return build_digest, render_markdown + + +def _reference_digest(lines: Iterable[str]) -> dict[str, Any]: + selected: dict[str, tuple[str, int, dict[str, str]]] = {} + required = {"id", "service", "severity", "timestamp", "summary"} + for index, line in enumerate(lines): + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError("invalid JSON") from error + if not isinstance(event, dict) or set(event) != required: + raise ValueError("invalid event fields") + if not all(isinstance(event[field], str) for field in required): + raise ValueError("event fields must be strings") + normalized = {field: event[field].strip() for field in required} + if ( + not normalized["id"] + or not normalized["service"] + or not normalized["summary"] + ): + raise ValueError("event contains an empty required value") + if normalized["severity"] not in {"critical", "warning", "info"}: + raise ValueError("invalid severity") + timestamp = normalized["timestamp"] + try: + parsed = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ") + except ValueError as error: + raise ValueError("invalid timestamp") from error + if parsed.strftime("%Y-%m-%dT%H:%M:%SZ") != timestamp: + raise ValueError("timestamp is not canonical") + current = selected.get(normalized["id"]) + key = (timestamp, index) + if current is None or key >= (current[0], current[1]): + selected[normalized["id"]] = (timestamp, index, normalized) + + severity_counts = {"critical": 0, "warning": 0, "info": 0} + services: dict[str, dict[str, Any]] = {} + latest: str | None = None + for timestamp, _, event in selected.values(): + severity = event["severity"] + severity_counts[severity] += 1 + service = services.setdefault( + event["service"], + { + "service": event["service"], + "critical": 0, + "warning": 0, + "info": 0, + "total": 0, + }, + ) + service[severity] += 1 + service["total"] += 1 + latest = timestamp if latest is None or timestamp > latest else latest + ordered_services = sorted( + services.values(), + key=lambda item: (-item["critical"], -item["total"], item["service"]), + ) + return { + "schema_version": "incident-digest.v1", + "total_events": len(selected), + "severity_counts": severity_counts, + "services": ordered_services, + "latest_event_at": latest, + } + + +def _reference_markdown(report: dict[str, Any]) -> str: + lines = [ + "# Incident digest", + "", + f"- Total events: {report['total_events']}", + f"- Latest event: {report['latest_event_at'] or 'none'}", + "", + "| Service | Critical | Warning | Info | Total |", + "| --- | ---: | ---: | ---: | ---: |", + ] + lines.extend( + f"| {item['service']} | {item['critical']} | {item['warning']} | {item['info']} | {item['total']} |" + for item in report["services"] + ) + return "\n".join(lines) + "\n" + + +def validate(workspace: Path) -> list[str]: + failures: list[str] = [] + if not workspace.is_dir(): + return ["workspace: directory does not exist"] + if {item.name for item in workspace.iterdir()} != EXPECTED_FILES: + failures.append("workspace: required file set differs") + if any(item.is_symlink() for item in workspace.rglob("*")): + failures.append("workspace: symbolic links are not allowed") + try: + if (workspace / "SPEC.md").read_text(encoding="utf-8") != FROZEN_SPEC: + failures.append("scope: SPEC.md changed") + if (workspace / "sample-events.jsonl").read_text( + encoding="utf-8" + ) != FROZEN_SAMPLE: + failures.append("scope: sample-events.jsonl changed") + build_digest, render_markdown = _load_library(workspace / "incident_digest.py") + _validate_imports( + workspace / "cli.py", allowed_local=frozenset({"incident_digest"}) + ) + except Exception as error: # noqa: BLE001 - candidate code is an evaluation input + failures.append(f"candidate: load failed ({type(error).__name__}: {error})") + return sorted(set(failures)) + + cases = [ + FROZEN_SAMPLE.splitlines(), + [], + [ + '{"id":"b","service":"beta","severity":"critical","timestamp":"2026-01-01T00:00:02Z","summary":"B"}', + '{"id":"a","service":"alpha","severity":"critical","timestamp":"2026-01-01T00:00:01Z","summary":"A"}', + '{"id":"c","service":"alpha","severity":"info","timestamp":"2026-01-01T00:00:03Z","summary":"C"}', + ], + [ + '{"id":"same","service":"old","severity":"warning","timestamp":"2026-01-01T00:00:00Z","summary":"old"}', + '{"id":"same","service":"new","severity":"info","timestamp":"2026-01-01T00:00:00Z","summary":"new"}', + ], + ] + for index, lines in enumerate(cases): + expected = _reference_digest(lines) + try: + observed = build_digest(iter(lines)) + markdown = render_markdown(observed) + except Exception as error: # noqa: BLE001 - candidate code is an evaluation input + failures.append(f"library[{index}]: raised {type(error).__name__}") + continue + if observed != expected: + failures.append(f"library[{index}]: report differs") + if markdown != _reference_markdown(expected): + failures.append(f"library[{index}]: Markdown differs") + + invalid_lines = [ + "not json", + '{"id":"x","service":"api","severity":"fatal","timestamp":"2026-01-01T00:00:00Z","summary":"x"}', + '{"id":"x","service":"api","severity":"info","timestamp":"2026-02-30T00:00:00Z","summary":"x"}', + '{"id":"x","service":"api","severity":"info","timestamp":"2026-01-01T00:00:00Z","summary":"x","extra":"no"}', + ] + for index, line in enumerate(invalid_lines): + try: + build_digest([line]) + except ValueError: + continue + except Exception as error: # noqa: BLE001 - candidate code is an evaluation input + failures.append( + f"invalid[{index}]: raised {type(error).__name__}, not ValueError" + ) + else: + failures.append(f"invalid[{index}]: ValueError was not raised") + + try: + schema = _load_json(workspace / "output.schema.json") + if schema != EXPECTED_SCHEMA: + failures.append( + "schema: output.schema.json differs from the frozen contract" + ) + except (OSError, ValueError) as error: + failures.append(f"schema: invalid JSON ({type(error).__name__})") + + try: + readme = (workspace / "README.md").read_text(encoding="utf-8") + required_phrases = ( + "python3 cli.py", + "--json-output", + "--markdown-output", + "invalid", + ) + if any(phrase not in readme for phrase in required_phrases): + failures.append( + "documentation: required usage or invalid-input behavior is missing" + ) + except OSError as error: + failures.append(f"documentation: README unavailable ({type(error).__name__})") + + with tempfile.TemporaryDirectory() as directory: + temp = Path(directory) + input_path = temp / "events.jsonl" + json_path = temp / "report.json" + markdown_path = temp / "report.md" + input_path.write_text(FROZEN_SAMPLE, encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(workspace / "cli.py"), + str(input_path), + "--json-output", + str(json_path), + "--markdown-output", + str(markdown_path), + ], + cwd=workspace, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + expected = _reference_digest(FROZEN_SAMPLE.splitlines()) + if result.returncode != 0: + failures.append("cli: valid input returned nonzero") + else: + try: + if _load_json(json_path) != expected: + failures.append("cli: JSON output differs") + if ( + json_path.read_text(encoding="utf-8") + != json.dumps(expected, indent=2, sort_keys=True) + "\n" + ): + failures.append("cli: JSON serialization is not canonical") + if markdown_path.read_text(encoding="utf-8") != _reference_markdown( + expected + ): + failures.append("cli: Markdown output differs") + except OSError as error: + failures.append(f"cli: output unavailable ({type(error).__name__})") + + invalid_input = temp / "invalid.jsonl" + invalid_json = temp / "invalid-report.json" + invalid_markdown = temp / "invalid-report.md" + invalid_input.write_text("not json\n", encoding="utf-8") + invalid_result = subprocess.run( + [ + sys.executable, + str(workspace / "cli.py"), + str(invalid_input), + "--json-output", + str(invalid_json), + "--markdown-output", + str(invalid_markdown), + ], + cwd=workspace, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + if invalid_result.returncode == 0: + failures.append("cli: invalid input returned zero") + if not invalid_result.stderr.strip() or "Traceback" in invalid_result.stderr: + failures.append("cli: invalid input lacks a concise stderr error") + if invalid_json.exists() or invalid_markdown.exists(): + failures.append("cli: invalid input wrote output artifacts") + return sorted(set(failures)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", required=True, type=Path) + args = parser.parse_args() + failures = validate(args.workspace) + print( + json.dumps( + {"failures": failures, "status": "pass" if not failures else "fail"}, + sort_keys=True, + ) + ) + return 0 if not failures else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/workspace/SPEC.md b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/workspace/SPEC.md new file mode 100644 index 0000000..990093c --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/workspace/SPEC.md @@ -0,0 +1,45 @@ +# Incident digest contract + +## Library + +`build_digest(lines)` accepts an iterable of JSONL strings. Blank lines are ignored. +Every non-blank line must be a JSON object with exactly these string fields: +`id`, `service`, `severity`, `timestamp`, and `summary`. + +- `id`, `service`, and `summary` must be non-empty after trimming. +- `severity` is exactly `info`, `warning`, or `critical`. +- `timestamp` is an uppercase-`Z` RFC 3339 instant with whole seconds: + `YYYY-MM-DDTHH:MM:SSZ`. Calendar-invalid values are rejected. +- Invalid JSON or an invalid event raises `ValueError`. + +Duplicate identifiers are replaced by the occurrence with the later timestamp. If +timestamps tie, the later input occurrence wins. The selected event values are +trimmed but otherwise preserved. + +Return this report shape: + +- `schema_version`: exact string `incident-digest.v1`; +- `total_events`: count of selected unique events; +- `severity_counts`: object with integer keys in this exact semantic set: + `critical`, `warning`, `info`; +- `services`: one object per service with `service`, `critical`, `warning`, `info`, + and `total`; order by critical count descending, total descending, then service + ascending; +- `latest_event_at`: latest selected timestamp, or null for empty input. + +`render_markdown(report)` returns a deterministic Markdown digest with one trailing +newline. It contains a title, total and latest bullets, and a service table in report +order. Use `none` when the latest timestamp is null. + +## CLI + +`python3 cli.py INPUT --json-output REPORT.json --markdown-output REPORT.md` reads +UTF-8 JSONL and writes UTF-8 outputs. JSON is sorted, indented by two spaces, and +ends with one newline. Invalid input prints a concise error to stderr and exits +nonzero without a traceback or writing either output artifact. + +## Schema and documentation + +`output.schema.json` uses Draft 2020-12 and rejects missing or additional report +properties. `README.md` includes the CLI form above and states that invalid events +fail the command. diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/workspace/sample-events.jsonl b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/workspace/sample-events.jsonl new file mode 100644 index 0000000..adf7367 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/decomposable-greenfield/workspace/sample-events.jsonl @@ -0,0 +1,3 @@ +{"id":"evt-1","service":"api","severity":"warning","timestamp":"2026-07-11T08:00:00Z","summary":"Synthetic latency warning"} +{"id":"evt-2","service":"worker","severity":"critical","timestamp":"2026-07-11T08:01:00Z","summary":"Synthetic queue stop"} +{"id":"evt-1","service":"api","severity":"info","timestamp":"2026-07-11T08:02:00Z","summary":"Synthetic recovery"} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/prompt.md b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/prompt.md new file mode 100644 index 0000000..6bb644a --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/prompt.md @@ -0,0 +1,12 @@ +# Maintenance negative control + +The support address for this small service changed from `ops@example.invalid` to +`help@example.invalid`. + +Update the address in `service-config.json` and `README.md`. Preserve every other +configuration value, all wording unrelated to the address, and `reference-notes.txt` +byte for byte. Do not add or remove files. + +Modify only files inside the provided workspace. Do not use network access or add +third-party dependencies. Run the supplied local validation command and report its +result concisely. diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/rubric.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/rubric.json new file mode 100644 index 0000000..5731515 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/rubric.json @@ -0,0 +1,27 @@ +{ + "schema_version": "route_pilot_rubric.v0.1", + "task_id": "maintenance-negative-control", + "scored": true, + "total_points": 100, + "hard_gates": [ + "Local deterministic validation passes.", + "No file outside the copied workspace is modified." + ], + "criteria": [ + { + "criterion_id": "configuration_update", + "description": "The configuration contains the new support address and no other semantic change.", + "points": 40 + }, + { + "criterion_id": "documentation_update", + "description": "The README contains the new address with all other wording preserved.", + "points": 35 + }, + { + "criterion_id": "scope_discipline", + "description": "The reference file and workspace file set are unchanged.", + "points": 25 + } + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/task.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/task.json new file mode 100644 index 0000000..37294cf --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/task.json @@ -0,0 +1,35 @@ +{ + "schema_version": "route_pilot_task.v0.1", + "task_id": "maintenance-negative-control", + "task_class": "maintenance_negative_control", + "title": "Update one support address without collateral changes", + "scored": true, + "deterministic": true, + "network_required": false, + "third_party_dependencies": false, + "path_scope": "fixture_relative", + "editable_paths": [ + "workspace/README.md", + "workspace/service-config.json" + ], + "protected_paths": [ + "prompt.md", + "rubric.json", + "task.json", + "validate.py", + "workspace/reference-notes.txt" + ], + "validation_commands": [ + [ + "/usr/local/bin/python3", + "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/validate.py", + "--workspace", + "{workspace}" + ] + ], + "success_criteria": [ + "The support address is help@example.invalid in configuration and documentation.", + "All non-target configuration fields and reference notes remain byte-for-byte unchanged.", + "No file is added, removed, or changed outside the two allowed write paths." + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/validate.py b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/validate.py new file mode 100644 index 0000000..c302ac0 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/validate.py @@ -0,0 +1,84 @@ +"""Validate the maintenance negative-control workspace.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +EXPECTED_FILES = {"README.md", "reference-notes.txt", "service-config.json"} +EXPECTED_CONFIG = { + "cache_ttl_seconds": 300, + "features": {"audit_log": True, "beta_banner": False}, + "schema_version": 1, + "service_name": "pilot-widget", + "support_email": "help@example.invalid", +} +EXPECTED_README = """# Pilot widget + +Pilot widget is a deterministic example service used for local maintenance tests. + +For support, contact `help@example.invalid`. + +The cache lifetime is five minutes, and audit logging remains enabled. +""" +EXPECTED_NOTES = """fixture-version=1 +owner=example-team +classification=synthetic +""" + + +def _load_json(path: Path) -> Any: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def validate(workspace: Path) -> list[str]: + failures: list[str] = [] + if not workspace.is_dir(): + return ["workspace: directory does not exist"] + if {item.name for item in workspace.iterdir()} != EXPECTED_FILES: + failures.append("workspace: file set changed") + if any(item.is_symlink() for item in workspace.rglob("*")): + failures.append("workspace: symbolic links are not allowed") + + try: + config = _load_json(workspace / "service-config.json") + except (OSError, ValueError) as error: + failures.append(f"configuration: invalid JSON ({type(error).__name__})") + else: + if config != EXPECTED_CONFIG: + failures.append("configuration: target or non-target values differ") + + try: + if (workspace / "README.md").read_text(encoding="utf-8") != EXPECTED_README: + failures.append("documentation: README differs from the required result") + if (workspace / "reference-notes.txt").read_text( + encoding="utf-8" + ) != EXPECTED_NOTES: + failures.append("scope: reference-notes.txt changed") + except OSError as error: + failures.append( + f"workspace: required file unavailable ({type(error).__name__})" + ) + return sorted(set(failures)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", required=True, type=Path) + args = parser.parse_args() + failures = validate(args.workspace) + print( + json.dumps( + {"failures": failures, "status": "pass" if not failures else "fail"}, + sort_keys=True, + ) + ) + return 0 if not failures else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/README.md b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/README.md new file mode 100644 index 0000000..d1a7512 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/README.md @@ -0,0 +1,7 @@ +# Pilot widget + +Pilot widget is a deterministic example service used for local maintenance tests. + +For support, contact `ops@example.invalid`. + +The cache lifetime is five minutes, and audit logging remains enabled. diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/reference-notes.txt b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/reference-notes.txt new file mode 100644 index 0000000..3e00087 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/reference-notes.txt @@ -0,0 +1,3 @@ +fixture-version=1 +owner=example-team +classification=synthetic diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/service-config.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/service-config.json new file mode 100644 index 0000000..4ffe27e --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/maintenance-negative-control/workspace/service-config.json @@ -0,0 +1,10 @@ +{ + "cache_ttl_seconds": 300, + "features": { + "audit_log": true, + "beta_banner": false + }, + "schema_version": 1, + "service_name": "pilot-widget", + "support_email": "ops@example.invalid" +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/prompt.md b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/prompt.md new file mode 100644 index 0000000..989b5b4 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/prompt.md @@ -0,0 +1,13 @@ +# Tightly coupled debugging task + +Repair `schedule_jobs` in `scheduler.py`. The current implementation violates +several interacting requirements from `README.md` concerning event time, priority, +worker availability, tie-breaking, validation, and input immutability. + +Implement the complete documented contract, not a special case. Keep the public +function name and return shape unchanged. `README.md` is frozen and must not be +edited. + +Modify only files inside the provided workspace. Do not use network access or add +third-party dependencies. Run the supplied local validation commands and report +their results concisely. diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/rubric.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/rubric.json new file mode 100644 index 0000000..397da46 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/rubric.json @@ -0,0 +1,32 @@ +{ + "schema_version": "route_pilot_rubric.v0.1", + "task_id": "tightly-coupled-debugging", + "scored": true, + "total_points": 100, + "hard_gates": [ + "Candidate code compiles and local deterministic validation passes.", + "No file outside the copied workspace is modified." + ], + "criteria": [ + { + "criterion_id": "event_simulation", + "description": "Scheduling respects readiness, availability, non-preemption, and half-open timing.", + "points": 40 + }, + { + "criterion_id": "ordering", + "description": "Priority, readiness, job identifier, and worker identifier tie-breaks are deterministic.", + "points": 25 + }, + { + "criterion_id": "validation", + "description": "Malformed inputs raise ValueError and caller inputs remain unchanged.", + "points": 20 + }, + { + "criterion_id": "scope_and_clarity", + "description": "The patch is stdlib-only, focused, readable, and preserves frozen files.", + "points": 15 + } + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/task.json b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/task.json new file mode 100644 index 0000000..e4bb039 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/task.json @@ -0,0 +1,34 @@ +{ + "schema_version": "route_pilot_task.v0.1", + "task_id": "tightly-coupled-debugging", + "task_class": "tightly_coupled", + "title": "Repair a deterministic non-preemptive scheduler", + "scored": true, + "deterministic": true, + "network_required": false, + "third_party_dependencies": false, + "path_scope": "fixture_relative", + "editable_paths": [ + "workspace/scheduler.py" + ], + "protected_paths": [ + "prompt.md", + "rubric.json", + "task.json", + "validate.py", + "workspace/README.md" + ], + "validation_commands": [ + [ + "/usr/local/bin/python3", + "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/validate.py", + "--workspace", + "{workspace}" + ] + ], + "success_criteria": [ + "schedule_jobs implements the complete event, priority, tie-break, and worker assignment contract.", + "Invalid jobs and worker counts raise ValueError without mutating the caller's input.", + "README.md and the workspace file set remain unchanged." + ] +} diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/validate.py b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/validate.py new file mode 100644 index 0000000..93e402e --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/validate.py @@ -0,0 +1,209 @@ +"""Behavioral validator for the tightly coupled scheduler task.""" + +from __future__ import annotations + +import argparse +import ast +import copy +import json +from pathlib import Path +import sys +from typing import Any, Callable + + +EXPECTED_FILES = {"README.md", "scheduler.py"} +EXPECTED_README = (Path(__file__).parent / "workspace" / "README.md").read_text( + encoding="utf-8" +) + + +def _valid_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _reference_schedule( + jobs: list[dict[str, Any]], worker_count: int +) -> list[dict[str, Any]]: + if not _valid_int(worker_count) or worker_count <= 0: + raise ValueError("worker_count must be a positive integer") + required = {"id", "ready_at", "duration", "priority"} + pending: list[dict[str, Any]] = [] + identifiers: set[str] = set() + if not isinstance(jobs, list): + raise ValueError("jobs must be a list") + for item in jobs: + if not isinstance(item, dict) or set(item) != required: + raise ValueError("each job must have exactly the required fields") + identifier = item["id"] + if ( + not isinstance(identifier, str) + or not identifier + or identifier in identifiers + ): + raise ValueError("job ids must be non-empty and unique") + if not _valid_int(item["ready_at"]) or item["ready_at"] < 0: + raise ValueError("ready_at must be a non-negative integer") + if not _valid_int(item["duration"]) or item["duration"] <= 0: + raise ValueError("duration must be a positive integer") + if not _valid_int(item["priority"]): + raise ValueError("priority must be an integer") + identifiers.add(identifier) + pending.append(dict(item)) + + worker_available = [0] * worker_count + output: list[dict[str, Any]] = [] + now = 0 + while pending: + ready = sorted( + (item for item in pending if item["ready_at"] <= now), + key=lambda item: (-item["priority"], item["ready_at"], item["id"]), + ) + workers = [ + worker_id + for worker_id, available_at in enumerate(worker_available) + if available_at <= now + ] + if ready and workers: + for worker_id, job in zip(workers, ready, strict=False): + end = now + job["duration"] + output.append( + { + "job_id": job["id"], + "worker_id": worker_id, + "start": now, + "end": end, + } + ) + worker_available[worker_id] = end + pending.remove(job) + continue + now = max( + min(item["ready_at"] for item in pending), + min(worker_available), + ) + return output + + +def _load_candidate(path: Path) -> Callable[[list[dict[str, Any]], int], Any]: + source = path.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(path)) + for node in ast.walk(tree): + names: list[str] = [] + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + names = [node.module] + for name in names: + if name.split(".", 1)[0] not in sys.stdlib_module_names: + raise ValueError(f"third-party import is not allowed: {name}") + namespace: dict[str, Any] = {"__name__": "candidate_scheduler"} + exec(compile(tree, str(path), "exec"), namespace) + candidate = namespace.get("schedule_jobs") + if not callable(candidate): + raise ValueError("schedule_jobs is missing or not callable") + return candidate + + +def validate(workspace: Path) -> list[str]: + failures: list[str] = [] + if not workspace.is_dir(): + return ["workspace: directory does not exist"] + if {item.name for item in workspace.iterdir()} != EXPECTED_FILES: + failures.append("workspace: file set changed") + if any(item.is_symlink() for item in workspace.rglob("*")): + failures.append("workspace: symbolic links are not allowed") + try: + if (workspace / "README.md").read_text(encoding="utf-8") != EXPECTED_README: + failures.append("scope: README.md changed") + candidate = _load_candidate(workspace / "scheduler.py") + except Exception as error: # noqa: BLE001 - candidate code is an evaluation input + failures.append(f"candidate: load failed ({type(error).__name__}: {error})") + return sorted(set(failures)) + + cases = [ + ( + [ + {"id": "low", "ready_at": 0, "duration": 4, "priority": 1}, + {"id": "high", "ready_at": 0, "duration": 2, "priority": 5}, + {"id": "future", "ready_at": 1, "duration": 1, "priority": 9}, + ], + 1, + ), + ( + [ + {"id": "b", "ready_at": 0, "duration": 3, "priority": 2}, + {"id": "a", "ready_at": 0, "duration": 3, "priority": 2}, + {"id": "c", "ready_at": 3, "duration": 2, "priority": 2}, + {"id": "d", "ready_at": 1, "duration": 4, "priority": 1}, + ], + 2, + ), + ( + [ + {"id": "late", "ready_at": 7, "duration": 2, "priority": -1}, + {"id": "later", "ready_at": 12, "duration": 1, "priority": 0}, + ], + 3, + ), + ( + [ + {"id": "first", "ready_at": 0, "duration": 2, "priority": 0}, + {"id": "boundary", "ready_at": 2, "duration": 2, "priority": 0}, + ], + 1, + ), + ] + for index, (jobs, worker_count) in enumerate(cases): + original = copy.deepcopy(jobs) + try: + observed = candidate(jobs, worker_count) + except Exception as error: # noqa: BLE001 - candidate code is an evaluation input + failures.append(f"behavior[{index}]: raised {type(error).__name__}") + continue + if jobs != original: + failures.append(f"behavior[{index}]: input was mutated") + if observed != _reference_schedule(original, worker_count): + failures.append(f"behavior[{index}]: schedule differs") + + invalid_cases: list[tuple[Any, Any]] = [ + ([], 0), + ([{"id": "x", "ready_at": 0, "duration": 0, "priority": 1}], 1), + ([{"id": "x", "ready_at": False, "duration": 1, "priority": 1}], 1), + ( + [ + {"id": "x", "ready_at": 0, "duration": 1, "priority": 1}, + {"id": "x", "ready_at": 1, "duration": 1, "priority": 2}, + ], + 1, + ), + ] + for index, (jobs, worker_count) in enumerate(invalid_cases): + try: + candidate(copy.deepcopy(jobs), worker_count) + except ValueError: + continue + except Exception as error: # noqa: BLE001 - candidate code is an evaluation input + failures.append( + f"validation[{index}]: raised {type(error).__name__}, not ValueError" + ) + else: + failures.append(f"validation[{index}]: ValueError was not raised") + return sorted(set(failures)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", required=True, type=Path) + args = parser.parse_args() + failures = validate(args.workspace) + print( + json.dumps( + {"failures": failures, "status": "pass" if not failures else "fail"}, + sort_keys=True, + ) + ) + return 0 if not failures else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/workspace/README.md b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/workspace/README.md new file mode 100644 index 0000000..608c197 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/workspace/README.md @@ -0,0 +1,21 @@ +# Deterministic job scheduler + +`schedule_jobs(jobs, worker_count)` schedules non-preemptive jobs on workers numbered +from zero. `jobs` is a list of objects with exactly these fields: + +- `id`: non-empty unique string; +- `ready_at`: non-negative integer; +- `duration`: positive integer; +- `priority`: integer, where larger values run first. + +Boolean values are not valid integers. Invalid input raises `ValueError`. The input +list and its dictionaries must not be mutated. + +At each event time, all free workers and ready unscheduled jobs are considered +together. Ready jobs are ordered by descending priority, ascending `ready_at`, then +lexicographic `id`. Free workers are ordered by numeric worker identifier. Pair them +in order. Jobs are non-preemptive and occupy the half-open interval `[start, end)`, +so a worker is free for a job ready exactly at the preceding job's end. + +Return schedule records in assignment order. Every record has exactly `job_id`, +`worker_id`, `start`, and `end`. diff --git a/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/workspace/scheduler.py b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/workspace/scheduler.py new file mode 100644 index 0000000..6457e35 --- /dev/null +++ b/fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tightly-coupled-debugging/workspace/scheduler.py @@ -0,0 +1,26 @@ +"""Buggy deterministic scheduler used by the governed route pilot.""" + +from __future__ import annotations + +from typing import Any + + +def schedule_jobs( + jobs: list[dict[str, Any]], worker_count: int +) -> list[dict[str, Any]]: + """Schedule jobs; this initial implementation intentionally contains defects.""" + + jobs.sort(key=lambda item: (item["ready_at"], -item["priority"])) + schedule: list[dict[str, Any]] = [] + for index, job in enumerate(jobs): + worker_id = index % worker_count + start = job["ready_at"] + schedule.append( + { + "job_id": job["id"], + "worker_id": worker_id, + "start": start, + "end": start + job["duration"], + } + ) + return schedule diff --git a/fixtures/valid/route_evaluation_manifest.v0.1.valid.json b/fixtures/valid/route_evaluation_manifest.v0.1.valid.json new file mode 100644 index 0000000..f59be47 --- /dev/null +++ b/fixtures/valid/route_evaluation_manifest.v0.1.valid.json @@ -0,0 +1,191 @@ +{ + "schema_id": "route_evaluation_manifest.v0.1", + "schema_version": "0.1", + "pilot_id": "gpt-5.6-sol-route-pilot-v0.1", + "status": "preregistered_not_started", + "preregistered_at": "2026-07-11T15:10:00Z", + "protocol_ref": "docs/evaluations/ultra-evaluation-protocol.v0.1.md", + "base_model_id": "gpt-5.6-sol", + "source_repository": { + "url": "https://github.com/hummbl-dev/model-routing-as-code", + "commit_sha": "4e9580f111111111111111111111111111111111", + "clean_required": true + }, + "harness": { + "runner_path": "tools/run_route_evaluation.py", + "runner_sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "validator_path": "tools/validate_route_evaluation.py", + "validator_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "receipt_schema_path": "schemas/route_evaluation_receipt.v0.2.schema.json", + "receipt_schema_sha256": "3333333333333333333333333333333333333333333333333333333333333333", + "python_requirement": ">=3.11", + "python_executable_path": "/usr/local/bin/python3", + "python_version": "3.13.7", + "codex_executable_path": "/usr/local/bin/codex", + "codex_executable_sha256": "c6eb747e4145ecb3bed2647dbd0f8464b190a5ccba964666ef7c98d4681a4a4c" + }, + "catalog_attestation": { + "retrieved_at": "2026-07-11T15:04:04Z", + "catalog_sha256": "3ade3703c3a258c2a7780c3e8314e52489baaa2c8634df2de795029f9364f3eb", + "model_record_sha256": "ad2e4fc51d9e8c355ec5a3c94020525f01e3afc3476a6578f304bffad3363c04", + "comp_hash": "3000", + "cli_version": "0.144.1", + "context_window": 372000, + "multi_agent_version": "v2", + "supported_reasoning_settings": ["xhigh", "max", "ultra"] + }, + "frozen_envelope": { + "fixture_tree_hash_algorithm": "sha256(sorted(relative_posix + NUL + file_sha256 + LF))", + "network_policy": "offline_tools_only", + "sandbox_mode": "workspace-write", + "approval_policy": "never", + "user_config_policy": "ignored", + "github_actions_policy": "prohibited", + "source_access_policy": "fixture_only", + "task_change_policy": "invalidate_all_cells", + "route_output_isolation": true, + "common_cli_flags": [ + "--ignore-user-config", + "--ignore-rules", + "--strict-config", + "--json", + "--output-last-message", + "--sandbox=workspace-write", + "--ask-for-approval=never" + ] + }, + "attestation": { + "fixture_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation", + "tree_sha256": "4444444444444444444444444444444444444444444444444444444444444444", + "prompt_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/attestation/prompt.md", + "prompt_sha256": "5555555555555555555555555555555555555555555555555555555555555555", + "validation_commands": [["python3", "validate.py"]], + "success_criteria": [ + "Effective model and route metadata are available.", + "The deterministic local check passes." + ], + "routes": ["xhigh", "max", "ultra"], + "invocations_per_route": 1, + "scored": false, + "required_before_scored_cells": true + }, + "tasks": [ + { + "task_id": "maintenance-control", + "task_class": "maintenance_negative_control", + "fixture_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/maintenance-control", + "tree_sha256": "6666666666666666666666666666666666666666666666666666666666666666", + "prompt_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/maintenance-control/prompt.md", + "prompt_sha256": "7777777777777777777777777777777777777777777777777777777777777777", + "rubric_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/maintenance-control/rubric.json", + "rubric_sha256": "8888888888888888888888888888888888888888888888888888888888888888", + "validation_commands": [["python3", "-m", "unittest", "discover", "-v"]], + "success_criteria": ["All frozen tests pass."] + }, + { + "task_id": "tight-coupled", + "task_class": "tightly_coupled", + "fixture_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/tight-coupled", + "tree_sha256": "9999999999999999999999999999999999999999999999999999999999999999", + "prompt_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/tight-coupled/prompt.md", + "prompt_sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "rubric_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/tight-coupled/rubric.json", + "rubric_sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "validation_commands": [["python3", "-m", "unittest", "discover", "-v"]], + "success_criteria": ["All coupled invariants hold."] + }, + { + "task_id": "decomposable-build", + "task_class": "decomposable", + "fixture_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/decomposable-build", + "tree_sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "prompt_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/decomposable-build/prompt.md", + "prompt_sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "rubric_path": "fixtures/pilots/gpt-5.6-sol-route-pilot/v0.1/tasks/decomposable-build/rubric.json", + "rubric_sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "validation_commands": [["python3", "-m", "unittest", "discover", "-v"]], + "success_criteria": ["All independent artifacts reconcile."] + } + ], + "arms": [ + { + "arm_id": "xhigh", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "xhigh", + "orchestration": "single_agent", + "expected_agent_count": 1 + }, + { + "arm_id": "max", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "max", + "orchestration": "single_agent", + "expected_agent_count": 1 + }, + { + "arm_id": "ultra", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "ultra", + "orchestration": "automatic_delegation", + "expected_agent_count": 4 + } + ], + "execution_order": [ + {"sequence": 1, "cell_id": "maintenance-control.xhigh.r1", "task_id": "maintenance-control", "arm_id": "xhigh", "repetition": 1}, + {"sequence": 2, "cell_id": "tight-coupled.max.r1", "task_id": "tight-coupled", "arm_id": "max", "repetition": 1}, + {"sequence": 3, "cell_id": "decomposable-build.ultra.r1", "task_id": "decomposable-build", "arm_id": "ultra", "repetition": 1}, + {"sequence": 4, "cell_id": "maintenance-control.max.r1", "task_id": "maintenance-control", "arm_id": "max", "repetition": 1}, + {"sequence": 5, "cell_id": "tight-coupled.ultra.r1", "task_id": "tight-coupled", "arm_id": "ultra", "repetition": 1}, + {"sequence": 6, "cell_id": "decomposable-build.xhigh.r1", "task_id": "decomposable-build", "arm_id": "xhigh", "repetition": 1}, + {"sequence": 7, "cell_id": "maintenance-control.ultra.r1", "task_id": "maintenance-control", "arm_id": "ultra", "repetition": 1}, + {"sequence": 8, "cell_id": "tight-coupled.xhigh.r1", "task_id": "tight-coupled", "arm_id": "xhigh", "repetition": 1}, + {"sequence": 9, "cell_id": "decomposable-build.max.r1", "task_id": "decomposable-build", "arm_id": "max", "repetition": 1} + ], + "budgets": { + "total_inference_invocations_max": 12, + "attestation_invocations": 3, + "scored_invocations": 9, + "attempts_per_cell": 1, + "cell_timeout_seconds": 600, + "pilot_timeout_seconds": 7200, + "resource_budget_regime": "route_native_capacity", + "usd_ceiling": { + "availability": "unavailable", + "reason": "The authenticated runtime does not expose a dollar ceiling." + }, + "token_ceiling": { + "availability": "unavailable", + "reason": "The authenticated runtime does not expose a trustworthy token ceiling." + }, + "per_agent_usd_ceiling": { + "availability": "unavailable", + "reason": "The authenticated runtime does not expose per-agent dollar ceilings." + }, + "per_agent_token_ceiling": { + "availability": "unavailable", + "reason": "The authenticated runtime does not expose per-agent token ceilings." + }, + "stop_conditions": [ + "Stop on a usage or rate limit.", + "Stop before exceeding the invocation maximum." + ] + }, + "raw_output_policy": { + "storage_root_template": "$HOME/_state/model-routing-as-code/route-pilots/{pilot_id}", + "tracked_in_git": false, + "publish_raw_outputs": false, + "content_review_required": true, + "published_receipts_sanitized": true, + "hash_algorithm": "sha256" + }, + "invalidation_conditions": [ + "effective-route-unverified", + "model-reroute", + "catalog-drift", + "fixture-drift", + "usage-limit", + "rate-limit", + "budget-bound", + "validation-contract-change" + ] +} diff --git a/fixtures/valid/route_evaluation_receipt.v0.2.max.valid.json b/fixtures/valid/route_evaluation_receipt.v0.2.max.valid.json new file mode 100644 index 0000000..8e6a314 --- /dev/null +++ b/fixtures/valid/route_evaluation_receipt.v0.2.max.valid.json @@ -0,0 +1,87 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "receipt_id": "route-eval-tight-max-r1", + "recorded_at": "2026-07-11T15:01:10Z", + "pilot_id": "gpt-5.6-sol-route-pilot-v0.1", + "cell": { + "evaluation_kind": "scored", + "cell_id": "tight-coupled.max.r1", + "task_id": "tight-coupled", + "arm_id": "max", + "route": "max", + "repetition": 1, + "sequence": 2, + "manifest_sha256": "1111111111111111111111111111111111111111111111111111111111111111" + }, + "run": { + "started_at": "2026-07-11T15:01:00Z", + "completed_at": {"availability": "available", "value": "2026-07-11T15:01:05Z"}, + "status": "completed", + "validity": "valid", + "invalidation_reasons": [], + "human_intervention_count": 0 + }, + "requested_configuration": { + "provider": "OpenAI", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "max", + "multi_agent_enabled": false, + "orchestration": "single_agent", + "expected_agent_count": 1, + "configuration_source": "Frozen pilot manifest." + }, + "verified_configuration": { + "availability": "available", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "max", + "verification_source": "state_database", + "thread_id": "thread-max-1", + "catalog_comp_hash": "3000" + }, + "reroutes": [], + "execution_units": [], + "delegation": { + "availability": "available", + "enabled": false, + "observed_agent_count": 1, + "max_concurrent_agents": {"availability": "available", "value": 1}, + "measurement_source": "Persisted root-thread metadata." + }, + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Dollar cost was not exposed."}, + "input_tokens": {"availability": "unavailable", "reason": "Input tokens were not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Output tokens were not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "A trustworthy total was not exposed."}, + "elapsed_wall_clock_time": {"availability": "available", "value": 5, "unit": "seconds", "measurement_kind": "harness_monotonic"}, + "token_aggregation_method": "unavailable" + }, + "quality_observations": { + "duplicate_findings": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_omissions": {"availability": "available", "assessment": "none_observed", "details": []}, + "rejected_findings": [] + }, + "validation_performed": [ + {"phase": "post_adversarial", "name": "fixture tests", "status": "passed", "command": ["python3", "-m", "unittest", "discover", "-v"], "result": "All tests passed.", "artifact_refs": ["private:test-log"]} + ], + "outcome": { + "task_success": "pass", + "rubric_score": {"availability": "available", "value": 9, "unit": "points", "measurement_kind": "adjudicated"}, + "critical_failures": [], + "unique_defects": ["coupled-invariant-fixed"], + "validation_failures_before_adversarial_review": 0, + "validation_failures_after_adversarial_review": 0 + }, + "adjudication": { + "status": "blinded_complete", + "reviewer_count": 2, + "decision": "pass", + "rubric_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "notes": ["Reviewers agreed."] + }, + "deviations": [], + "raw_artifacts": [ + {"artifact_id": "stdout-jsonl", "path_ref": "private:stdout.jsonl", "sha256": "3333333333333333333333333333333333333333333333333333333333333333", "contains_sensitive_data": true, "publication_status": "private_only"} + ], + "limitations": ["Dollar and token telemetry were unavailable."] +} diff --git a/fixtures/valid/route_evaluation_receipt.v0.2.ultra.valid.json b/fixtures/valid/route_evaluation_receipt.v0.2.ultra.valid.json new file mode 100644 index 0000000..7d1a92e --- /dev/null +++ b/fixtures/valid/route_evaluation_receipt.v0.2.ultra.valid.json @@ -0,0 +1,186 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "receipt_id": "route-eval-decomposable-ultra-r1", + "recorded_at": "2026-07-11T15:02:10Z", + "pilot_id": "gpt-5.6-sol-route-pilot-v0.1", + "cell": { + "evaluation_kind": "scored", + "cell_id": "decomposable-build.ultra.r1", + "task_id": "decomposable-build", + "arm_id": "ultra", + "route": "ultra", + "repetition": 1, + "sequence": 3, + "manifest_sha256": "1111111111111111111111111111111111111111111111111111111111111111" + }, + "run": { + "started_at": "2026-07-11T15:02:00Z", + "completed_at": {"availability": "available", "value": "2026-07-11T15:02:05Z"}, + "status": "completed", + "validity": "valid", + "invalidation_reasons": [], + "human_intervention_count": 0 + }, + "requested_configuration": { + "provider": "OpenAI", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "ultra", + "multi_agent_enabled": true, + "orchestration": "automatic_delegation", + "expected_agent_count": 4, + "configuration_source": "Frozen pilot manifest." + }, + "verified_configuration": { + "availability": "available", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "ultra", + "verification_source": "runtime_metadata", + "thread_id": "thread-ultra-root", + "catalog_comp_hash": "3000" + }, + "reroutes": [], + "execution_units": [ + { + "unit_id": "root", + "role": "root", + "agent_identity": "root", + "thread_id": {"availability": "available", "value": "thread-ultra-root"}, + "parent_unit_id": null, + "scope": "Own final reconciliation and validation.", + "started_at": {"availability": "available", "value": "2026-07-11T15:02:00Z"}, + "completed_at": {"availability": "available", "value": "2026-07-11T15:02:05Z"}, + "final_state": "completed", + "findings": ["All delegate artifacts reconciled."], + "artifact_refs": ["private:final-output"], + "contribution": "Synthesized and validated the result.", + "duplicate_work": [], + "rejected_findings": [], + "synthesis_omissions": [], + "telemetry": { + "cost": {"availability": "unavailable", "reason": "Per-unit cost was not exposed."}, + "input_tokens": {"availability": "unavailable", "reason": "Per-unit input tokens were not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Per-unit output tokens were not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "Per-unit total tokens were not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Per-unit elapsed time was not exposed."} + } + }, + { + "unit_id": "implementation", + "role": "delegate", + "agent_identity": "implementation-agent", + "thread_id": {"availability": "available", "value": "thread-ultra-implementation"}, + "parent_unit_id": "root", + "scope": "Implement one independent artifact.", + "started_at": {"availability": "available", "value": "2026-07-11T15:02:01Z"}, + "completed_at": {"availability": "available", "value": "2026-07-11T15:02:04Z"}, + "final_state": "completed", + "findings": ["Implementation passed its local checks."], + "artifact_refs": ["private:implementation-diff"], + "contribution": "Produced the implementation artifact.", + "duplicate_work": [], + "rejected_findings": [], + "synthesis_omissions": [], + "telemetry": { + "cost": {"availability": "unavailable", "reason": "Per-unit cost was not exposed."}, + "input_tokens": {"availability": "unavailable", "reason": "Per-unit input tokens were not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Per-unit output tokens were not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "Per-unit total tokens were not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Per-unit elapsed time was not exposed."} + } + }, + { + "unit_id": "evidence", + "role": "delegate", + "agent_identity": "evidence-agent", + "thread_id": {"availability": "available", "value": "thread-ultra-evidence"}, + "parent_unit_id": "root", + "scope": "Verify independent evidence.", + "started_at": {"availability": "available", "value": "2026-07-11T15:02:01Z"}, + "completed_at": {"availability": "available", "value": "2026-07-11T15:02:04Z"}, + "final_state": "completed", + "findings": ["Evidence boundary was preserved."], + "artifact_refs": ["private:evidence-report"], + "contribution": "Verified evidence independently.", + "duplicate_work": ["Rechecked one implementation claim."], + "rejected_findings": [], + "synthesis_omissions": [], + "telemetry": { + "cost": {"availability": "unavailable", "reason": "Per-unit cost was not exposed."}, + "input_tokens": {"availability": "unavailable", "reason": "Per-unit input tokens were not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Per-unit output tokens were not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "Per-unit total tokens were not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Per-unit elapsed time was not exposed."} + } + }, + { + "unit_id": "adversarial-review", + "role": "delegate", + "agent_identity": "adversarial-review-agent", + "thread_id": {"availability": "available", "value": "thread-ultra-review"}, + "parent_unit_id": "root", + "scope": "Adversarially review the completed artifacts.", + "started_at": {"availability": "available", "value": "2026-07-11T15:02:01Z"}, + "completed_at": {"availability": "available", "value": "2026-07-11T15:02:04Z"}, + "final_state": "completed", + "findings": ["No unresolved critical defect remained."], + "artifact_refs": ["private:review-report"], + "contribution": "Performed independent adversarial review.", + "duplicate_work": [], + "rejected_findings": ["One suspected defect was disproved by a test."], + "synthesis_omissions": [], + "telemetry": { + "cost": {"availability": "unavailable", "reason": "Per-unit cost was not exposed."}, + "input_tokens": {"availability": "unavailable", "reason": "Per-unit input tokens were not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Per-unit output tokens were not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "Per-unit total tokens were not exposed."}, + "elapsed_wall_clock_time": {"availability": "unavailable", "reason": "Per-unit elapsed time was not exposed."} + } + } + ], + "delegation": { + "availability": "available", + "enabled": true, + "observed_agent_count": 4, + "max_concurrent_agents": {"availability": "available", "value": 4}, + "measurement_source": "Persisted thread and spawn-edge metadata." + }, + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Dollar cost was not exposed."}, + "input_tokens": {"availability": "unavailable", "reason": "Input tokens were not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Output tokens were not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "A trustworthy aggregate was not exposed."}, + "elapsed_wall_clock_time": {"availability": "available", "value": 5, "unit": "seconds", "measurement_kind": "harness_monotonic"}, + "token_aggregation_method": "unavailable" + }, + "quality_observations": { + "duplicate_findings": {"availability": "available", "assessment": "observed", "details": ["Evidence lane independently rechecked one implementation claim."]}, + "synthesis_omissions": {"availability": "available", "assessment": "none_observed", "details": []}, + "rejected_findings": ["One suspected defect was disproved by a test."] + }, + "validation_performed": [ + {"phase": "pre_adversarial", "name": "initial fixture tests", "status": "failed", "command": ["python3", "-m", "unittest", "discover", "-v"], "result": "One defect remained.", "artifact_refs": ["private:initial-test-log"]}, + {"phase": "post_adversarial", "name": "final fixture tests", "status": "passed", "command": ["python3", "-m", "unittest", "discover", "-v"], "result": "All tests passed.", "artifact_refs": ["private:final-test-log"]} + ], + "outcome": { + "task_success": "pass", + "rubric_score": {"availability": "available", "value": 10, "unit": "points", "measurement_kind": "adjudicated"}, + "critical_failures": [], + "unique_defects": ["adversarial-only-defect"], + "validation_failures_before_adversarial_review": 1, + "validation_failures_after_adversarial_review": 0 + }, + "adjudication": { + "status": "blinded_complete", + "reviewer_count": 2, + "decision": "pass", + "rubric_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "notes": ["Reviewers agreed."] + }, + "deviations": [], + "raw_artifacts": [ + {"artifact_id": "stdout-jsonl", "path_ref": "private:stdout.jsonl", "sha256": "3333333333333333333333333333333333333333333333333333333333333333", "contains_sensitive_data": true, "publication_status": "private_only"}, + {"artifact_id": "workspace-diff", "path_ref": "private:workspace.diff", "sha256": "4444444444444444444444444444444444444444444444444444444444444444", "contains_sensitive_data": false, "publication_status": "public_safe"} + ], + "limitations": ["Dollar and token telemetry were unavailable."] +} diff --git a/fixtures/valid/route_evaluation_receipt.v0.2.xhigh.valid.json b/fixtures/valid/route_evaluation_receipt.v0.2.xhigh.valid.json new file mode 100644 index 0000000..830495a --- /dev/null +++ b/fixtures/valid/route_evaluation_receipt.v0.2.xhigh.valid.json @@ -0,0 +1,87 @@ +{ + "schema_id": "route_evaluation_receipt.v0.2", + "schema_version": "0.2", + "receipt_id": "route-eval-maintenance-xhigh-r1", + "recorded_at": "2026-07-11T15:00:10Z", + "pilot_id": "gpt-5.6-sol-route-pilot-v0.1", + "cell": { + "evaluation_kind": "scored", + "cell_id": "maintenance-control.xhigh.r1", + "task_id": "maintenance-control", + "arm_id": "xhigh", + "route": "xhigh", + "repetition": 1, + "sequence": 1, + "manifest_sha256": "1111111111111111111111111111111111111111111111111111111111111111" + }, + "run": { + "started_at": "2026-07-11T15:00:00Z", + "completed_at": {"availability": "available", "value": "2026-07-11T15:00:05Z"}, + "status": "completed", + "validity": "valid", + "invalidation_reasons": [], + "human_intervention_count": 0 + }, + "requested_configuration": { + "provider": "OpenAI", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "xhigh", + "multi_agent_enabled": false, + "orchestration": "single_agent", + "expected_agent_count": 1, + "configuration_source": "Frozen pilot manifest." + }, + "verified_configuration": { + "availability": "available", + "model_id": "gpt-5.6-sol", + "reasoning_setting": "xhigh", + "verification_source": "runtime_metadata", + "thread_id": "thread-xhigh-1", + "catalog_comp_hash": "3000" + }, + "reroutes": [], + "execution_units": [], + "delegation": { + "availability": "available", + "enabled": false, + "observed_agent_count": 1, + "max_concurrent_agents": {"availability": "available", "value": 1}, + "measurement_source": "Persisted root-thread metadata." + }, + "telemetry": { + "aggregate_cost": {"availability": "unavailable", "reason": "Dollar cost was not exposed."}, + "input_tokens": {"availability": "unavailable", "reason": "Input tokens were not exposed."}, + "output_tokens": {"availability": "unavailable", "reason": "Output tokens were not exposed."}, + "total_tokens": {"availability": "unavailable", "reason": "A trustworthy total was not exposed."}, + "elapsed_wall_clock_time": {"availability": "available", "value": 5, "unit": "seconds", "measurement_kind": "harness_monotonic"}, + "token_aggregation_method": "unavailable" + }, + "quality_observations": { + "duplicate_findings": {"availability": "available", "assessment": "none_observed", "details": []}, + "synthesis_omissions": {"availability": "available", "assessment": "none_observed", "details": []}, + "rejected_findings": [] + }, + "validation_performed": [ + {"phase": "post_adversarial", "name": "fixture tests", "status": "passed", "command": ["python3", "-m", "unittest", "discover", "-v"], "result": "All tests passed.", "artifact_refs": ["private:test-log"]} + ], + "outcome": { + "task_success": "pass", + "rubric_score": {"availability": "available", "value": 10, "unit": "points", "measurement_kind": "adjudicated"}, + "critical_failures": [], + "unique_defects": ["bounded-defect-fixed"], + "validation_failures_before_adversarial_review": 0, + "validation_failures_after_adversarial_review": 0 + }, + "adjudication": { + "status": "blinded_complete", + "reviewer_count": 2, + "decision": "pass", + "rubric_sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "notes": ["Reviewers agreed."] + }, + "deviations": [], + "raw_artifacts": [ + {"artifact_id": "stdout-jsonl", "path_ref": "private:stdout.jsonl", "sha256": "3333333333333333333333333333333333333333333333333333333333333333", "contains_sensitive_data": true, "publication_status": "private_only"} + ], + "limitations": ["Dollar and token telemetry were unavailable."] +} diff --git a/schemas/route_evaluation_manifest.v0.1.schema.json b/schemas/route_evaluation_manifest.v0.1.schema.json new file mode 100644 index 0000000..8546eef --- /dev/null +++ b/schemas/route_evaluation_manifest.v0.1.schema.json @@ -0,0 +1,391 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/hummbl-dev/model-routing-as-code/main/schemas/route_evaluation_manifest.v0.1.schema.json", + "title": "Route Evaluation Manifest v0.1", + "description": "Preregistered, route-neutral contract for a frozen xhigh, Max, and Ultra evaluation pilot.", + "type": "object", + "required": [ + "schema_id", + "schema_version", + "pilot_id", + "status", + "preregistered_at", + "protocol_ref", + "base_model_id", + "source_repository", + "harness", + "catalog_attestation", + "frozen_envelope", + "attestation", + "tasks", + "arms", + "execution_order", + "budgets", + "raw_output_policy", + "invalidation_conditions" + ], + "properties": { + "schema_id": { + "const": "route_evaluation_manifest.v0.1" + }, + "schema_version": { + "const": "0.1" + }, + "pilot_id": { + "$ref": "#/$defs/identifier" + }, + "status": { + "const": "preregistered_not_started" + }, + "preregistered_at": { + "type": "string", + "format": "date-time" + }, + "protocol_ref": { + "$ref": "#/$defs/nonEmptyString" + }, + "base_model_id": { + "type": "string", + "const": "gpt-5.6-sol" + }, + "source_repository": { + "type": "object", + "required": ["url", "commit_sha", "clean_required"], + "properties": { + "url": {"$ref": "#/$defs/nonEmptyString"}, + "commit_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "clean_required": {"const": true} + }, + "additionalProperties": false + }, + "harness": { + "type": "object", + "required": [ + "runner_path", + "runner_sha256", + "validator_path", + "validator_sha256", + "receipt_schema_path", + "receipt_schema_sha256", + "python_requirement", + "python_executable_path", + "python_version", + "codex_executable_path", + "codex_executable_sha256" + ], + "properties": { + "runner_path": {"$ref": "#/$defs/relativePath"}, + "runner_sha256": {"$ref": "#/$defs/sha256"}, + "validator_path": {"$ref": "#/$defs/relativePath"}, + "validator_sha256": {"$ref": "#/$defs/sha256"}, + "receipt_schema_path": {"$ref": "#/$defs/relativePath"}, + "receipt_schema_sha256": {"$ref": "#/$defs/sha256"}, + "python_requirement": {"const": ">=3.11"}, + "python_executable_path": {"const": "/usr/local/bin/python3"}, + "python_version": {"const": "3.13.7"}, + "codex_executable_path": {"const": "/usr/local/bin/codex"}, + "codex_executable_sha256": {"$ref": "#/$defs/sha256"} + }, + "additionalProperties": false + }, + "catalog_attestation": { + "type": "object", + "required": [ + "retrieved_at", + "catalog_sha256", + "model_record_sha256", + "comp_hash", + "cli_version", + "context_window", + "multi_agent_version", + "supported_reasoning_settings" + ], + "properties": { + "retrieved_at": {"type": "string", "format": "date-time"}, + "catalog_sha256": {"$ref": "#/$defs/sha256"}, + "model_record_sha256": {"$ref": "#/$defs/sha256"}, + "comp_hash": {"const": "3000"}, + "cli_version": {"const": "0.144.1"}, + "context_window": {"const": 372000}, + "multi_agent_version": {"const": "v2"}, + "supported_reasoning_settings": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "uniqueItems": true, + "items": {"$ref": "#/$defs/route"} + } + }, + "additionalProperties": false + }, + "frozen_envelope": { + "type": "object", + "required": [ + "fixture_tree_hash_algorithm", + "network_policy", + "sandbox_mode", + "approval_policy", + "user_config_policy", + "github_actions_policy", + "source_access_policy", + "task_change_policy", + "route_output_isolation", + "common_cli_flags" + ], + "properties": { + "fixture_tree_hash_algorithm": { + "const": "sha256(sorted(relative_posix + NUL + file_sha256 + LF))" + }, + "network_policy": {"const": "offline_tools_only"}, + "sandbox_mode": {"const": "workspace-write"}, + "approval_policy": {"const": "never"}, + "user_config_policy": {"const": "ignored"}, + "github_actions_policy": {"const": "prohibited"}, + "source_access_policy": {"const": "fixture_only"}, + "task_change_policy": {"const": "invalidate_all_cells"}, + "route_output_isolation": {"const": true}, + "common_cli_flags": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/nonEmptyString"} + } + }, + "additionalProperties": false + }, + "attestation": { + "type": "object", + "required": [ + "fixture_path", + "tree_sha256", + "prompt_path", + "prompt_sha256", + "validation_commands", + "success_criteria", + "routes", + "invocations_per_route", + "scored", + "required_before_scored_cells" + ], + "properties": { + "fixture_path": {"$ref": "#/$defs/relativePath"}, + "tree_sha256": {"$ref": "#/$defs/sha256"}, + "prompt_path": {"$ref": "#/$defs/relativePath"}, + "prompt_sha256": {"$ref": "#/$defs/sha256"}, + "validation_commands": {"$ref": "#/$defs/commands"}, + "success_criteria": {"$ref": "#/$defs/stringList"}, + "routes": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "uniqueItems": true, + "items": {"$ref": "#/$defs/route"} + }, + "invocations_per_route": {"const": 1}, + "scored": {"const": false}, + "required_before_scored_cells": {"const": true} + }, + "additionalProperties": false + }, + "tasks": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": {"$ref": "#/$defs/task"} + }, + "arms": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": {"$ref": "#/$defs/arm"} + }, + "execution_order": { + "type": "array", + "minItems": 9, + "maxItems": 9, + "items": {"$ref": "#/$defs/cell"} + }, + "budgets": { + "type": "object", + "required": [ + "total_inference_invocations_max", + "attestation_invocations", + "scored_invocations", + "attempts_per_cell", + "cell_timeout_seconds", + "pilot_timeout_seconds", + "resource_budget_regime", + "usd_ceiling", + "token_ceiling", + "per_agent_usd_ceiling", + "per_agent_token_ceiling", + "stop_conditions" + ], + "properties": { + "total_inference_invocations_max": {"type": "integer", "minimum": 1}, + "attestation_invocations": {"type": "integer", "minimum": 0}, + "scored_invocations": {"type": "integer", "minimum": 1}, + "attempts_per_cell": {"const": 1}, + "cell_timeout_seconds": {"type": "integer", "minimum": 1}, + "pilot_timeout_seconds": {"type": "integer", "minimum": 1}, + "resource_budget_regime": { + "type": "string", + "enum": ["route_native_capacity", "equal_aggregate_budget"] + }, + "usd_ceiling": {"$ref": "#/$defs/budgetLimit"}, + "token_ceiling": {"$ref": "#/$defs/budgetLimit"}, + "per_agent_usd_ceiling": {"$ref": "#/$defs/budgetLimit"}, + "per_agent_token_ceiling": {"$ref": "#/$defs/budgetLimit"}, + "stop_conditions": {"$ref": "#/$defs/stringList"} + }, + "additionalProperties": false + }, + "raw_output_policy": { + "type": "object", + "required": [ + "storage_root_template", + "tracked_in_git", + "publish_raw_outputs", + "content_review_required", + "published_receipts_sanitized", + "hash_algorithm" + ], + "properties": { + "storage_root_template": {"$ref": "#/$defs/nonEmptyString"}, + "tracked_in_git": {"const": false}, + "publish_raw_outputs": {"const": false}, + "content_review_required": {"const": true}, + "published_receipts_sanitized": {"const": true}, + "hash_algorithm": {"const": "sha256"} + }, + "additionalProperties": false + }, + "invalidation_conditions": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/identifier"} + } + }, + "additionalProperties": false, + "$defs": { + "nonEmptyString": {"type": "string", "minLength": 1}, + "identifier": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "relativePath": {"type": "string", "minLength": 1}, + "route": {"type": "string", "enum": ["xhigh", "max", "ultra"]}, + "stringList": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/nonEmptyString"} + }, + "commands": { + "type": "array", + "minItems": 1, + "items": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/nonEmptyString"} + } + }, + "task": { + "type": "object", + "required": [ + "task_id", + "task_class", + "fixture_path", + "tree_sha256", + "prompt_path", + "prompt_sha256", + "rubric_path", + "rubric_sha256", + "validation_commands", + "success_criteria" + ], + "properties": { + "task_id": {"$ref": "#/$defs/identifier"}, + "task_class": { + "type": "string", + "enum": [ + "maintenance_negative_control", + "tightly_coupled", + "decomposable" + ] + }, + "fixture_path": {"$ref": "#/$defs/relativePath"}, + "tree_sha256": {"$ref": "#/$defs/sha256"}, + "prompt_path": {"$ref": "#/$defs/relativePath"}, + "prompt_sha256": {"$ref": "#/$defs/sha256"}, + "rubric_path": {"$ref": "#/$defs/relativePath"}, + "rubric_sha256": {"$ref": "#/$defs/sha256"}, + "validation_commands": {"$ref": "#/$defs/commands"}, + "success_criteria": {"$ref": "#/$defs/stringList"} + }, + "additionalProperties": false + }, + "arm": { + "type": "object", + "required": [ + "arm_id", + "model_id", + "reasoning_setting", + "orchestration", + "expected_agent_count" + ], + "properties": { + "arm_id": {"$ref": "#/$defs/route"}, + "model_id": {"const": "gpt-5.6-sol"}, + "reasoning_setting": {"$ref": "#/$defs/route"}, + "orchestration": { + "type": "string", + "enum": ["single_agent", "automatic_delegation"] + }, + "expected_agent_count": {"type": "integer", "minimum": 1} + }, + "additionalProperties": false + }, + "cell": { + "type": "object", + "required": ["sequence", "cell_id", "task_id", "arm_id", "repetition"], + "properties": { + "sequence": {"type": "integer", "minimum": 1, "maximum": 9}, + "cell_id": {"$ref": "#/$defs/identifier"}, + "task_id": {"$ref": "#/$defs/identifier"}, + "arm_id": {"$ref": "#/$defs/route"}, + "repetition": {"const": 1} + }, + "additionalProperties": false + }, + "budgetLimit": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "value", "unit", "measurement_kind"], + "properties": { + "availability": {"const": "available"}, + "value": {"type": "number", "minimum": 0}, + "unit": {"type": "string", "enum": ["USD", "tokens"]}, + "measurement_kind": {"const": "configured"} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + } + } +} diff --git a/schemas/route_evaluation_receipt.v0.2.schema.json b/schemas/route_evaluation_receipt.v0.2.schema.json new file mode 100644 index 0000000..fb55aa4 --- /dev/null +++ b/schemas/route_evaluation_receipt.v0.2.schema.json @@ -0,0 +1,633 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/hummbl-dev/model-routing-as-code/main/schemas/route_evaluation_receipt.v0.2.schema.json", + "title": "Route Evaluation Receipt v0.2", + "description": "Route-neutral execution receipt for xhigh, Max, Ultra, and their optional delegation units, with explicit unavailable telemetry.", + "type": "object", + "required": [ + "schema_id", + "schema_version", + "receipt_id", + "recorded_at", + "pilot_id", + "cell", + "run", + "requested_configuration", + "verified_configuration", + "reroutes", + "execution_units", + "delegation", + "telemetry", + "quality_observations", + "validation_performed", + "outcome", + "adjudication", + "deviations", + "raw_artifacts", + "limitations" + ], + "properties": { + "schema_id": {"const": "route_evaluation_receipt.v0.2"}, + "schema_version": {"const": "0.2"}, + "receipt_id": {"$ref": "#/$defs/identifier"}, + "recorded_at": {"type": "string", "format": "date-time"}, + "pilot_id": {"$ref": "#/$defs/identifier"}, + "cell": {"$ref": "#/$defs/cell"}, + "run": {"$ref": "#/$defs/run"}, + "requested_configuration": {"$ref": "#/$defs/requestedConfiguration"}, + "verified_configuration": {"$ref": "#/$defs/verifiedConfiguration"}, + "reroutes": { + "type": "array", + "items": {"$ref": "#/$defs/reroute"} + }, + "execution_units": { + "type": "array", + "items": {"$ref": "#/$defs/executionUnit"} + }, + "delegation": {"$ref": "#/$defs/delegation"}, + "telemetry": {"$ref": "#/$defs/telemetry"}, + "quality_observations": {"$ref": "#/$defs/qualityObservations"}, + "validation_performed": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/validationRecord"} + }, + "outcome": {"$ref": "#/$defs/outcome"}, + "adjudication": {"$ref": "#/$defs/adjudication"}, + "deviations": { + "type": "array", + "items": {"$ref": "#/$defs/deviation"} + }, + "raw_artifacts": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/rawArtifact"} + }, + "limitations": {"$ref": "#/$defs/nonEmptyStringList"} + }, + "additionalProperties": false, + "$defs": { + "nonEmptyString": {"type": "string", "minLength": 1}, + "identifier": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "route": {"type": "string", "enum": ["xhigh", "max", "ultra"]}, + "stringList": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/nonEmptyString"} + }, + "nonEmptyStringList": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": {"$ref": "#/$defs/nonEmptyString"} + }, + "timestampAvailability": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "value"], + "properties": { + "availability": {"const": "available"}, + "value": {"type": "string", "format": "date-time"} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "integerAvailability": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "value"], + "properties": { + "availability": {"const": "available"}, + "value": {"type": "integer", "minimum": 0} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "stringAvailability": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "value"], + "properties": { + "availability": {"const": "available"}, + "value": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "costTelemetry": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "value", "unit", "measurement_kind"], + "properties": { + "availability": {"const": "available"}, + "value": {"type": "number", "minimum": 0}, + "unit": {"const": "USD"}, + "measurement_kind": { + "type": "string", + "enum": ["observed", "provider_estimate"] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "tokenTelemetry": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "value", "unit", "measurement_kind"], + "properties": { + "availability": {"const": "available"}, + "value": {"type": "integer", "minimum": 0}, + "unit": {"const": "tokens"}, + "measurement_kind": { + "type": "string", + "enum": ["observed", "runtime_counter"] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "durationTelemetry": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "value", "unit", "measurement_kind"], + "properties": { + "availability": {"const": "available"}, + "value": {"type": "number", "minimum": 0}, + "unit": {"const": "seconds"}, + "measurement_kind": {"const": "harness_monotonic"} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "scoreAvailability": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "value", "unit", "measurement_kind"], + "properties": { + "availability": {"const": "available"}, + "value": {"type": "number", "minimum": 0}, + "unit": {"const": "points"}, + "measurement_kind": { + "type": "string", + "enum": ["automated", "adjudicated"] + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "cell": { + "type": "object", + "required": [ + "evaluation_kind", + "cell_id", + "task_id", + "arm_id", + "route", + "repetition", + "sequence", + "manifest_sha256" + ], + "properties": { + "evaluation_kind": { + "type": "string", + "enum": ["attestation", "scored"] + }, + "cell_id": {"$ref": "#/$defs/identifier"}, + "task_id": {"$ref": "#/$defs/identifier"}, + "arm_id": {"$ref": "#/$defs/route"}, + "route": {"$ref": "#/$defs/route"}, + "repetition": {"type": "integer", "minimum": 1}, + "sequence": {"type": "integer", "minimum": 0}, + "manifest_sha256": {"$ref": "#/$defs/sha256"} + }, + "additionalProperties": false + }, + "run": { + "type": "object", + "required": [ + "started_at", + "completed_at", + "status", + "validity", + "invalidation_reasons", + "human_intervention_count" + ], + "properties": { + "started_at": {"type": "string", "format": "date-time"}, + "completed_at": {"$ref": "#/$defs/timestampAvailability"}, + "status": { + "type": "string", + "enum": ["completed", "failed", "timeout", "aborted"] + }, + "validity": { + "type": "string", + "enum": ["valid", "invalid", "inconclusive"] + }, + "invalidation_reasons": {"$ref": "#/$defs/stringList"}, + "human_intervention_count": {"type": "integer", "minimum": 0} + }, + "additionalProperties": false + }, + "requestedConfiguration": { + "type": "object", + "required": [ + "provider", + "model_id", + "reasoning_setting", + "multi_agent_enabled", + "orchestration", + "expected_agent_count", + "configuration_source" + ], + "properties": { + "provider": {"const": "OpenAI"}, + "model_id": {"const": "gpt-5.6-sol"}, + "reasoning_setting": {"$ref": "#/$defs/route"}, + "multi_agent_enabled": {"type": "boolean"}, + "orchestration": { + "type": "string", + "enum": ["single_agent", "automatic_delegation"] + }, + "expected_agent_count": {"type": "integer", "minimum": 1}, + "configuration_source": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + }, + "verifiedConfiguration": { + "oneOf": [ + { + "type": "object", + "required": [ + "availability", + "model_id", + "reasoning_setting", + "verification_source", + "thread_id", + "catalog_comp_hash" + ], + "properties": { + "availability": {"const": "available"}, + "model_id": {"$ref": "#/$defs/nonEmptyString"}, + "reasoning_setting": {"$ref": "#/$defs/route"}, + "verification_source": { + "type": "string", + "enum": ["runtime_metadata", "state_database"] + }, + "thread_id": {"$ref": "#/$defs/nonEmptyString"}, + "catalog_comp_hash": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "reroute": { + "type": "object", + "required": ["occurred_at", "from_model_id", "to_model_id", "reason"], + "properties": { + "occurred_at": {"type": "string", "format": "date-time"}, + "from_model_id": {"$ref": "#/$defs/nonEmptyString"}, + "to_model_id": {"$ref": "#/$defs/nonEmptyString"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + }, + "unitTelemetry": { + "type": "object", + "required": ["cost", "input_tokens", "output_tokens", "total_tokens", "elapsed_wall_clock_time"], + "properties": { + "cost": {"$ref": "#/$defs/costTelemetry"}, + "input_tokens": {"$ref": "#/$defs/tokenTelemetry"}, + "output_tokens": {"$ref": "#/$defs/tokenTelemetry"}, + "total_tokens": {"$ref": "#/$defs/tokenTelemetry"}, + "elapsed_wall_clock_time": {"$ref": "#/$defs/durationTelemetry"} + }, + "additionalProperties": false + }, + "executionUnit": { + "type": "object", + "required": [ + "unit_id", + "role", + "agent_identity", + "thread_id", + "parent_unit_id", + "scope", + "started_at", + "completed_at", + "final_state", + "findings", + "artifact_refs", + "contribution", + "duplicate_work", + "rejected_findings", + "synthesis_omissions", + "telemetry" + ], + "properties": { + "unit_id": {"$ref": "#/$defs/identifier"}, + "role": {"type": "string", "enum": ["root", "delegate"]}, + "agent_identity": {"$ref": "#/$defs/nonEmptyString"}, + "thread_id": {"$ref": "#/$defs/stringAvailability"}, + "parent_unit_id": { + "type": ["string", "null"] + }, + "scope": {"$ref": "#/$defs/nonEmptyString"}, + "started_at": {"$ref": "#/$defs/timestampAvailability"}, + "completed_at": {"$ref": "#/$defs/timestampAvailability"}, + "final_state": { + "type": "string", + "enum": ["completed", "failed", "interrupted", "outstanding"] + }, + "findings": {"$ref": "#/$defs/stringList"}, + "artifact_refs": {"$ref": "#/$defs/stringList"}, + "contribution": {"$ref": "#/$defs/nonEmptyString"}, + "duplicate_work": {"$ref": "#/$defs/stringList"}, + "rejected_findings": {"$ref": "#/$defs/stringList"}, + "synthesis_omissions": {"$ref": "#/$defs/stringList"}, + "telemetry": {"$ref": "#/$defs/unitTelemetry"} + }, + "additionalProperties": false + }, + "delegation": { + "oneOf": [ + { + "type": "object", + "required": [ + "availability", + "enabled", + "observed_agent_count", + "max_concurrent_agents", + "measurement_source" + ], + "properties": { + "availability": {"const": "available"}, + "enabled": {"type": "boolean"}, + "observed_agent_count": {"type": "integer", "minimum": 1}, + "max_concurrent_agents": {"$ref": "#/$defs/integerAvailability"}, + "measurement_source": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "telemetry": { + "type": "object", + "required": [ + "aggregate_cost", + "input_tokens", + "output_tokens", + "total_tokens", + "elapsed_wall_clock_time", + "token_aggregation_method" + ], + "properties": { + "aggregate_cost": {"$ref": "#/$defs/costTelemetry"}, + "input_tokens": {"$ref": "#/$defs/tokenTelemetry"}, + "output_tokens": {"$ref": "#/$defs/tokenTelemetry"}, + "total_tokens": {"$ref": "#/$defs/tokenTelemetry"}, + "elapsed_wall_clock_time": {"$ref": "#/$defs/durationTelemetry"}, + "token_aggregation_method": { + "type": "string", + "enum": [ + "unavailable", + "runtime_reported_aggregate", + "root_only", + "sum_execution_units" + ] + } + }, + "additionalProperties": false + }, + "observationalAssessment": { + "oneOf": [ + { + "type": "object", + "required": ["availability", "assessment", "details"], + "properties": { + "availability": {"const": "available"}, + "assessment": { + "type": "string", + "enum": ["observed", "none_observed"] + }, + "details": {"$ref": "#/$defs/stringList"} + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["availability", "reason"], + "properties": { + "availability": {"const": "unavailable"}, + "reason": {"$ref": "#/$defs/nonEmptyString"} + }, + "additionalProperties": false + } + ] + }, + "qualityObservations": { + "type": "object", + "required": ["duplicate_findings", "synthesis_omissions", "rejected_findings"], + "properties": { + "duplicate_findings": {"$ref": "#/$defs/observationalAssessment"}, + "synthesis_omissions": {"$ref": "#/$defs/observationalAssessment"}, + "rejected_findings": {"$ref": "#/$defs/stringList"} + }, + "additionalProperties": false + }, + "validationRecord": { + "type": "object", + "required": ["phase", "name", "status", "command", "result", "artifact_refs"], + "properties": { + "phase": {"type": "string", "enum": ["pre_adversarial", "post_adversarial"]}, + "name": {"$ref": "#/$defs/nonEmptyString"}, + "status": {"type": "string", "enum": ["passed", "failed", "not_run"]}, + "command": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/nonEmptyString"} + }, + "result": {"$ref": "#/$defs/nonEmptyString"}, + "artifact_refs": {"$ref": "#/$defs/stringList"} + }, + "additionalProperties": false + }, + "outcome": { + "type": "object", + "required": [ + "task_success", + "rubric_score", + "critical_failures", + "unique_defects", + "validation_failures_before_adversarial_review", + "validation_failures_after_adversarial_review" + ], + "properties": { + "task_success": { + "type": "string", + "enum": ["pass", "fail", "inconclusive", "not_adjudicated"] + }, + "rubric_score": {"$ref": "#/$defs/scoreAvailability"}, + "critical_failures": {"$ref": "#/$defs/stringList"}, + "unique_defects": {"$ref": "#/$defs/stringList"}, + "validation_failures_before_adversarial_review": { + "type": "integer", + "minimum": 0 + }, + "validation_failures_after_adversarial_review": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "adjudication": { + "type": "object", + "required": ["status", "reviewer_count", "decision", "rubric_sha256", "notes"], + "properties": { + "status": { + "type": "string", + "enum": ["blinded_complete", "unblinded_complete", "not_required", "not_performed"] + }, + "reviewer_count": {"type": "integer", "minimum": 0}, + "decision": { + "type": "string", + "enum": ["pass", "fail", "inconclusive", "not_applicable"] + }, + "rubric_sha256": {"$ref": "#/$defs/sha256"}, + "notes": {"$ref": "#/$defs/stringList"} + }, + "additionalProperties": false + }, + "deviation": { + "type": "object", + "required": ["deviation_id", "severity", "description", "invalidates_run"], + "properties": { + "deviation_id": {"$ref": "#/$defs/identifier"}, + "severity": {"type": "string", "enum": ["minor", "material", "critical"]}, + "description": {"$ref": "#/$defs/nonEmptyString"}, + "invalidates_run": {"type": "boolean"} + }, + "additionalProperties": false + }, + "rawArtifact": { + "type": "object", + "required": [ + "artifact_id", + "path_ref", + "sha256", + "contains_sensitive_data", + "publication_status" + ], + "properties": { + "artifact_id": {"$ref": "#/$defs/identifier"}, + "path_ref": {"$ref": "#/$defs/nonEmptyString"}, + "sha256": {"$ref": "#/$defs/sha256"}, + "contains_sensitive_data": {"type": "boolean"}, + "publication_status": { + "type": "string", + "enum": ["private_only", "sanitized_published", "public_safe"] + } + }, + "additionalProperties": false + } + } +} diff --git a/tests/test_route_evaluation_manifest.py b/tests/test_route_evaluation_manifest.py new file mode 100644 index 0000000..a14612a --- /dev/null +++ b/tests/test_route_evaluation_manifest.py @@ -0,0 +1,232 @@ +"""Tests for the route-neutral pilot manifest contract.""" + +from __future__ import annotations + +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = ROOT / "schemas" / "route_evaluation_manifest.v0.1.schema.json" +VALID_PATH = ROOT / "fixtures" / "valid" / "route_evaluation_manifest.v0.1.valid.json" +INVALID_DIR = ROOT / "fixtures" / "invalid" +VALIDATOR_PATH = ROOT / "tools" / "validate_route_evaluation.py" + + +def load_json(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def load_validator(): + spec = importlib.util.spec_from_file_location( + "route_evaluation_validator", VALIDATOR_PATH + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not import {VALIDATOR_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class RouteEvaluationManifestTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.validator = load_validator() + cls.schema = load_json(SCHEMA_PATH) + cls.valid = load_json(VALID_PATH) + + def test_valid_manifest_passes(self) -> None: + self.assertEqual([], self.validator.validate_document(self.valid, self.schema)) + + def test_schema_is_portable_draft_2020_12(self) -> None: + self.assertEqual( + "https://json-schema.org/draft/2020-12/schema", self.schema["$schema"] + ) + self.assertEqual("route_evaluation_manifest.v0.1", self.valid["schema_id"]) + refs: list[str] = [] + + def collect(value: object) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key == "$ref": + refs.append(item) + collect(item) + elif isinstance(value, list): + for item in value: + collect(item) + + collect(self.schema) + self.assertTrue(all(ref.startswith("#/") for ref in refs)) + + def test_manifest_freezes_three_by_three_order_and_attestation(self) -> None: + self.assertEqual(3, len(self.valid["tasks"])) + self.assertEqual(3, len(self.valid["arms"])) + self.assertEqual(9, len(self.valid["execution_order"])) + self.assertEqual(["xhigh", "max", "ultra"], self.valid["attestation"]["routes"]) + self.assertFalse(self.valid["attestation"]["scored"]) + pairs = { + (cell["task_id"], cell["arm_id"]) for cell in self.valid["execution_order"] + } + self.assertEqual(9, len(pairs)) + + def test_invalid_manifest_fixtures_fail_deterministically(self) -> None: + expectations = { + "route_evaluation_manifest.v0.1.sequence.invalid.json": ( + "$.execution_order", + "sequence", + ), + "route_evaluation_manifest.v0.1.arm.invalid.json": ( + "$.arms", + "ultra", + ), + "route_evaluation_manifest.v0.1.budget.invalid.json": ( + "$.budgets.total_inference_invocations_max", + ), + "route_evaluation_manifest.v0.1.hash.invalid.json": ("sha256",), + "route_evaluation_manifest.v0.1.invalidation.invalid.json": ( + "$.invalidation_conditions", + "model-reroute", + ), + } + names = { + path.name + for path in INVALID_DIR.glob( + "route_evaluation_manifest.v0.1.*.invalid.json" + ) + } + self.assertEqual(set(expectations), names) + for name, fragments in expectations.items(): + with self.subTest(name=name): + document = load_json(INVALID_DIR / name) + first = self.validator.validate_document(document, self.schema) + second = self.validator.validate_document(document, self.schema) + self.assertEqual(first, second) + self.assertEqual(sorted(first), first) + rendered = "\n".join(first) + for fragment in fragments: + self.assertIn(fragment, rendered) + + def test_duplicate_task_route_cell_is_rejected(self) -> None: + manifest = copy.deepcopy(self.valid) + manifest["execution_order"][1]["task_id"] = manifest["execution_order"][0][ + "task_id" + ] + manifest["execution_order"][1]["arm_id"] = manifest["execution_order"][0][ + "arm_id" + ] + errors = self.validator.validate_document(manifest, self.schema) + self.assertIn("duplicate task/arm/repetition cell", "\n".join(errors)) + + def test_budget_units_and_arithmetic_are_enforced(self) -> None: + manifest = copy.deepcopy(self.valid) + manifest["budgets"]["token_ceiling"] = { + "availability": "available", + "value": 100, + "unit": "USD", + "measurement_kind": "configured", + } + manifest["budgets"]["scored_invocations"] = 8 + rendered = "\n".join(self.validator.validate_document(manifest, self.schema)) + self.assertIn("$.budgets.token_ceiling.unit", rendered) + self.assertIn("$.budgets.scored_invocations", rendered) + + def test_task_paths_must_be_safe_and_hash_scoped(self) -> None: + manifest = copy.deepcopy(self.valid) + manifest["tasks"][0]["prompt_path"] = "../outside.md" + errors = self.validator.validate_document(manifest, self.schema) + self.assertIn( + "$.tasks[0].prompt_path: must be a safe repository-relative path", errors + ) + + def test_task_prompt_and_rubric_must_stay_inside_fixture(self) -> None: + manifest = copy.deepcopy(self.valid) + manifest["tasks"][0]["rubric_path"] = "fixtures/elsewhere/rubric.json" + errors = self.validator.validate_document(manifest, self.schema) + self.assertIn( + "$.tasks[0].rubric_path: must be contained by fixture_path", errors + ) + + def test_persisted_thread_attestation_flags_are_frozen(self) -> None: + manifest = copy.deepcopy(self.valid) + flags = manifest["frozen_envelope"]["common_cli_flags"] + flags.remove("--strict-config") + flags.append("--ephemeral") + rendered = "\n".join(self.validator.validate_document(manifest, self.schema)) + self.assertIn("must freeze config/rules", rendered) + self.assertIn("--ephemeral is prohibited", rendered) + + def test_harness_hash_placeholders_are_rejected(self) -> None: + manifest = copy.deepcopy(self.valid) + manifest["harness"]["runner_sha256"] = "0" * 64 + errors = self.validator.validate_document(manifest, self.schema) + self.assertIn( + "$.harness.runner_sha256: sha256 must not be a placeholder", errors + ) + + def test_catalog_and_arm_sets_cannot_drift(self) -> None: + manifest = copy.deepcopy(self.valid) + manifest["catalog_attestation"]["supported_reasoning_settings"] = [ + "xhigh", + "max", + "max", + ] + manifest["arms"][2]["model_id"] = "gpt-5.6-terra" + rendered = "\n".join(self.validator.validate_document(manifest, self.schema)) + self.assertIn("supported_reasoning_settings", rendered) + self.assertIn("$.arms[2].model_id", rendered) + + def test_pilot_timeout_must_cover_all_scored_cells(self) -> None: + manifest = copy.deepcopy(self.valid) + manifest["budgets"]["pilot_timeout_seconds"] = 5399 + errors = self.validator.validate_document(manifest, self.schema) + self.assertIn( + "$.budgets.pilot_timeout_seconds: must cover all scored cell timeouts", + errors, + ) + + def test_non_rfc3339_timestamp_is_rejected(self) -> None: + manifest = copy.deepcopy(self.valid) + manifest["preregistered_at"] = "2026-07-11 10:00:00" + errors = self.validator.validate_document(manifest, self.schema) + self.assertIn( + "$.preregistered_at: must be an RFC 3339 date-time with timezone", + errors, + ) + + def test_cli_accepts_valid_and_rejects_duplicate_json_keys(self) -> None: + valid = subprocess.run( + [sys.executable, str(VALIDATOR_PATH), str(VALID_PATH)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(0, valid.returncode, valid.stderr) + self.assertIn("VALID route_evaluation_manifest.v0.1", valid.stdout) + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "duplicate.json" + path.write_text( + '{"schema_id":"route_evaluation_manifest.v0.1",' + '"schema_id":"route_evaluation_manifest.v0.1"}', + encoding="utf-8", + ) + invalid = subprocess.run( + [sys.executable, str(VALIDATOR_PATH), str(path)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(2, invalid.returncode) + self.assertIn("duplicate JSON object key", invalid.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_route_evaluation_receipt.py b/tests/test_route_evaluation_receipt.py new file mode 100644 index 0000000..6ab48ba --- /dev/null +++ b/tests/test_route_evaluation_receipt.py @@ -0,0 +1,400 @@ +"""Tests for honest route-neutral xhigh, Max, and Ultra receipts.""" + +from __future__ import annotations + +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = ROOT / "schemas" / "route_evaluation_receipt.v0.2.schema.json" +VALID_DIR = ROOT / "fixtures" / "valid" +INVALID_DIR = ROOT / "fixtures" / "invalid" +VALIDATOR_PATH = ROOT / "tools" / "validate_route_evaluation.py" + + +def load_json(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def load_validator(): + spec = importlib.util.spec_from_file_location( + "route_evaluation_validator", VALIDATOR_PATH + ) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not import {VALIDATOR_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class RouteEvaluationReceiptTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.validator = load_validator() + cls.schema = load_json(SCHEMA_PATH) + cls.valid_by_route = { + route: load_json( + VALID_DIR / f"route_evaluation_receipt.v0.2.{route}.valid.json" + ) + for route in ("xhigh", "max", "ultra") + } + + def test_all_three_route_fixtures_pass(self) -> None: + for route, receipt in self.valid_by_route.items(): + with self.subTest(route=route): + self.assertEqual( + [], self.validator.validate_document(receipt, self.schema) + ) + + def test_receipt_schema_is_portable_draft_2020_12(self) -> None: + self.assertEqual( + "https://json-schema.org/draft/2020-12/schema", self.schema["$schema"] + ) + refs: list[str] = [] + + def collect(value: object) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key == "$ref": + refs.append(item) + collect(item) + elif isinstance(value, list): + for item in value: + collect(item) + + collect(self.schema) + self.assertTrue(all(ref.startswith("#/") for ref in refs)) + + def test_single_agent_routes_do_not_fabricate_delegation_units(self) -> None: + for route in ("xhigh", "max"): + receipt = self.valid_by_route[route] + self.assertEqual([], receipt["execution_units"]) + self.assertEqual( + { + "availability": "available", + "enabled": False, + "observed_agent_count": 1, + "max_concurrent_agents": { + "availability": "available", + "value": 1, + }, + "measurement_source": "Persisted root-thread metadata.", + }, + receipt["delegation"], + ) + + def test_ultra_records_root_and_delegate_units(self) -> None: + receipt = self.valid_by_route["ultra"] + self.assertTrue(receipt["delegation"]["enabled"]) + self.assertGreaterEqual(receipt["delegation"]["observed_agent_count"], 2) + self.assertEqual( + {"root", "delegate"}, + {unit["role"] for unit in receipt["execution_units"]}, + ) + + def test_invalid_receipt_fixtures_fail_deterministically(self) -> None: + expectations = { + "route_evaluation_receipt.v0.2.unavailable-value.invalid.json": ( + "$.telemetry.aggregate_cost", + "unavailable", + ), + "route_evaluation_receipt.v0.2.route-verification.invalid.json": ( + "$.verified_configuration", + "effective route", + ), + "route_evaluation_receipt.v0.2.reroute.invalid.json": ( + "$.run.validity", + "model-reroute", + ), + "route_evaluation_receipt.v0.2.delegation.invalid.json": ( + "$.delegation", + "ultra", + ), + "route_evaluation_receipt.v0.2.token-sum.invalid.json": ( + "$.telemetry.total_tokens.value", + ), + "route_evaluation_receipt.v0.2.timestamp.invalid.json": ( + "$.run.completed_at.value", + ), + "route_evaluation_receipt.v0.2.deviation.invalid.json": ( + "$.run.validity", + "invalidates_run", + ), + "route_evaluation_receipt.v0.2.numeric-overflow.invalid.json": ( + "$.telemetry.aggregate_cost.value", + "finite", + ), + } + names = { + path.name + for path in INVALID_DIR.glob("route_evaluation_receipt.v0.2.*.invalid.json") + } + self.assertEqual(set(expectations), names) + for name, fragments in expectations.items(): + with self.subTest(name=name): + document = load_json(INVALID_DIR / name) + first = self.validator.validate_document(document, self.schema) + second = self.validator.validate_document(document, self.schema) + self.assertEqual(first, second) + self.assertEqual(sorted(first), first) + rendered = "\n".join(first) + for fragment in fragments: + self.assertIn(fragment, rendered) + + def test_requested_and_verified_routes_must_match_for_valid_run(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["max"]) + receipt["verified_configuration"]["reasoning_setting"] = "xhigh" + rendered = "\n".join(self.validator.validate_document(receipt, self.schema)) + self.assertIn("effective route", rendered) + self.assertIn("$.run.validity", rendered) + + def test_reroute_cannot_be_hidden_in_valid_run(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["xhigh"]) + receipt["reroutes"] = [ + { + "occurred_at": "2026-07-11T15:00:03Z", + "from_model_id": "gpt-5.6-sol", + "to_model_id": "gpt-5.6-terra", + "reason": "Capacity reroute.", + } + ] + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.run.validity: model-reroute must invalidate or make the run inconclusive", + errors, + ) + + def test_ultra_valid_run_requires_observed_delegation(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["ultra"]) + receipt["delegation"] = { + "availability": "available", + "enabled": False, + "observed_agent_count": 1, + "max_concurrent_agents": {"availability": "available", "value": 1}, + "measurement_source": "Persisted thread metadata.", + } + receipt["execution_units"] = [] + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.delegation: a valid ultra run requires observed automatic delegation", + errors, + ) + + def test_single_agent_route_rejects_delegate_claim(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["xhigh"]) + receipt["delegation"].update( + { + "enabled": True, + "observed_agent_count": 2, + "max_concurrent_agents": {"availability": "available", "value": 2}, + } + ) + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.delegation: xhigh must not fabricate automatic delegation", errors + ) + + def test_valid_run_rejects_unavailable_effective_route(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["max"]) + receipt["verified_configuration"] = { + "availability": "unavailable", + "reason": "Runtime metadata was not exposed.", + } + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.verified_configuration: effective route unavailable; run cannot be valid", + errors, + ) + + def test_total_tokens_equal_input_plus_output_when_all_available(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["max"]) + receipt["telemetry"]["input_tokens"] = { + "availability": "available", + "value": 40, + "unit": "tokens", + "measurement_kind": "observed", + } + receipt["telemetry"]["output_tokens"] = { + "availability": "available", + "value": 20, + "unit": "tokens", + "measurement_kind": "observed", + } + receipt["telemetry"]["total_tokens"] = { + "availability": "available", + "value": 59, + "unit": "tokens", + "measurement_kind": "observed", + } + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.telemetry.total_tokens.value: must equal input_tokens + output_tokens", + errors, + ) + + def test_sum_execution_units_requires_exact_aggregate(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["ultra"]) + for unit in receipt["execution_units"]: + unit["telemetry"]["total_tokens"] = { + "availability": "available", + "value": 10, + "unit": "tokens", + "measurement_kind": "runtime_counter", + } + receipt["telemetry"]["total_tokens"] = { + "availability": "available", + "value": 39, + "unit": "tokens", + "measurement_kind": "runtime_counter", + } + receipt["telemetry"]["token_aggregation_method"] = "sum_execution_units" + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.telemetry.total_tokens.value: must equal summed execution-unit totals", + errors, + ) + + def test_available_token_data_requires_named_aggregation(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["xhigh"]) + receipt["telemetry"]["total_tokens"] = { + "availability": "available", + "value": 100, + "unit": "tokens", + "measurement_kind": "runtime_counter", + } + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.telemetry.token_aggregation_method: cannot be unavailable when token telemetry is available", + errors, + ) + + def test_wall_clock_must_reconcile_with_run_timestamps(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["xhigh"]) + receipt["telemetry"]["elapsed_wall_clock_time"]["value"] = 50 + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.telemetry.elapsed_wall_clock_time.value: differs from run timestamps by more than one second", + errors, + ) + + def test_invalidating_deviation_cannot_leave_run_valid(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["xhigh"]) + receipt["deviations"] = [ + { + "deviation_id": "fixture-drift", + "severity": "material", + "description": "Fixture hash changed after preregistration.", + "invalidates_run": True, + } + ] + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.run.validity: cannot be valid when a deviation invalidates_run", + errors, + ) + + def test_validation_failure_counts_must_match_records(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["xhigh"]) + receipt["outcome"]["validation_failures_before_adversarial_review"] = 2 + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.outcome.validation_failures_before_adversarial_review: must equal " + "failed pre_adversarial validation records", + errors, + ) + + def test_unit_parent_and_timestamps_are_governed(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["ultra"]) + delegate = receipt["execution_units"][1] + delegate["parent_unit_id"] = "missing-root" + delegate["started_at"]["value"] = "2026-07-11T15:02:02Z" + delegate["completed_at"]["value"] = "2026-07-11T15:01:59Z" + rendered = "\n".join(self.validator.validate_document(receipt, self.schema)) + self.assertIn("delegate parent must exist", rendered) + self.assertIn("precedes unit start", rendered) + + def test_quality_observation_shapes_are_honest(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["ultra"]) + receipt["quality_observations"]["duplicate_findings"]["details"] = [] + receipt["quality_observations"]["synthesis_omissions"]["details"] = [ + "Contradictory detail." + ] + rendered = "\n".join(self.validator.validate_document(receipt, self.schema)) + self.assertIn("observed requires at least one detail", rendered) + self.assertIn("none_observed requires an empty array", rendered) + + def test_sensitive_raw_artifact_cannot_be_marked_public_safe(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["xhigh"]) + receipt["raw_artifacts"][0]["publication_status"] = "public_safe" + errors = self.validator.validate_document(receipt, self.schema) + self.assertIn( + "$.raw_artifacts[0].publication_status: sensitive artifact cannot be public_safe", + errors, + ) + + def test_scored_valid_run_requires_completed_adjudication(self) -> None: + receipt = copy.deepcopy(self.valid_by_route["max"]) + receipt["adjudication"].update( + { + "status": "not_performed", + "reviewer_count": 0, + "decision": "not_applicable", + } + ) + rendered = "\n".join(self.validator.validate_document(receipt, self.schema)) + self.assertIn("valid scored run requires completion", rendered) + self.assertIn("valid scored run requires pass", rendered) + + def test_non_finite_in_memory_values_are_rejected(self) -> None: + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(value=repr(value)): + receipt = copy.deepcopy(self.valid_by_route["xhigh"]) + receipt["telemetry"]["aggregate_cost"] = { + "availability": "available", + "value": value, + "unit": "USD", + "measurement_kind": "observed", + } + self.assertIn( + "$.telemetry.aggregate_cost.value: number must be finite", + self.validator.validate_document(receipt, self.schema), + ) + + def test_cli_accepts_valid_and_rejects_nonstandard_number(self) -> None: + valid_path = VALID_DIR / "route_evaluation_receipt.v0.2.xhigh.valid.json" + valid = subprocess.run( + [sys.executable, str(VALIDATOR_PATH), str(valid_path)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(0, valid.returncode, valid.stderr) + self.assertIn("VALID route_evaluation_receipt.v0.2", valid.stdout) + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "nan.json" + path.write_text( + '{"schema_id":"route_evaluation_receipt.v0.2","value":NaN}', + encoding="utf-8", + ) + invalid = subprocess.run( + [sys.executable, str(VALIDATOR_PATH), str(path)], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(2, invalid.returncode) + self.assertIn("non-standard JSON numeric constant", invalid.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_route_evaluation_runner.py b/tests/test_route_evaluation_runner.py new file mode 100644 index 0000000..6ba24e3 --- /dev/null +++ b/tests/test_route_evaluation_runner.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import json +from pathlib import Path +import sqlite3 +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from tools import run_route_evaluation as runner + + +class FakeProcess: + def __init__( + self, + argv: list[str], + *, + stdout: bytes, + stderr: bytes = b"", + returncode: int = 0, + mutate=None, + timeout: bool = False, + ) -> None: + self.argv = argv + self.stdout = stdout + self.stderr = stderr + self.returncode = returncode + self.mutate = mutate + self.timeout = timeout + self.inputs: list[bytes | None] = [] + self.killed = False + + def communicate(self, input=None, timeout=None): + self.inputs.append(input) + if self.timeout and not self.killed: + raise subprocess.TimeoutExpired(self.argv, timeout) + if self.mutate is not None and not self.killed: + workspace = Path(self.argv[self.argv.index("-C") + 1]) + self.mutate(workspace) + return self.stdout, self.stderr + + def kill(self) -> None: + self.killed = True + self.returncode = -9 + + +class RouteEvaluationRunnerTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory(dir="/tmp") + self.root = Path(self.temp.name) + self.repo = self.root / "repo" + self.task = self.repo / "fixture" + self.workspace_source = self.task / "workspace" + self.workspace_source.mkdir(parents=True) + (self.workspace_source / "target.txt").write_text("before\n", encoding="utf-8") + (self.workspace_source / "README.md").write_text("protected\n", encoding="utf-8") + self.prompt_bytes = b"Exact prompt bytes\n\xef\xbb\xbfsecond line\n" + (self.task / "prompt.md").write_bytes(self.prompt_bytes) + (self.task / "rubric.json").write_text("{}\n", encoding="utf-8") + (self.task / "validate.py").write_text("# frozen\n", encoding="utf-8") + task = { + "schema_version": "route_pilot_task.v0.1", + "task_id": "fixture-task", + "task_class": "tightly_coupled", + "scored": True, + "path_scope": "fixture_relative", + "editable_paths": ["workspace/target.txt"], + "protected_paths": [ + "prompt.md", + "rubric.json", + "task.json", + "validate.py", + "workspace/README.md", + ], + "validation_commands": [ + ["python3", "validate.py", "--workspace", "{workspace}"] + ], + } + (self.task / "task.json").write_text(json.dumps(task), encoding="utf-8") + self.state_root = self.root / "private-state" + self.temp_root = self.root / "isolated" + self.temp_root.mkdir() + + def tearDown(self) -> None: + self.temp.cleanup() + + def config(self, **changes) -> runner.RunConfig: + values = { + "mode": "scored", + "route": "max", + "pilot_id": "pilot-v0.1", + "cell_id": "fixture.max.r1", + "repetition": 1, + "sequence": 1, + "task_dir": self.task, + "repo_root": self.repo, + "state_root": self.state_root, + "temp_root": self.temp_root, + "timeout_seconds": 30.0, + "invocation_ceiling": 1, + "codex_executable": "codex", + "state_db": None, + "manifest_path": None, + } + values.update(changes) + return runner.RunConfig(**values) + + def test_command_pins_route_treatment_and_common_flags(self) -> None: + workspace = Path("/tmp/route-workspace") + for route in ("xhigh", "max", "ultra"): + with self.subTest(route=route): + command = runner.build_codex_command("codex", route, workspace) + self.assertEqual(command[:2], ["codex", "exec"]) + for flag in ( + "--ignore-user-config", + "--ignore-rules", + "--strict-config", + "--json", + "--skip-git-repo-check", + "--output-last-message", + ): + self.assertIn(flag, command) + self.assertNotIn("--ephemeral", command) + self.assertNotIn("--output-schema", command) + self.assertEqual(command[command.index("-m") + 1], "gpt-5.6-sol") + self.assertIn(f'model_reasoning_effort="{route}"', command) + self.assertIn('web_search="disabled"', command) + feature_flag = "--enable" if route == "ultra" else "--disable" + self.assertEqual(command[command.index("multi_agent") - 1], feature_flag) + self.assertEqual(command[-1], "-") + + def test_dry_run_never_starts_codex(self) -> None: + with patch.object( + runner.subprocess, + "Popen", + side_effect=AssertionError("dry-run attempted model call"), + ): + summary, bundle = runner.run_evaluation( + self.config(mode="dry-run", invocation_ceiling=0) + ) + self.assertEqual(summary["run"]["status"], "dry_run") + self.assertEqual(summary["invocation"]["actual_count"], 0) + self.assertTrue((bundle / "workspace_final" / "target.txt").is_file()) + + def test_scored_run_preserves_prompt_and_captures_runtime_telemetry(self) -> None: + events = b"\n".join( + [ + b'{"type":"thread.started","thread_id":"thread-1","model":"gpt-5.6-sol","reasoning_effort":"max"}', + b'{"type":"turn.completed","usage":{"input_tokens":12,"output_tokens":7,"total_tokens":19}}', + ] + ) + processes: list[FakeProcess] = [] + + def popen(argv, **kwargs): + process = FakeProcess( + argv, + stdout=events, + mutate=lambda workspace: (workspace / "target.txt").write_text( + "after\n", encoding="utf-8" + ), + ) + processes.append(process) + return process + + def validate(argv, **kwargs): + self.assertFalse(kwargs.get("shell", False)) + self.assertIsInstance(argv, list) + return subprocess.CompletedProcess(argv, 0, b"passed\n", b"") + + summary, bundle = runner.run_evaluation( + self.config(), popen_factory=popen, run_command=validate + ) + self.assertEqual(processes[0].inputs, [self.prompt_bytes]) + self.assertEqual(summary["run"]["status"], "completed") + self.assertEqual(summary["run"]["validity"], "valid") + self.assertEqual(summary["verified_configuration"]["thread_id"], "thread-1") + self.assertEqual(summary["telemetry"]["total_tokens"]["value"], 19) + self.assertEqual(summary["telemetry"]["cost"]["availability"], "unavailable") + self.assertEqual(summary["workspace"]["changed_paths"], ["workspace/target.txt"]) + self.assertTrue((bundle / "stdout.jsonl").is_file()) + self.assertTrue((bundle / "workspace_final" / "target.txt").is_file()) + + def test_protected_path_drift_invalidates_even_when_validation_passes(self) -> None: + events = b'{"type":"thread.started","thread_id":"t","model":"gpt-5.6-sol","reasoning_effort":"max"}\n' + + def popen(argv, **kwargs): + return FakeProcess( + argv, + stdout=events, + mutate=lambda workspace: (workspace / "README.md").write_text( + "tampered\n", encoding="utf-8" + ), + ) + + summary, _ = runner.run_evaluation( + self.config(), + popen_factory=popen, + run_command=lambda argv, **kwargs: subprocess.CompletedProcess( + argv, 0, b"pass", b"" + ), + ) + self.assertEqual(summary["run"]["validity"], "invalid") + self.assertIn("protected_workspace_drift", summary["run"]["invalidation_reasons"]) + self.assertEqual( + summary["workspace"]["protected_drift"], ["workspace/README.md"] + ) + + def test_timeout_kills_process_and_skips_validation(self) -> None: + process_holder: list[FakeProcess] = [] + + def popen(argv, **kwargs): + process = FakeProcess(argv, stdout=b"", timeout=True) + process_holder.append(process) + return process + + summary, _ = runner.run_evaluation( + self.config(timeout_seconds=0.01), + popen_factory=popen, + run_command=lambda *args, **kwargs: self.fail("validation ran after timeout"), + ) + self.assertTrue(process_holder[0].killed) + self.assertEqual(summary["run"]["status"], "timeout") + self.assertEqual(summary["run"]["validity"], "invalid") + + def test_jsonl_parser_marks_missing_and_ambiguous_metadata(self) -> None: + parsed = runner.parse_jsonl( + b"not-json\n" + b'{"type":"thread.started","thread_id":"one","model":"gpt-5.6-sol","reasoning_effort":"max"}\n' + b'{"type":"thread.started","thread_id":"two","model":"other","reasoning_effort":"ultra"}\n' + ) + self.assertEqual(parsed["thread_id"]["availability"], "ambiguous") + self.assertEqual(parsed["model_id"]["availability"], "ambiguous") + self.assertEqual(parsed["usage"]["availability"], "unavailable") + self.assertEqual(parsed["parse_error_count"], 1) + + def test_rate_limit_event_stops_before_validation(self) -> None: + events = b'{"type":"error","error":{"code":"rate_limit_exceeded","message":"429 quota"}}\n' + summary, _ = runner.run_evaluation( + self.config(), + popen_factory=lambda argv, **kwargs: FakeProcess(argv, stdout=events), + run_command=lambda *args, **kwargs: self.fail( + "validation ran after rate limit" + ), + ) + self.assertTrue(summary["limits"]["rate_or_usage_limit_detected"]) + self.assertIn("usage_or_rate_limit", summary["run"]["invalidation_reasons"]) + + def test_state_database_attestation_reads_only_allowed_columns(self) -> None: + database = self.root / "state.sqlite" + with sqlite3.connect(database) as db: + db.execute( + "CREATE TABLE threads (id TEXT PRIMARY KEY, model TEXT, " + "reasoning_effort TEXT, source TEXT, cli_version TEXT, " + "tokens_used INTEGER, title TEXT, preview TEXT)" + ) + db.execute( + "CREATE TABLE thread_spawn_edges (parent_thread_id TEXT, " + "child_thread_id TEXT, status TEXT)" + ) + db.execute( + "INSERT INTO threads VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ("parent", "gpt-5.6-sol", "ultra", "exec", "0.144.1", 42, "secret", "secret"), + ) + db.execute( + "INSERT INTO thread_spawn_edges VALUES (?, ?, ?)", + ("parent", "child", "completed"), + ) + result = runner.read_state_attestation(database, "parent") + self.assertEqual(result["availability"], "available") + self.assertEqual(result["model_id"], "gpt-5.6-sol") + self.assertEqual(result["tokens_used"], 42) + self.assertEqual(result["children"], [{"thread_id": "child", "status": "completed"}]) + self.assertNotIn("title", json.dumps(result)) + + def test_state_database_tokens_do_not_become_aggregate_totals(self) -> None: + events = b'{"type":"thread.started","thread_id":"parent","model":"gpt-5.6-sol","reasoning_effort":"ultra"}\n' + database = self.root / "state.sqlite" + with sqlite3.connect(database) as db: + db.execute( + "CREATE TABLE threads (id TEXT PRIMARY KEY, model TEXT, " + "reasoning_effort TEXT, source TEXT, cli_version TEXT, " + "tokens_used INTEGER)" + ) + db.execute( + "CREATE TABLE thread_spawn_edges (parent_thread_id TEXT, " + "child_thread_id TEXT, status TEXT)" + ) + db.execute( + "INSERT INTO threads VALUES (?, ?, ?, ?, ?, ?)", + ("parent", "gpt-5.6-sol", "ultra", "exec", "0.144.1", 42), + ) + + summary, _ = runner.run_evaluation( + self.config(route="ultra", state_db=database), + popen_factory=lambda argv, **kwargs: FakeProcess(argv, stdout=events), + run_command=lambda argv, **kwargs: subprocess.CompletedProcess( + argv, 0, b"pass", b"" + ), + ) + self.assertEqual(summary["state_attestation"]["tokens_used"], 42) + self.assertEqual(summary["telemetry"]["total_tokens"]["availability"], "unavailable") + + def test_temp_root_under_agents_ancestor_is_rejected(self) -> None: + guarded_root = self.root / "guarded" + guarded_root.mkdir() + (guarded_root / "AGENTS.md").write_text("guard\n", encoding="utf-8") + temp_root = guarded_root / "tmp" + temp_root.mkdir() + with self.assertRaisesRegex(runner.RunnerError, "outside ancestors containing AGENTS.md"): + runner.run_evaluation(self.config(mode="dry-run", invocation_ceiling=0, temp_root=temp_root)) + + def test_task_rejects_shell_string_and_real_run_rejects_zero_ceiling(self) -> None: + task = json.loads((self.task / "task.json").read_text(encoding="utf-8")) + task["validation_commands"] = ["python3 validate.py"] + (self.task / "task.json").write_text(json.dumps(task), encoding="utf-8") + with self.assertRaisesRegex(runner.RunnerError, "argv arrays"): + runner.run_evaluation(self.config(mode="dry-run", invocation_ceiling=0)) + + task["validation_commands"] = [["python3", "validate.py"]] + (self.task / "task.json").write_text(json.dumps(task), encoding="utf-8") + with self.assertRaisesRegex(runner.RunnerError, "invocation ceiling"): + runner.run_evaluation(self.config(invocation_ceiling=0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_route_pilot_packet.py b/tests/test_route_pilot_packet.py new file mode 100644 index 0000000..de530e3 --- /dev/null +++ b/tests/test_route_pilot_packet.py @@ -0,0 +1,395 @@ +"""Contract tests for the frozen GPT-5.6 Sol route pilot packet.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +PILOT_ROOT = ROOT / "experiments" / "gpt-5.6-sol-route-pilot" / "v0.1" +FIXTURE_ROOT = ROOT / "fixtures" / "pilots" / "gpt-5.6-sol-route-pilot" / "v0.1" +MANIFEST_PATH = PILOT_ROOT / "pilot-manifest.json" +PREREGISTRATION_PATH = PILOT_ROOT / "PREREGISTRATION.md" + +SCORED_TASKS = { + "maintenance-negative-control": "maintenance_negative_control", + "tightly-coupled-debugging": "tightly_coupled", + "decomposable-greenfield": "decomposable", +} +PROTECTED_WORKSPACE_PATHS = { + "maintenance-negative-control": {"workspace/reference-notes.txt"}, + "tightly-coupled-debugging": {"workspace/README.md"}, + "decomposable-greenfield": { + "workspace/SPEC.md", + "workspace/sample-events.jsonl", + }, +} +EXPECTED_ORDER = [ + (1, "maintenance-negative-control", "xhigh"), + (2, "maintenance-negative-control", "max"), + (3, "maintenance-negative-control", "ultra"), + (4, "tightly-coupled-debugging", "max"), + (5, "tightly-coupled-debugging", "ultra"), + (6, "tightly-coupled-debugging", "xhigh"), + (7, "decomposable-greenfield", "ultra"), + (8, "decomposable-greenfield", "xhigh"), + (9, "decomposable-greenfield", "max"), +] + + +def load_json(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError(f"expected JSON object: {path}") + return value + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def fixture_tree_sha256(path: Path) -> str: + """Hash sorted relative paths and content hashes for a fixture tree.""" + + lines: list[bytes] = [] + for child in sorted(item for item in path.rglob("*") if item.is_file()): + relative = child.relative_to(path).as_posix() + lines.append(f"{relative}\0{file_sha256(child)}\n".encode()) + return hashlib.sha256(b"".join(lines)).hexdigest() + + +def run_validator(fixture: Path, workspace: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(fixture / "validate.py"), + "--workspace", + str(workspace), + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + + +class RoutePilotPacketTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.manifest = load_json(MANIFEST_PATH) + + def test_manifest_freezes_source_catalog_and_exploratory_status(self) -> None: + self.assertEqual(self.manifest["schema_id"], "route_evaluation_manifest.v0.1") + self.assertEqual(self.manifest["schema_version"], "0.1") + self.assertEqual(self.manifest["status"], "preregistered_not_started") + self.assertEqual(self.manifest["base_model_id"], "gpt-5.6-sol") + source = self.manifest["source_repository"] + self.assertEqual( + source["commit_sha"], + "4e9580f816d0d7982e14149e82c40fe7b67670f2", + ) + self.assertTrue(source["clean_required"]) + harness = self.manifest["harness"] + self.assertEqual(harness["python_executable_path"], "/usr/local/bin/python3") + self.assertEqual(harness["python_version"], "3.13.7") + self.assertEqual(harness["codex_executable_path"], "/usr/local/bin/codex") + self.assertEqual( + harness["codex_executable_sha256"], + "c6eb747e4145ecb3bed2647dbd0f8464b190a5ccba964666ef7c98d4681a4a4c", + ) + catalog = self.manifest["catalog_attestation"] + self.assertEqual(catalog["cli_version"], "0.144.1") + self.assertEqual( + catalog["catalog_sha256"], + "3ade3703c3a258c2a7780c3e8314e52489baaa2c8634df2de795029f9364f3eb", + ) + self.assertEqual(catalog["comp_hash"], "3000") + self.assertEqual(catalog["context_window"], 372000) + self.assertEqual(catalog["multi_agent_version"], "v2") + flags = set(self.manifest["frozen_envelope"]["common_cli_flags"]) + self.assertTrue( + { + "--ignore-user-config", + "--ignore-rules", + "--strict-config", + "--json", + "--output-last-message", + } + <= flags + ) + self.assertNotIn("--ephemeral", flags) + self.assertNotIn("--output-schema", flags) + + def test_harness_paths_and_hashes_reconcile(self) -> None: + harness = self.manifest["harness"] + for path_field, hash_field in ( + ("runner_path", "runner_sha256"), + ("validator_path", "validator_sha256"), + ("receipt_schema_path", "receipt_schema_sha256"), + ): + with self.subTest(path_field=path_field): + path = ROOT / harness[path_field] + self.assertTrue(path.is_file(), path) + self.assertEqual(harness[hash_field], file_sha256(path)) + + def test_latin_square_execution_order_is_frozen(self) -> None: + observed = [ + (cell["sequence"], cell["task_id"], cell["arm_id"]) + for cell in self.manifest["execution_order"] + ] + self.assertEqual(observed, EXPECTED_ORDER) + self.assertTrue( + all(cell["repetition"] == 1 for cell in self.manifest["execution_order"]) + ) + self.assertEqual( + self.manifest["attestation"]["routes"], + ["xhigh", "max", "ultra"], + ) + self.assertEqual(self.manifest["attestation"]["invocations_per_route"], 1) + self.assertFalse(self.manifest["attestation"]["scored"]) + + def test_route_arms_freeze_single_agent_and_delegated_shapes(self) -> None: + arms = {item["arm_id"]: item for item in self.manifest["arms"]} + self.assertEqual(set(arms), {"xhigh", "max", "ultra"}) + for arm_id in ("xhigh", "max"): + self.assertEqual(arms[arm_id]["model_id"], "gpt-5.6-sol") + self.assertEqual(arms[arm_id]["reasoning_setting"], arm_id) + self.assertEqual(arms[arm_id]["orchestration"], "single_agent") + self.assertEqual(arms[arm_id]["expected_agent_count"], 1) + self.assertEqual(arms["ultra"]["model_id"], "gpt-5.6-sol") + self.assertEqual(arms["ultra"]["reasoning_setting"], "ultra") + self.assertEqual(arms["ultra"]["orchestration"], "automatic_delegation") + self.assertEqual(arms["ultra"]["expected_agent_count"], 4) + + def test_resource_ceiling_and_stop_rules_are_explicit(self) -> None: + budgets = self.manifest["budgets"] + self.assertEqual(budgets["resource_budget_regime"], "route_native_capacity") + self.assertEqual(budgets["total_inference_invocations_max"], 12) + self.assertEqual(budgets["attestation_invocations"], 3) + self.assertEqual(budgets["scored_invocations"], 9) + self.assertEqual(budgets["attempts_per_cell"], 1) + self.assertEqual(budgets["cell_timeout_seconds"], 600) + self.assertEqual(budgets["pilot_timeout_seconds"], 7200) + for field in ( + "usd_ceiling", + "token_ceiling", + "per_agent_usd_ceiling", + "per_agent_token_ceiling", + ): + self.assertEqual(budgets[field]["availability"], "unavailable") + self.assertNotIn("value", budgets[field]) + self.assertIn("no enforceable interface", budgets[field]["reason"]) + self.assertIn("usage_limit", budgets["stop_conditions"]) + self.assertIn("rate_limit", budgets["stop_conditions"]) + + def test_scored_task_files_and_manifest_hashes_reconcile(self) -> None: + manifest_tasks = {item["task_id"]: item for item in self.manifest["tasks"]} + self.assertEqual(set(manifest_tasks), set(SCORED_TASKS)) + for task_id, task_class in SCORED_TASKS.items(): + with self.subTest(task_id=task_id): + fixture = FIXTURE_ROOT / task_id + expected_files = { + "task.json", + "prompt.md", + "rubric.json", + "validate.py", + } + self.assertTrue( + expected_files <= {item.name for item in fixture.iterdir()} + ) + task = load_json(fixture / "task.json") + rubric = load_json(fixture / "rubric.json") + self.assertEqual(task["task_id"], task_id) + self.assertEqual(task["task_class"], task_class) + self.assertEqual(rubric["task_id"], task_id) + self.assertTrue(task["deterministic"]) + self.assertFalse(task["network_required"]) + self.assertFalse(task["third_party_dependencies"]) + self.assertEqual(task["path_scope"], "fixture_relative") + self.assertTrue(task["editable_paths"]) + self.assertTrue(task["protected_paths"]) + self.assertFalse( + set(task["editable_paths"]) & set(task["protected_paths"]) + ) + self.assertTrue( + all( + path.startswith("workspace/") for path in task["editable_paths"] + ) + ) + for required_path in ( + "prompt.md", + "rubric.json", + "task.json", + "validate.py", + ): + self.assertIn(required_path, task["protected_paths"]) + self.assertTrue( + PROTECTED_WORKSPACE_PATHS[task_id] <= set(task["protected_paths"]) + ) + self.assertTrue( + all( + command[0] == "/usr/local/bin/python3" + for command in task["validation_commands"] + ) + ) + self.assertEqual( + sum(item["points"] for item in rubric["criteria"]), 100 + ) + entry = manifest_tasks[task_id] + self.assertEqual( + entry["fixture_path"], fixture.relative_to(ROOT).as_posix() + ) + self.assertEqual(entry["tree_sha256"], fixture_tree_sha256(fixture)) + self.assertEqual( + entry["prompt_path"], + (fixture / "prompt.md").relative_to(ROOT).as_posix(), + ) + self.assertEqual( + entry["prompt_sha256"], file_sha256(fixture / "prompt.md") + ) + self.assertEqual( + entry["rubric_path"], + (fixture / "rubric.json").relative_to(ROOT).as_posix(), + ) + self.assertEqual( + entry["rubric_sha256"], file_sha256(fixture / "rubric.json") + ) + self.assertEqual( + entry["validation_commands"], task["validation_commands"] + ) + self.assertEqual(entry["success_criteria"], task["success_criteria"]) + + def test_attestation_is_unscored_route_neutral_and_hash_bound(self) -> None: + fixture = FIXTURE_ROOT / "attestation" + task = load_json(fixture / "task.json") + rubric = load_json(fixture / "rubric.json") + entry = self.manifest["attestation"] + self.assertFalse(task["scored"]) + self.assertFalse(rubric["scored"]) + self.assertEqual(rubric["total_points"], 0) + self.assertEqual(len(task["independent_checks"]), 3) + self.assertEqual(task["path_scope"], "fixture_relative") + self.assertEqual(task["editable_paths"], ["workspace/**"]) + self.assertFalse(set(task["editable_paths"]) & set(task["protected_paths"])) + self.assertEqual(entry["fixture_path"], fixture.relative_to(ROOT).as_posix()) + self.assertEqual(entry["tree_sha256"], fixture_tree_sha256(fixture)) + self.assertEqual( + entry["prompt_path"], (fixture / "prompt.md").relative_to(ROOT).as_posix() + ) + self.assertEqual(entry["prompt_sha256"], file_sha256(fixture / "prompt.md")) + prompt = (fixture / "prompt.md").read_text(encoding="utf-8").lower() + self.assertNotIn("xhigh", prompt) + self.assertNotIn("max", prompt) + self.assertNotIn("ultra", prompt) + self.assertIn("configured route", prompt) + + def test_prompts_are_route_neutral_and_safe(self) -> None: + prohibited = ("xhigh", "max", "ultra", "credential", "secret", "token") + for task_id in SCORED_TASKS: + with self.subTest(task_id=task_id): + prompt = (FIXTURE_ROOT / task_id / "prompt.md").read_text( + encoding="utf-8" + ) + lowered = prompt.lower() + for word in prohibited: + self.assertNotIn(word, lowered) + self.assertIn("Do not use network access", prompt) + self.assertIn("Modify only files inside the provided workspace", prompt) + + def test_frozen_fixtures_contain_no_generated_python_artifacts(self) -> None: + generated = [ + path + for path in FIXTURE_ROOT.rglob("*") + if path.name == "__pycache__" or path.suffix == ".pyc" + ] + self.assertEqual(generated, []) + + def test_preregistration_disclaims_inference_and_policy_update(self) -> None: + text = PREREGISTRATION_PATH.read_text(encoding="utf-8") + normalized = " ".join(text.lower().split()) + self.assertIn("exploratory", normalized) + self.assertIn("one run per cell", normalized) + self.assertIn("no policy update", normalized) + self.assertIn("raw outputs", normalized) + self.assertIn("outside the repository", normalized) + self.assertIn("no inference has been run", normalized) + + def test_frozen_envelope_and_raw_output_policy_are_explicit(self) -> None: + envelope = self.manifest["frozen_envelope"] + self.assertEqual( + envelope["fixture_tree_hash_algorithm"], + "sha256(sorted(relative_posix + NUL + file_sha256 + LF))", + ) + self.assertEqual(envelope["network_policy"], "offline_tools_only") + self.assertEqual(envelope["source_access_policy"], "fixture_only") + self.assertEqual(envelope["task_change_policy"], "invalidate_all_cells") + self.assertTrue(envelope["route_output_isolation"]) + raw_output = self.manifest["raw_output_policy"] + self.assertFalse(raw_output["tracked_in_git"]) + self.assertFalse(raw_output["publish_raw_outputs"]) + self.assertTrue(raw_output["content_review_required"]) + self.assertTrue(raw_output["published_receipts_sanitized"]) + self.assertEqual(raw_output["hash_algorithm"], "sha256") + + def test_frozen_initial_workspaces_do_not_already_pass(self) -> None: + for fixture_name in (*SCORED_TASKS, "attestation"): + with self.subTest(fixture=fixture_name): + fixture = FIXTURE_ROOT / fixture_name + result = run_validator(fixture, fixture / "workspace") + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + + def test_maintenance_and_attestation_validators_accept_known_good_edits( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + temp = Path(directory) + + maintenance = FIXTURE_ROOT / "maintenance-negative-control" + maintenance_workspace = temp / "maintenance" + shutil.copytree(maintenance / "workspace", maintenance_workspace) + config = load_json(maintenance_workspace / "service-config.json") + config["support_email"] = "help@example.invalid" + (maintenance_workspace / "service-config.json").write_text( + json.dumps(config, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + readme = (maintenance_workspace / "README.md").read_text(encoding="utf-8") + (maintenance_workspace / "README.md").write_text( + readme.replace("ops@example.invalid", "help@example.invalid"), + encoding="utf-8", + ) + result = run_validator(maintenance, maintenance_workspace) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + attestation = FIXTURE_ROOT / "attestation" + attestation_workspace = temp / "attestation" + shutil.copytree(attestation / "workspace", attestation_workspace) + (attestation_workspace / "status.txt").write_text( + "READY\n", encoding="utf-8" + ) + numbers = load_json(attestation_workspace / "numbers.json") + numbers["sum"] = sum(numbers["values"]) + (attestation_workspace / "numbers.json").write_text( + json.dumps(numbers, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + slug_module = attestation_workspace / "slug.py" + slug_module.write_text( + "def normalize_slug(value: str) -> str:\n" + " return '-'.join(value.strip().lower().split())\n", + encoding="utf-8", + ) + result = run_validator(attestation, attestation_workspace) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/run_route_evaluation.py b/tools/run_route_evaluation.py new file mode 100644 index 0000000..18b1e2b --- /dev/null +++ b/tools/run_route_evaluation.py @@ -0,0 +1,1207 @@ +#!/usr/bin/env python3 +"""Run one isolated, governed GPT-5.6 Sol route-evaluation cell.""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from datetime import datetime, timezone +import fnmatch +import hashlib +import json +import os +from pathlib import Path +import signal +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import time +from typing import Any + +try: + from tools import validate_route_evaluation as contract_validator +except ImportError: # Script execution places the adjacent tools directory first. + import validate_route_evaluation as contract_validator # type: ignore[no-redef] + + +MODEL_ID = "gpt-5.6-sol" +ROUTES = ("xhigh", "max", "ultra") +CLI_VERSION = "0.144.1" +CATALOG_SHA256 = "3ade3703c3a258c2a7780c3e8314e52489baaa2c8634df2de795029f9364f3eb" +MODEL_RECORD_SHA256 = ( + "ad2e4fc51d9e8c355ec5a3c94020525f01e3afc3476a6578f304bffad3363c04" +) +CATALOG_COMP_HASH = "3000" +IDENTIFIER = re.compile(r"^[a-z0-9][a-z0-9._-]*$") +LIMIT_PATTERN = re.compile( + r"(?:rate[ _-]?limit|usage[ _-]?limit|quota|too many requests|" + r"insufficient_quota|resource_exhausted|\b429\b)", + re.IGNORECASE, +) +SAFE_ENV_ALLOWLIST = { + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "PATH", + "PYTHONDONTWRITEBYTECODE", + "SHELL", + "TERM", + "TMPDIR", + "USER", +} + + +class RunnerError(ValueError): + """Raised when a run would violate the frozen harness contract.""" + + +@dataclass(frozen=True) +class RunConfig: + mode: str + route: str + pilot_id: str + cell_id: str + repetition: int + sequence: int + task_dir: Path + repo_root: Path + state_root: Path + temp_root: Path + timeout_seconds: float + invocation_ceiling: int + codex_executable: str + state_db: Path | None + manifest_path: Path | None + validation_commands: list[list[str]] | None = None + prompt_path: Path | None = None + workspace_source: Path | None = None + manifest_sha256: str | None = None + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _sha256_file(path: Path) -> str: + return _sha256_bytes(path.read_bytes()) + + +def _canonical_json(value: Any) -> bytes: + return ( + json.dumps(value, allow_nan=False, indent=2, sort_keys=True) + "\n" + ).encode("utf-8") + + +def _unavailable(reason: str) -> dict[str, Any]: + return {"availability": "unavailable", "reason": reason} + + +def _available(value: Any, source: str) -> dict[str, Any]: + return {"availability": "available", "value": value, "source": source} + + +def _candidate(values: list[str], label: str) -> dict[str, Any]: + unique = sorted(set(values)) + if not unique: + return _unavailable(f"No {label} was exposed in structured runtime events.") + if len(unique) > 1: + return { + "availability": "ambiguous", + "reason": f"Multiple {label} values were exposed.", + "candidates": unique, + } + return _available(unique[0], "runtime_jsonl") + + +def build_codex_command( + executable: str, + route: str, + workspace: Path, + last_message_path: Path | None = None, +) -> list[str]: + """Build the pinned Codex 0.144.1 treatment command.""" + + if route not in ROUTES: + raise RunnerError(f"unsupported route: {route!r}") + if last_message_path is None: + last_message_path = workspace.parent / "last-message.txt" + feature_flag = "--enable" if route == "ultra" else "--disable" + return [ + executable, + "exec", + "--ignore-user-config", + "--ignore-rules", + "--strict-config", + "--json", + "--color", + "never", + "--skip-git-repo-check", + "--output-last-message", + str(last_message_path), + "-s", + "workspace-write", + "-C", + str(workspace), + "-m", + MODEL_ID, + "-c", + f'model_reasoning_effort="{route}"', + "-c", + 'approval_policy="never"', + "-c", + "sandbox_workspace_write.network_access=false", + "-c", + 'web_search="disabled"', + feature_flag, + "multi_agent", + "-", + ] + + +def parse_jsonl(data: bytes) -> dict[str, Any]: + """Extract only structured execution metadata from Codex JSONL.""" + + thread_ids: list[str] = [] + models: list[str] = [] + efforts: list[str] = [] + usage_records: list[dict[str, int]] = [] + reroutes: list[dict[str, Any]] = [] + limit_events: list[dict[str, Any]] = [] + parse_errors = 0 + event_count = 0 + + for index, raw_line in enumerate(data.splitlines()): + if not raw_line.strip(): + continue + try: + event = json.loads(raw_line) + except (UnicodeDecodeError, json.JSONDecodeError): + parse_errors += 1 + continue + if not isinstance(event, dict): + parse_errors += 1 + continue + event_count += 1 + event_type = event.get("type") + if event_type == "thread.started" and isinstance(event.get("thread_id"), str): + thread_ids.append(event["thread_id"]) + + containers = [event] + for key in ("metadata", "config", "configuration", "thread"): + child = event.get(key) + if isinstance(child, dict): + containers.append(child) + for container in containers: + for key in ("model", "model_id", "effective_model"): + value = container.get(key) + if isinstance(value, str) and value: + models.append(value) + for key in ("reasoning_effort", "reasoning_setting", "effort"): + value = container.get(key) + if isinstance(value, str) and value in ROUTES: + efforts.append(value) + + usage = event.get("usage") + if isinstance(usage, dict): + selected = { + key: value + for key, value in usage.items() + if key + in { + "input_tokens", + "cached_input_tokens", + "output_tokens", + "total_tokens", + } + and isinstance(value, int) + and not isinstance(value, bool) + and value >= 0 + } + if selected: + usage_records.append(selected) + + rendered_type = event_type if isinstance(event_type, str) else "" + if "reroute" in rendered_type.lower() or event.get("rerouted") is True: + reroutes.append( + { + "event_index": index, + "event_type": rendered_type or "unspecified", + "from_model": event.get("from_model"), + "to_model": event.get("to_model") or event.get("effective_model"), + "reason": event.get("reason"), + } + ) + + if rendered_type.lower() in {"error", "turn.failed", "request.failed"}: + error = event.get("error") + error_text = json.dumps(error if error is not None else event, sort_keys=True) + if LIMIT_PATTERN.search(error_text): + limit_events.append( + {"event_index": index, "event_type": rendered_type or "error"} + ) + + if not usage_records: + usage: dict[str, Any] = _unavailable( + "No token usage was exposed in structured runtime events." + ) + elif len(usage_records) > 1: + usage = { + "availability": "ambiguous", + "reason": "Multiple usage records were exposed; aggregation semantics are unknown.", + "candidate_count": len(usage_records), + } + else: + usage = _available(usage_records[0], "runtime_jsonl") + + return { + "event_count": event_count, + "parse_error_count": parse_errors, + "thread_id": _candidate(thread_ids, "thread id"), + "model_id": _candidate(models, "effective model"), + "reasoning_effort": _candidate(efforts, "effective reasoning effort"), + "usage": usage, + "reroutes": reroutes, + "limit_events": limit_events, + } + + +def read_state_attestation(database: Path, thread_id: str) -> dict[str, Any]: + """Read only approved execution columns from the Codex state database.""" + + if not database.is_file(): + return _unavailable("Codex state database is unavailable.") + try: + uri = database.resolve().as_uri() + "?mode=ro" + with sqlite3.connect(uri, uri=True) as connection: + thread_columns = { + row[1] for row in connection.execute("PRAGMA table_info(threads)") + } + required = { + "id", + "model", + "reasoning_effort", + "source", + "cli_version", + "tokens_used", + } + if not required.issubset(thread_columns): + return _unavailable("Codex state database threads schema is incompatible.") + row = connection.execute( + "SELECT model, reasoning_effort, source, cli_version, tokens_used " + "FROM threads WHERE id = ?", + (thread_id,), + ).fetchone() + if row is None: + return _unavailable("Thread id was not found in the Codex state database.") + + edge_columns = { + item[1] + for item in connection.execute("PRAGMA table_info(thread_spawn_edges)") + } + children: list[dict[str, str]] = [] + if {"parent_thread_id", "child_thread_id", "status"}.issubset( + edge_columns + ): + child_rows = connection.execute( + "SELECT child_thread_id, status FROM thread_spawn_edges " + "WHERE parent_thread_id = ? ORDER BY child_thread_id", + (thread_id,), + ).fetchall() + children = [ + {"thread_id": child_id, "status": status} + for child_id, status in child_rows + if isinstance(child_id, str) and isinstance(status, str) + ] + except sqlite3.Error as error: + return _unavailable(f"Codex state database read failed: {error}") + + tokens = row[4] if isinstance(row[4], int) and not isinstance(row[4], bool) else None + return { + "availability": "available", + "source": "state_database", + "thread_id": thread_id, + "model_id": row[0], + "reasoning_effort": row[1], + "thread_source": row[2], + "cli_version": row[3], + "tokens_used": tokens, + "children": children, + } + + +def _tree_state(root: Path) -> tuple[str, dict[str, dict[str, Any]]]: + records: dict[str, dict[str, Any]] = {} + digest_input = bytearray() + for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()): + if path.is_symlink(): + raise RunnerError(f"workspace contains a prohibited symlink: {path}") + if not path.is_file() or ".git" in path.relative_to(root).parts: + continue + relative = path.relative_to(root).as_posix() + digest = _sha256_file(path) + records[relative] = {"sha256": digest, "size_bytes": path.stat().st_size} + digest_input.extend(relative.encode("utf-8")) + digest_input.extend(b"\0") + digest_input.extend(digest.encode("ascii")) + digest_input.extend(b"\n") + return _sha256_bytes(bytes(digest_input)), records + + +def _copy_workspace(source: Path, destination: Path) -> None: + if not source.is_dir(): + raise RunnerError(f"workspace source is not a directory: {source}") + for path in source.rglob("*"): + if path.is_symlink(): + raise RunnerError(f"workspace source contains a symlink: {path}") + shutil.copytree( + source, + destination, + ignore=shutil.ignore_patterns(".git", "__pycache__", ".pytest_cache"), + ) + + +def _changed_paths( + before: dict[str, dict[str, Any]], after: dict[str, dict[str, Any]] +) -> list[str]: + return [ + f"workspace/{path}" + for path in sorted(set(before) | set(after)) + if before.get(path) != after.get(path) + ] + + +def _validate_task(task_dir: Path) -> dict[str, Any]: + task_path = task_dir / "task.json" + try: + task = json.loads(task_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RunnerError(f"could not load task.json: {error}") from error + if not isinstance(task, dict): + raise RunnerError("task.json must contain an object") + commands = task.get("validation_commands") + if not isinstance(commands, list) or not commands or not all( + isinstance(command, list) + and command + and all(isinstance(argument, str) and argument for argument in command) + for command in commands + ): + raise RunnerError("validation_commands must contain non-empty argv arrays") + for command in commands: + if Path(command[0]).name in {"git", "gh", "codex"}: + raise RunnerError("validation commands may not invoke git, gh, or codex") + if task.get("path_scope") != "fixture_relative": + raise RunnerError("task path_scope must be fixture_relative") + allowed = task.get("editable_paths") + if not isinstance(allowed, list) or not allowed or not all( + isinstance(pattern, str) + and pattern.startswith("workspace/") + and ".." not in Path(pattern).parts + for pattern in allowed + ): + raise RunnerError("editable_paths must use fixture-relative workspace/ paths") + protected = task.get("protected_paths") + if not isinstance(protected, list) or not protected or not all( + isinstance(path, str) + and not Path(path).is_absolute() + and ".." not in Path(path).parts + for path in protected + ): + raise RunnerError("protected_paths must use safe fixture-relative paths") + return task + + +def _protected_hashes(task_dir: Path) -> dict[str, str]: + paths = [ + task_dir / name + for name in ("task.json", "prompt.md", "rubric.json", "validate.py") + if (task_dir / name).is_file() + ] + paths.extend(path for path in task_dir.rglob("AGENTS.md") if path.is_file()) + return { + path.relative_to(task_dir).as_posix(): _sha256_file(path) + for path in sorted(set(paths)) + } + + +def _under(path: Path, parent: Path) -> bool: + try: + path.resolve().relative_to(parent.resolve()) + except ValueError: + return False + return True + + +def _validate_paths(config: RunConfig) -> None: + if config.mode not in {"dry-run", "attestation", "scored"}: + raise RunnerError(f"unsupported mode: {config.mode!r}") + if config.route not in ROUTES: + raise RunnerError(f"unsupported route: {config.route!r}") + for label, value in (("pilot_id", config.pilot_id), ("cell_id", config.cell_id)): + if IDENTIFIER.fullmatch(value) is None: + raise RunnerError(f"invalid {label}: {value!r}") + if config.timeout_seconds <= 0: + raise RunnerError("timeout ceiling must be positive") + if config.mode != "dry-run" and config.invocation_ceiling < 1: + raise RunnerError("invocation ceiling prevents the requested model call") + if _under(config.state_root, config.repo_root): + raise RunnerError("state root must be outside the repository") + for ancestor in (config.temp_root, *config.temp_root.parents): + if (ancestor / "AGENTS.md").is_file(): + raise RunnerError( + "temporary workspaces must be outside ancestors containing AGENTS.md" + ) + + +def _sanitized_env() -> dict[str, str]: + env = { + key: value + for key, value in os.environ.items() + if key in SAFE_ENV_ALLOWLIST and isinstance(value, str) + } + env["PYTHONDONTWRITEBYTECODE"] = "1" + return env + + +def _kill_process_group(process: Any) -> None: + pid = getattr(process, "pid", None) + if isinstance(pid, int) and pid > 0: + try: + os.killpg(pid, signal.SIGKILL) + return + except (OSError, ProcessLookupError): + pass + process.kill() + + +def _require_clean_worktree(repo_root: Path) -> None: + status = subprocess.run( + ["git", "status", "--porcelain"], + cwd=repo_root, + check=False, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if status.returncode != 0: + detail = status.stderr.strip() or "git status failed" + raise RunnerError(f"could not verify clean worktree: {detail}") + if status.stdout.strip(): + raise RunnerError("source repository worktree is dirty; manifest requires clean_required") + + +def _run_validations( + commands: list[list[str]], + workspace: Path, + repo_root: Path, + bundle: Path, + deadline: float, + *, + run_command: Callable[..., subprocess.CompletedProcess[bytes]], + clock: Callable[[], float], +) -> tuple[list[dict[str, Any]], bool]: + results: list[dict[str, Any]] = [] + all_passed = True + validation_dir = bundle / "validation" + validation_dir.mkdir() + for index, frozen_command in enumerate(commands): + command = [argument.replace("{workspace}", str(workspace)) for argument in frozen_command] + remaining = deadline - clock() + if remaining <= 0: + results.append( + { + "index": index, + "argv": command, + "status": "timeout", + "reason": "Cell monotonic deadline expired before validation.", + } + ) + all_passed = False + break + started = clock() + try: + completed = run_command( + command, + cwd=repo_root, + env=_sanitized_env(), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=remaining, + shell=False, + ) + stdout = completed.stdout or b"" + stderr = completed.stderr or b"" + status = "passed" if completed.returncode == 0 else "failed" + returncode: int | None = completed.returncode + except subprocess.TimeoutExpired as error: + stdout = error.stdout or b"" + stderr = error.stderr or b"" + status = "timeout" + returncode = None + duration = max(0.0, clock() - started) + stdout_path = validation_dir / f"{index:02d}.stdout" + stderr_path = validation_dir / f"{index:02d}.stderr" + stdout_path.write_bytes(stdout) + stderr_path.write_bytes(stderr) + results.append( + { + "index": index, + "argv": command, + "status": status, + "exit_code": returncode, + "duration_seconds": duration, + "stdout_sha256": _sha256_bytes(stdout), + "stderr_sha256": _sha256_bytes(stderr), + } + ) + if status != "passed": + all_passed = False + break + return results, all_passed + + +def _verified_configuration( + parsed: dict[str, Any], state: dict[str, Any], route: str +) -> tuple[dict[str, Any], list[str]]: + invalid: list[str] = [] + runtime_thread = parsed["thread_id"] + runtime_model = parsed["model_id"] + runtime_effort = parsed["reasoning_effort"] + runtime_complete = all( + item.get("availability") == "available" + for item in (runtime_thread, runtime_model, runtime_effort) + ) + state_complete = ( + state.get("availability") == "available" + and isinstance(state.get("thread_id"), str) + and isinstance(state.get("model_id"), str) + and isinstance(state.get("reasoning_effort"), str) + ) + if runtime_complete: + result = { + "availability": "available", + "verification_source": "runtime_metadata", + "thread_id": runtime_thread["value"], + "model_id": runtime_model["value"], + "reasoning_setting": runtime_effort["value"], + "catalog_comp_hash": CATALOG_COMP_HASH, + } + elif state_complete: + result = { + "availability": "available", + "verification_source": "state_database", + "thread_id": state["thread_id"], + "model_id": state["model_id"], + "reasoning_setting": state["reasoning_effort"], + "catalog_comp_hash": CATALOG_COMP_HASH, + } + else: + result = _unavailable( + "Effective model, reasoning effort, or thread id was not exposed unambiguously." + ) + invalid.append("effective_configuration_unverified") + return result, invalid + + if result["model_id"] != MODEL_ID or result["reasoning_setting"] != route: + invalid.append("effective_configuration_mismatch") + if runtime_complete and state_complete and ( + runtime_model["value"] != state["model_id"] + or runtime_effort["value"] != state["reasoning_effort"] + or runtime_thread["value"] != state["thread_id"] + ): + invalid.append("runtime_state_attestation_conflict") + return result, invalid + + +def run_evaluation( + config: RunConfig, + *, + popen_factory: Callable[..., Any] = subprocess.Popen, + run_command: Callable[..., subprocess.CompletedProcess[bytes]] = subprocess.run, + clock: Callable[[], float] = time.monotonic, +) -> tuple[dict[str, Any], Path]: + """Run or preview one cell and return its raw private summary and bundle.""" + + _validate_paths(config) + task = _validate_task(config.task_dir) + scored = task.get("scored") is True + if config.mode == "attestation" and scored: + raise RunnerError("attestation mode requires an unscored task") + if config.mode == "scored" and not scored: + raise RunnerError("scored mode requires a scored task") + prompt_path = config.prompt_path or config.task_dir / "prompt.md" + workspace_source = config.workspace_source or config.task_dir / "workspace" + prompt_bytes = prompt_path.read_bytes() + commands = config.validation_commands or task["validation_commands"] + + bundle = config.state_root / config.pilot_id / config.cell_id + if bundle.exists(): + raise RunnerError("cell bundle already exists; one-attempt policy forbids retry") + bundle.mkdir(parents=True) + (bundle / "prompt.bin").write_bytes(prompt_bytes) + protected_before = _protected_hashes(config.task_dir) + started_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + started_monotonic = clock() + deadline = started_monotonic + config.timeout_seconds + temporary = Path(tempfile.mkdtemp(prefix="route-eval-", dir=config.temp_root)) + workspace = temporary / "workspace" + last_message_path = temporary / "last-message.txt" + + stdout = b"" + stderr = b"" + exit_code: int | None = None + timed_out = False + parsed = parse_jsonl(b"") + state_attestation: dict[str, Any] = _unavailable("No model invocation occurred.") + validation_results: list[dict[str, Any]] = [] + validation_passed = False + invalidation_reasons: list[str] = [] + actual_invocations = 0 + command = build_codex_command( + config.codex_executable, config.route, workspace, last_message_path + ) + + try: + _copy_workspace(workspace_source, workspace) + initial_tree_hash, initial_files = _tree_state(workspace) + + if config.mode != "dry-run": + actual_invocations = 1 + process = popen_factory( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=workspace, + env=_sanitized_env(), + shell=False, + start_new_session=True, + ) + try: + stdout, stderr = process.communicate( + input=prompt_bytes, + timeout=max(0.0, deadline - clock()), + ) + except subprocess.TimeoutExpired: + timed_out = True + _kill_process_group(process) + stdout, stderr = process.communicate() + exit_code = process.returncode + (bundle / "stdout.jsonl").write_bytes(stdout) + (bundle / "stderr.log").write_bytes(stderr) + if last_message_path.is_file(): + shutil.copy2(last_message_path, bundle / "last-message.txt") + parsed = parse_jsonl(stdout) + thread = parsed["thread_id"] + if ( + config.state_db is not None + and thread.get("availability") == "available" + ): + state_attestation = read_state_attestation( + config.state_db, thread["value"] + ) + else: + state_attestation = _unavailable( + "State database attestation was disabled or thread id unavailable." + ) + + stderr_limit = LIMIT_PATTERN.search( + stderr.decode("utf-8", errors="replace") + ) + limit_detected = bool(parsed["limit_events"] or stderr_limit) + if timed_out: + invalidation_reasons.append("cell_timeout") + elif limit_detected: + invalidation_reasons.append("usage_or_rate_limit") + elif exit_code != 0: + invalidation_reasons.append("codex_nonzero_exit") + else: + validation_results, validation_passed = _run_validations( + commands, + workspace, + config.repo_root, + bundle, + deadline, + run_command=run_command, + clock=clock, + ) + if not validation_passed: + invalidation_reasons.append("local_validation_failed") + + verified, verification_invalid = _verified_configuration( + parsed, state_attestation, config.route + ) + invalidation_reasons.extend(verification_invalid) + if parsed["reroutes"]: + invalidation_reasons.append("runtime_reroute_detected") + else: + limit_detected = False + verified = _unavailable("Dry-run performs no model invocation.") + + final_tree_hash, final_files = _tree_state(workspace) + changed = _changed_paths(initial_files, final_files) + allowed_patterns = task["editable_paths"] + protected_drift = [ + path + for path in changed + if not any(fnmatch.fnmatchcase(path, pattern) for pattern in allowed_patterns) + ] + if protected_drift: + invalidation_reasons.append("protected_workspace_drift") + git_paths = [ + f"workspace/{path.relative_to(workspace).as_posix()}" + for path in workspace.rglob(".git") + ] + if git_paths: + invalidation_reasons.append("git_metadata_created") + + protected_after = _protected_hashes(config.task_dir) + harness_drift = sorted( + path + for path in set(protected_before) | set(protected_after) + if protected_before.get(path) != protected_after.get(path) + ) + if harness_drift: + invalidation_reasons.append("protected_harness_drift") + + shutil.copytree( + workspace, + bundle / "workspace_final", + ignore=shutil.ignore_patterns(".git", "__pycache__", ".pytest_cache"), + ) + finally: + shutil.rmtree(temporary, ignore_errors=True) + + elapsed = max(0.0, clock() - started_monotonic) + invalidation_reasons = sorted(set(invalidation_reasons)) + if config.mode == "dry-run": + status = "dry_run" + validity = "inconclusive" + elif timed_out: + status = "timeout" + validity = "invalid" + elif exit_code != 0: + status = "failed" + validity = "invalid" + else: + status = "completed" + validity = "invalid" if invalidation_reasons else "valid" + + usage = parsed["usage"] + telemetry: dict[str, Any] = { + "input_tokens": _unavailable("Input-token telemetry was not exposed."), + "cached_input_tokens": _unavailable( + "Cached-input-token telemetry was not exposed." + ), + "output_tokens": _unavailable("Output-token telemetry was not exposed."), + "total_tokens": _unavailable("Total-token telemetry was not exposed."), + "cost": _unavailable( + "The runtime exposed no trustworthy per-run cost; no estimate was fabricated." + ), + "elapsed_wall_clock": { + "availability": "available", + "value": elapsed, + "unit": "seconds", + "measurement_kind": "harness_monotonic", + }, + } + if usage.get("availability") == "available": + for key, value in usage["value"].items(): + telemetry[key] = { + "availability": "available", + "value": value, + "unit": "tokens", + "measurement_kind": "runtime_counter", + "source": "runtime_jsonl", + } + summary = { + "schema_id": "route_evaluation_raw_bundle.v0.1", + "pilot_id": config.pilot_id, + "cell": { + "cell_id": config.cell_id, + "task_id": task.get("task_id"), + "route": config.route, + "mode": config.mode, + "repetition": config.repetition, + "sequence": config.sequence, + "manifest_sha256": config.manifest_sha256, + }, + "run": { + "started_at": started_at, + "completed_at": datetime.now(timezone.utc) + .isoformat() + .replace("+00:00", "Z"), + "status": status, + "validity": validity, + "invalidation_reasons": invalidation_reasons, + }, + "requested_configuration": { + "provider": "OpenAI", + "model_id": MODEL_ID, + "reasoning_setting": config.route, + "multi_agent_enabled": config.route == "ultra", + "orchestration": ( + "automatic_delegation" if config.route == "ultra" else "single_agent" + ), + "configuration_source": "pinned Codex CLI invocation", + }, + "verified_configuration": verified, + "reroutes": parsed["reroutes"], + "execution_units": state_attestation.get("children", []), + "telemetry": telemetry, + "limits": { + "invocation_ceiling": config.invocation_ceiling, + "actual_invocations": actual_invocations, + "timeout_seconds": config.timeout_seconds, + "rate_or_usage_limit_detected": limit_detected, + "structured_limit_events": parsed["limit_events"], + }, + "invocation": { + "argv": command, + "actual_count": actual_invocations, + "prompt_sha256": _sha256_bytes(prompt_bytes), + "prompt_size_bytes": len(prompt_bytes), + "stdin_transport": "exact_prompt_bytes", + "exit_code": exit_code, + "stdout_sha256": _sha256_bytes(stdout), + "stderr_sha256": _sha256_bytes(stderr), + "jsonl_event_count": parsed["event_count"], + "jsonl_parse_error_count": parsed["parse_error_count"], + }, + "state_attestation": state_attestation, + "validation_performed": validation_results, + "workspace": { + "isolation_root_kind": "temporary_outside_repository_rule_ancestors", + "initial_tree_sha256": initial_tree_hash, + "final_tree_sha256": final_tree_hash, + "changed_paths": changed, + "path_normalization": ( + "Actual workspace-relative paths are prefixed with workspace/ before " + "matching fixture-relative editable_paths." + ), + "source_access_limitation": ( + "Codex workspace-write sandbox prevents network access but does not " + "cryptographically confine read access to fixture-only bytes." + ), + "editable_paths": allowed_patterns, + "protected_drift": protected_drift, + "git_metadata_paths": git_paths, + "final_snapshot": "workspace_final", + }, + "protected_harness": { + "before_sha256": protected_before, + "after_sha256": protected_after, + "drift": harness_drift, + }, + "catalog_attestation": { + "codex_cli_version": CLI_VERSION, + "catalog_sha256": CATALOG_SHA256, + "model_record_sha256": MODEL_RECORD_SHA256, + "comp_hash": CATALOG_COMP_HASH, + }, + "harness": { + "runner_sha256": _sha256_file(Path(__file__)), + "git_writes_performed_by_harness": False, + "raw_bundle_is_private": True, + }, + } + (bundle / "summary.json").write_bytes(_canonical_json(summary)) + return summary, bundle + + +def _load_json_object(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RunnerError(f"could not load {path}: {error}") from error + if not isinstance(value, dict): + raise RunnerError(f"JSON document must be an object: {path}") + return value + + +def _summary_valid(state_root: Path, pilot_id: str, cell_id: str) -> bool: + summary_path = state_root / pilot_id / cell_id / "summary.json" + if not summary_path.is_file(): + return False + try: + summary = _load_json_object(summary_path) + except RunnerError: + return False + return summary.get("run", {}).get("validity") == "valid" + + +def _require_manifest_file_hash( + repo_root: Path, relative_path: Any, expected_sha256: Any, label: str +) -> Path: + if not isinstance(relative_path, str) or not isinstance(expected_sha256, str): + raise RunnerError(f"manifest {label} path/hash is invalid") + path = (repo_root / relative_path).resolve() + if not _under(path, repo_root) or not path.is_file(): + raise RunnerError(f"manifest {label} path is unavailable or escapes the repository") + if _sha256_file(path) != expected_sha256: + raise RunnerError(f"manifest {label} hash does not match local bytes") + return path + + +def _reconcile_manifest_harness( + manifest: dict[str, Any], repo_root: Path, task_dir: Path, task_entry: dict[str, Any] +) -> tuple[Path, Path]: + harness = manifest.get("harness") + if not isinstance(harness, dict): + raise RunnerError("manifest harness block is invalid") + runner_path = _require_manifest_file_hash( + repo_root, + harness.get("runner_path"), + harness.get("runner_sha256"), + "runner", + ) + if runner_path != Path(__file__).resolve(): + raise RunnerError("manifest runner path does not identify this runner") + _require_manifest_file_hash( + repo_root, + harness.get("validator_path"), + harness.get("validator_sha256"), + "validator", + ) + _require_manifest_file_hash( + repo_root, + harness.get("receipt_schema_path"), + harness.get("receipt_schema_sha256"), + "receipt schema", + ) + codex_path_value = harness.get("codex_executable_path") + if not isinstance(codex_path_value, str): + raise RunnerError("manifest Codex executable path is invalid") + codex_path = Path(codex_path_value).resolve() + if not codex_path.is_file() or _sha256_file(codex_path) != harness.get( + "codex_executable_sha256" + ): + raise RunnerError("manifest Codex executable hash does not match local bytes") + + catalog = manifest.get("catalog_attestation") + expected_catalog = { + "catalog_sha256": CATALOG_SHA256, + "model_record_sha256": MODEL_RECORD_SHA256, + "comp_hash": CATALOG_COMP_HASH, + "cli_version": CLI_VERSION, + } + if not isinstance(catalog, dict) or any( + catalog.get(key) != value for key, value in expected_catalog.items() + ): + raise RunnerError("manifest model catalog attestation drifted") + + tree_hash, _ = _tree_state(task_dir) + if tree_hash != task_entry.get("tree_sha256"): + raise RunnerError("manifest fixture tree hash does not match local bytes") + prompt_path = _require_manifest_file_hash( + repo_root, + task_entry.get("prompt_path"), + task_entry.get("prompt_sha256"), + "task prompt", + ) + rubric_path_value = task_entry.get("rubric_path") + rubric_hash_value = task_entry.get("rubric_sha256") + if rubric_path_value is not None or rubric_hash_value is not None: + _require_manifest_file_hash( + repo_root, + rubric_path_value, + rubric_hash_value, + "task rubric", + ) + return codex_path, prompt_path + + +def _config_from_manifest(args: argparse.Namespace) -> RunConfig: + manifest_path = args.manifest.resolve() + kind, errors = contract_validator.validate_path(manifest_path) + if kind != "route_evaluation_manifest.v0.1" or errors: + rendered = "; ".join(errors[:5]) if errors else f"unexpected kind {kind!r}" + raise RunnerError(f"manifest failed strict contract validation: {rendered}") + manifest = contract_validator.load_json_strict(manifest_path) + if not isinstance(manifest, dict): + raise RunnerError("validated manifest must be an object") + pilot_id = manifest.get("pilot_id") + if not isinstance(pilot_id, str) or IDENTIFIER.fullmatch(pilot_id) is None: + raise RunnerError("manifest pilot_id is invalid") + if manifest.get("base_model_id") != MODEL_ID: + raise RunnerError("manifest base_model_id does not match the pinned model") + budgets = manifest.get("budgets") + if not isinstance(budgets, dict) or budgets.get("attempts_per_cell") != 1: + raise RunnerError("manifest must preregister one attempt per cell") + timeout = budgets.get("cell_timeout_seconds") + if not isinstance(timeout, int) or isinstance(timeout, bool) or timeout <= 0: + raise RunnerError("manifest cell timeout is invalid") + route = args.route + mode = args.mode + preview = bool(getattr(args, "preview", False)) + repo_root = args.repo_root.resolve() + if mode != "dry-run" and not preview: + _require_clean_worktree(repo_root) + + if mode == "attestation": + attestation = manifest.get("attestation") + if not isinstance(attestation, dict) or attestation.get("scored") is not False: + raise RunnerError("manifest attestation block is invalid") + routes = attestation.get("routes") + if not isinstance(routes, list) or route not in routes: + raise RunnerError("route is not preregistered for attestation") + if args.task_id not in {"route-attestation", "attestation"}: + raise RunnerError("attestation mode requires the frozen attestation task") + fixture_path = attestation.get("fixture_path") + if not isinstance(fixture_path, str): + raise RunnerError("manifest attestation fixture path is invalid") + task_dir = repo_root / fixture_path + commands = attestation.get("validation_commands") + sequence = routes.index(route) + cell_id = f"attestation.{route}.r1" + if not preview: + for prior_route in routes[:sequence]: + if not _summary_valid( + args.state_root, pilot_id, f"attestation.{prior_route}.r1" + ): + raise RunnerError("attestation cells must run in preregistered order") + elif mode == "scored": + tasks = manifest.get("tasks") + if not isinstance(tasks, list): + raise RunnerError("manifest tasks are invalid") + task_entry = next( + ( + item + for item in tasks + if isinstance(item, dict) and item.get("task_id") == args.task_id + ), + None, + ) + if task_entry is None or not isinstance(task_entry.get("fixture_path"), str): + raise RunnerError("task is not preregistered in the manifest") + task_dir = repo_root / task_entry["fixture_path"] + commands = task_entry.get("validation_commands") + attestation = manifest.get("attestation", {}) + attestation_routes = attestation.get("routes", []) + if not preview: + for attestation_route in attestation_routes: + if not _summary_valid( + args.state_root, + pilot_id, + f"attestation.{attestation_route}.r1", + ): + raise RunnerError("all attestation cells must pass before scored cells") + order = manifest.get("execution_order") + if not isinstance(order, list): + raise RunnerError("manifest execution order is invalid") + ordered = sorted( + (entry for entry in order if isinstance(entry, dict)), + key=lambda entry: entry.get("sequence", -1), + ) + if preview: + pending = next( + ( + item + for item in ordered + if item.get("task_id") == args.task_id + and item.get("arm_id") == route + ), + None, + ) + else: + pending = next( + ( + item + for item in ordered + if isinstance(item.get("cell_id"), str) + and not _summary_valid(args.state_root, pilot_id, item["cell_id"]) + ), + None, + ) + if ( + pending is None + or pending.get("task_id") != args.task_id + or pending.get("arm_id") != route + ): + raise RunnerError("requested scored cell is not next in preregistered order") + cell_id = pending["cell_id"] + sequence = pending["sequence"] + else: + raise RunnerError("manifest execution supports attestation or scored mode") + + if not isinstance(commands, list): + raise RunnerError("manifest validation commands are invalid") + task = _validate_task(task_dir) + if task.get("task_id") != args.task_id: + raise RunnerError("task.json identity does not match the requested task") + if task.get("validation_commands") != commands: + raise RunnerError("task validation commands drift from the frozen manifest") + return RunConfig( + mode=mode, + route=route, + pilot_id=pilot_id, + cell_id=cell_id, + repetition=1, + sequence=sequence, + task_dir=task_dir, + repo_root=repo_root, + state_root=args.state_root, + temp_root=args.temp_root, + timeout_seconds=float(timeout), + invocation_ceiling=1, + codex_executable=manifest.get("harness", {}).get( + "codex_executable_path", "codex" + ), + state_db=None if args.no_state_db_attestation else args.state_db, + manifest_path=manifest_path, + validation_commands=commands, + manifest_sha256=_sha256_file(manifest_path), + ) + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--task-id", required=True) + parser.add_argument("--route", choices=ROUTES, required=True) + parser.add_argument("--mode", choices=("dry-run", "attestation", "scored"), required=True) + parser.add_argument("--state-root", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--temp-root", type=Path, default=Path("/tmp")) + parser.add_argument( + "--state-db", type=Path, default=Path.home() / ".codex" / "state_5.sqlite" + ) + parser.add_argument("--no-state-db-attestation", action="store_true") + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + try: + if args.mode == "dry-run": + # A dry-run resolves the frozen cell without checking or advancing order. + preview_args = argparse.Namespace(**vars(args)) + preview_args.mode = "attestation" if args.task_id in {"route-attestation", "attestation"} else "scored" + preview_args.preview = True + config = _config_from_manifest(preview_args) + config = RunConfig( + **{ + **config.__dict__, + "mode": "dry-run", + "cell_id": f"dry-run.{config.cell_id}.{time.time_ns()}", + "invocation_ceiling": 0, + } + ) + else: + config = _config_from_manifest(args) + summary, bundle = run_evaluation(config) + except RunnerError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 2 + print(json.dumps({"bundle": str(bundle), "status": summary["run"]["status"]})) + return 0 if summary["run"]["validity"] != "invalid" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/validate_route_evaluation.py b/tools/validate_route_evaluation.py new file mode 100644 index 0000000..8b4212f --- /dev/null +++ b/tools/validate_route_evaluation.py @@ -0,0 +1,1064 @@ +#!/usr/bin/env python3 +"""Deterministically validate route-evaluation manifests and receipts. + +The repository intentionally has no runtime JSON Schema dependency. This tool +implements the Draft 2020-12 subset used by the adjacent portable schemas, then +applies cross-field invariants that JSON Schema alone cannot express clearly. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from datetime import datetime, timedelta +import json +import math +from pathlib import Path, PurePosixPath +import re +import sys +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +META_SCHEMA = "https://json-schema.org/draft/2020-12/schema" +SCHEMA_CONFIG = { + "route_evaluation_manifest.v0.1": { + "version": "0.1", + "path": ROOT / "schemas" / "route_evaluation_manifest.v0.1.schema.json", + "uri": ( + "https://raw.githubusercontent.com/hummbl-dev/model-routing-as-code/main/" + "schemas/route_evaluation_manifest.v0.1.schema.json" + ), + }, + "route_evaluation_receipt.v0.2": { + "version": "0.2", + "path": ROOT / "schemas" / "route_evaluation_receipt.v0.2.schema.json", + "uri": ( + "https://raw.githubusercontent.com/hummbl-dev/model-routing-as-code/main/" + "schemas/route_evaluation_receipt.v0.2.schema.json" + ), + }, +} +ROUTES = ("xhigh", "max", "ultra") +REQUIRED_INVALIDATION_CONDITIONS = { + "effective-route-unverified", + "model-reroute", + "catalog-drift", + "fixture-drift", + "usage-limit", + "rate-limit", + "budget-bound", + "validation-contract-change", +} +EXPECTED_CATALOG_SHA256 = ( + "3ade3703c3a258c2a7780c3e8314e52489baaa2c8634df2de795029f9364f3eb" +) +EXPECTED_MODEL_RECORD_SHA256 = ( + "ad2e4fc51d9e8c355ec5a3c94020525f01e3afc3476a6578f304bffad3363c04" +) +EXPECTED_CODEX_SHA256 = ( + "c6eb747e4145ecb3bed2647dbd0f8464b190a5ccba964666ef7c98d4681a4a4c" +) +_RFC3339_DATE_TIME = re.compile( + r"^(?P[0-9]{4}-[0-9]{2}-[0-9]{2})" + r"[Tt](?P[01][0-9]|2[0-3]):(?P[0-5][0-9]):" + r"(?P[0-5][0-9]|60)(?:\.[0-9]+)?" + r"(?P[Zz]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])$" +) + + +class RouteEvaluationValidationError(ValueError): + """Raised when a JSON artifact cannot be parsed or routed to a schema.""" + + +def _json_type_matches(value: Any, expected: str) -> bool: + if expected == "object": + return isinstance(value, dict) + if expected == "array": + return isinstance(value, list) + if expected == "string": + return isinstance(value, str) + if expected == "boolean": + return isinstance(value, bool) + if expected == "null": + return value is None + if expected == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if expected == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + return False + + +def _display_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + +def _property_path(path: str, key: str) -> str: + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + return f"{path}.{key}" + return f"{path}[{json.dumps(key, ensure_ascii=False)}]" + + +def _resolve_local_ref(root_schema: dict[str, Any], ref: str) -> Any: + if not ref.startswith("#/"): + raise ValueError(f"only local JSON Pointer references are supported: {ref}") + current: Any = root_schema + for raw_part in ref[2:].split("/"): + part = raw_part.replace("~1", "/").replace("~0", "~") + if not isinstance(current, dict) or part not in current: + raise ValueError(f"unresolvable schema reference: {ref}") + current = current[part] + return current + + +def _parse_date_time(value: Any) -> datetime | None: + if not isinstance(value, str): + return None + match = _RFC3339_DATE_TIME.fullmatch(value) + if match is None: + return None + normalized = value[:-1] + "+00:00" if value[-1] in "Zz" else value + leap_second = match.group("second") == "60" + if leap_second: + if match.group("minute") != "59": + return None + start, end = match.span("second") + normalized = normalized[:start] + "59" + normalized[end:] + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + return None + if parsed.tzinfo is None or parsed.utcoffset() is None: + return None + return parsed + timedelta(seconds=1) if leap_second else parsed + + +def _validate_schema_subset( + value: Any, + schema: Any, + root_schema: dict[str, Any], + path: str, + errors: list[str], +) -> None: + if not isinstance(schema, dict): + errors.append(f"{path}: internal schema node must be an object") + return + + ref = schema.get("$ref") + if ref is not None: + if not isinstance(ref, str): + errors.append(f"{path}: internal schema $ref must be a string") + return + try: + target = _resolve_local_ref(root_schema, ref) + except ValueError as exc: + errors.append(f"{path}: internal schema error: {exc}") + return + _validate_schema_subset(value, target, root_schema, path, errors) + return + + branches = schema.get("oneOf") + if branches is not None: + if not isinstance(branches, list) or not branches: + errors.append(f"{path}: internal schema oneOf must be a non-empty array") + return + branch_errors: list[list[str]] = [] + for branch in branches: + candidate: list[str] = [] + _validate_schema_subset(value, branch, root_schema, path, candidate) + branch_errors.append(candidate) + matches = sum(not candidate for candidate in branch_errors) + if matches == 1: + return + if matches > 1: + errors.append(f"{path}: matches more than one permitted shape") + return + _, closest = min( + enumerate(branch_errors), key=lambda item: (len(item[1]), item[0]) + ) + errors.extend(closest) + return + + expected_type = schema.get("type") + if expected_type is not None: + expected_types = ( + expected_type if isinstance(expected_type, list) else [expected_type] + ) + if not all(isinstance(item, str) for item in expected_types): + errors.append(f"{path}: internal schema type must contain strings") + return + if not any(_json_type_matches(value, item) for item in expected_types): + errors.append(f"{path}: expected {' or '.join(expected_types)}") + return + + if "const" in schema and value != schema["const"]: + errors.append(f"{path}: must equal {_display_json(schema['const'])}") + enum = schema.get("enum") + if enum is not None and value not in enum: + errors.append( + f"{path}: must be one of " + ", ".join(_display_json(item) for item in enum) + ) + + if isinstance(value, dict): + required = schema.get("required", []) + if isinstance(required, list): + for key in sorted(item for item in required if isinstance(item, str)): + if key not in value: + errors.append( + f"{_property_path(path, key)}: required property missing" + ) + properties = schema.get("properties", {}) + if isinstance(properties, dict): + for key in sorted(value): + if key in properties: + _validate_schema_subset( + value[key], + properties[key], + root_schema, + _property_path(path, key), + errors, + ) + elif schema.get("additionalProperties") is False: + errors.append( + f"{_property_path(path, key)}: additional property not allowed" + ) + + if isinstance(value, list): + minimum = schema.get("minItems") + maximum = schema.get("maxItems") + if isinstance(minimum, int) and len(value) < minimum: + errors.append(f"{path}: must contain at least {minimum} item(s)") + if isinstance(maximum, int) and len(value) > maximum: + errors.append(f"{path}: must contain at most {maximum} item(s)") + if schema.get("uniqueItems") is True: + serialized = [ + json.dumps(item, sort_keys=True, separators=(",", ":")) + for item in value + ] + if len(serialized) != len(set(serialized)): + errors.append(f"{path}: array items must be unique") + item_schema = schema.get("items") + if isinstance(item_schema, dict): + for index, item in enumerate(value): + _validate_schema_subset( + item, item_schema, root_schema, f"{path}[{index}]", errors + ) + + if isinstance(value, str): + minimum_length = schema.get("minLength") + if isinstance(minimum_length, int) and len(value) < minimum_length: + errors.append( + f"{path}: must contain at least {minimum_length} character(s)" + ) + pattern = schema.get("pattern") + if isinstance(pattern, str) and re.search(pattern, value) is None: + errors.append(f"{path}: must match pattern {pattern!r}") + if schema.get("format") == "date-time" and _parse_date_time(value) is None: + errors.append(f"{path}: must be an RFC 3339 date-time with timezone") + + if isinstance(value, (int, float)) and not isinstance(value, bool): + if isinstance(value, float) and not math.isfinite(value): + errors.append(f"{path}: number must be finite") + return + minimum = schema.get("minimum") + maximum = schema.get("maximum") + if isinstance(minimum, (int, float)) and value < minimum: + errors.append(f"{path}: must be greater than or equal to {minimum}") + if isinstance(maximum, (int, float)) and value > maximum: + errors.append(f"{path}: must be less than or equal to {maximum}") + + +def _is_safe_relative_path(value: Any) -> bool: + if not isinstance(value, str) or not value or "\\" in value: + return False + path = PurePosixPath(value) + return not path.is_absolute() and ".." not in path.parts and "." not in path.parts + + +def _require_safe_path(value: Any, path: str, errors: list[str]) -> None: + if not _is_safe_relative_path(value): + errors.append(f"{path}: must be a safe repository-relative path") + + +def _require_unique_ids( + values: Any, key: str, path: str, errors: list[str] +) -> set[str]: + result: set[str] = set() + if not isinstance(values, list): + return result + for index, value in enumerate(values): + if not isinstance(value, dict) or not isinstance(value.get(key), str): + continue + identifier = value[key] + if identifier in result: + errors.append(f"{path}[{index}].{key}: duplicate {key} {identifier!r}") + result.add(identifier) + return result + + +def _available_value(value: Any) -> int | float | None: + if not isinstance(value, dict) or value.get("availability") != "available": + return None + result = value.get("value") + if isinstance(result, (int, float)) and not isinstance(result, bool): + return result + return None + + +def _validate_unavailable(value: Any, path: str, errors: list[str]) -> None: + if not isinstance(value, dict) or value.get("availability") != "unavailable": + return + forbidden = sorted( + key + for key in ("value", "unit", "measurement_kind", "enabled", "details") + if key in value + ) + if forbidden: + errors.append( + f"{path}: availability is unavailable; omit " + ", ".join(forbidden) + ) + + +def _validate_manifest(manifest: dict[str, Any], errors: list[str]) -> None: + if manifest.get("base_model_id") not in (None, "gpt-5.6-sol"): + errors.append("$.base_model_id: must remain gpt-5.6-sol") + + source = manifest.get("source_repository") + if isinstance(source, dict) and source.get("clean_required") is not True: + errors.append("$.source_repository.clean_required: must be true") + + harness = manifest.get("harness") + if isinstance(harness, dict): + expected_paths = { + "runner_path": "tools/run_route_evaluation.py", + "validator_path": "tools/validate_route_evaluation.py", + "receipt_schema_path": "schemas/route_evaluation_receipt.v0.2.schema.json", + } + for key, expected in expected_paths.items(): + if harness.get(key) != expected: + errors.append(f"$.harness.{key}: must equal {expected!r}") + if harness.get("codex_executable_sha256") not in (None, EXPECTED_CODEX_SHA256): + errors.append("$.harness.codex_executable_sha256: local binary hash drift") + for key in ( + "runner_sha256", + "validator_sha256", + "receipt_schema_sha256", + "codex_executable_sha256", + ): + if harness.get(key) == "0" * 64: + errors.append(f"$.harness.{key}: sha256 must not be a placeholder") + for key in ("runner_path", "validator_path", "receipt_schema_path"): + if key in harness: + _require_safe_path(harness[key], f"$.harness.{key}", errors) + + catalog = manifest.get("catalog_attestation") + if isinstance(catalog, dict): + if catalog.get("catalog_sha256") not in (None, EXPECTED_CATALOG_SHA256): + errors.append("$.catalog_attestation.catalog_sha256: catalog drift") + if catalog.get("model_record_sha256") not in ( + None, + EXPECTED_MODEL_RECORD_SHA256, + ): + errors.append( + "$.catalog_attestation.model_record_sha256: model record drift" + ) + settings = catalog.get("supported_reasoning_settings") + if isinstance(settings, list) and set(settings) != set(ROUTES): + errors.append( + "$.catalog_attestation.supported_reasoning_settings: must contain " + "exactly xhigh, max, and ultra" + ) + + envelope = manifest.get("frozen_envelope") + if isinstance(envelope, dict): + flags = envelope.get("common_cli_flags") + required_flags = { + "--ignore-user-config", + "--ignore-rules", + "--strict-config", + "--json", + "--output-last-message", + } + if isinstance(flags, list) and not required_flags.issubset(set(flags)): + errors.append( + "$.frozen_envelope.common_cli_flags: must freeze config/rules, strict " + "parsing, JSON events, and output-last-message" + ) + if isinstance(flags, list) and "--ephemeral" in flags: + errors.append( + "$.frozen_envelope.common_cli_flags: --ephemeral is prohibited because " + "route attestation requires persisted thread state" + ) + + attestation = manifest.get("attestation") + if isinstance(attestation, dict): + for key in ("fixture_path", "prompt_path"): + if key in attestation: + _require_safe_path(attestation[key], f"$.attestation.{key}", errors) + if isinstance(attestation.get("routes"), list) and set( + attestation["routes"] + ) != set(ROUTES): + errors.append( + "$.attestation.routes: must contain exactly xhigh, max, ultra" + ) + + tasks = manifest.get("tasks") + task_ids = _require_unique_ids(tasks, "task_id", "$.tasks", errors) + if isinstance(tasks, list): + classes = { + task.get("task_class") + for task in tasks + if isinstance(task, dict) and isinstance(task.get("task_class"), str) + } + expected_classes = { + "maintenance_negative_control", + "tightly_coupled", + "decomposable", + } + if classes and classes != expected_classes: + errors.append("$.tasks: must contain one task from each frozen task class") + for index, task in enumerate(tasks): + if not isinstance(task, dict): + continue + for key in ("fixture_path", "prompt_path", "rubric_path"): + if key in task: + _require_safe_path(task[key], f"$.tasks[{index}].{key}", errors) + fixture = task.get("fixture_path") + if isinstance(fixture, str): + prefix = fixture.rstrip("/") + "/" + for key in ("prompt_path", "rubric_path"): + value = task.get(key) + if isinstance(value, str) and not value.startswith(prefix): + errors.append( + f"$.tasks[{index}].{key}: must be contained by fixture_path" + ) + + arms = manifest.get("arms") + arm_ids = _require_unique_ids(arms, "arm_id", "$.arms", errors) + if isinstance(arms, list): + if arm_ids and arm_ids != set(ROUTES): + errors.append("$.arms: must contain exactly xhigh, max, and ultra") + for index, arm in enumerate(arms): + if not isinstance(arm, dict) or arm.get("arm_id") not in ROUTES: + continue + route = arm["arm_id"] + if arm.get("reasoning_setting") != route: + errors.append( + f"$.arms[{index}]: {route} reasoning_setting must equal arm_id" + ) + expected = ( + ("automatic_delegation", 4) if route == "ultra" else ("single_agent", 1) + ) + if (arm.get("orchestration"), arm.get("expected_agent_count")) != expected: + errors.append( + f"$.arms[{index}]: {route} orchestration/agent-count contract drift" + ) + if arm.get("model_id") not in (None, manifest.get("base_model_id")): + errors.append(f"$.arms[{index}].model_id: must equal base_model_id") + + order = manifest.get("execution_order") + if isinstance(order, list): + sequences = [cell.get("sequence") for cell in order if isinstance(cell, dict)] + if sequences != list(range(1, len(order) + 1)): + errors.append( + "$.execution_order: sequence must be contiguous in listed order" + ) + _require_unique_ids(order, "cell_id", "$.execution_order", errors) + seen: set[tuple[Any, Any, Any]] = set() + for index, cell in enumerate(order): + if not isinstance(cell, dict): + continue + identity = ( + cell.get("task_id"), + cell.get("arm_id"), + cell.get("repetition"), + ) + if identity in seen: + errors.append( + f"$.execution_order[{index}]: duplicate task/arm/repetition cell" + ) + seen.add(identity) + if task_ids and cell.get("task_id") not in task_ids: + errors.append(f"$.execution_order[{index}].task_id: unknown task") + if arm_ids and cell.get("arm_id") not in arm_ids: + errors.append(f"$.execution_order[{index}].arm_id: unknown arm") + expected_pairs = { + (task_id, arm_id, 1) for task_id in task_ids for arm_id in arm_ids + } + if task_ids and arm_ids and seen != expected_pairs: + errors.append( + "$.execution_order: must contain every task/arm pair exactly once" + ) + + budgets = manifest.get("budgets") + if isinstance(budgets, dict): + scored = budgets.get("scored_invocations") + attested = budgets.get("attestation_invocations") + total = budgets.get("total_inference_invocations_max") + expected_scored = len(order) if isinstance(order, list) else None + expected_attested = None + if isinstance(attestation, dict): + routes = attestation.get("routes") + per_route = attestation.get("invocations_per_route") + if isinstance(routes, list) and isinstance(per_route, int): + expected_attested = len(routes) * per_route + if expected_scored is not None and scored != expected_scored: + errors.append( + "$.budgets.scored_invocations: must equal execution_order length" + ) + if expected_attested is not None and attested != expected_attested: + errors.append( + "$.budgets.attestation_invocations: must equal routes * invocations_per_route" + ) + if ( + isinstance(scored, int) + and isinstance(attested, int) + and total != scored + attested + ): + errors.append( + "$.budgets.total_inference_invocations_max: must equal attestation + scored invocations" + ) + for field, unit in ( + ("usd_ceiling", "USD"), + ("token_ceiling", "tokens"), + ("per_agent_usd_ceiling", "USD"), + ("per_agent_token_ceiling", "tokens"), + ): + ceiling = budgets.get(field) + _validate_unavailable(ceiling, f"$.budgets.{field}", errors) + if ( + isinstance(ceiling, dict) + and ceiling.get("availability") == "available" + and ceiling.get("unit") != unit + ): + errors.append(f"$.budgets.{field}.unit: must be {unit!r}") + cell_timeout = budgets.get("cell_timeout_seconds") + pilot_timeout = budgets.get("pilot_timeout_seconds") + if ( + isinstance(cell_timeout, int) + and isinstance(pilot_timeout, int) + and isinstance(scored, int) + and pilot_timeout < cell_timeout * scored + ): + errors.append( + "$.budgets.pilot_timeout_seconds: must cover all scored cell timeouts" + ) + + conditions = manifest.get("invalidation_conditions") + if isinstance(conditions, list): + missing = sorted(REQUIRED_INVALIDATION_CONDITIONS - set(conditions)) + for condition in missing: + errors.append( + f"$.invalidation_conditions: required condition {condition!r} missing" + ) + + +def _parse_available_timestamp(value: Any) -> datetime | None: + if not isinstance(value, dict) or value.get("availability") != "available": + return None + return _parse_date_time(value.get("value")) + + +def _parse_available_int(value: Any) -> int | None: + if not isinstance(value, dict) or value.get("availability") != "available": + return None + result = value.get("value") + if isinstance(result, int) and not isinstance(result, bool): + return result + return None + + +def _validate_token_arithmetic(telemetry: Any, path: str, errors: list[str]) -> None: + if not isinstance(telemetry, dict): + return + for field in ( + "aggregate_cost", + "cost", + "input_tokens", + "output_tokens", + "total_tokens", + "elapsed_wall_clock_time", + ): + if field in telemetry: + _validate_unavailable(telemetry[field], f"{path}.{field}", errors) + input_tokens = _available_value(telemetry.get("input_tokens")) + output_tokens = _available_value(telemetry.get("output_tokens")) + total_tokens = _available_value(telemetry.get("total_tokens")) + if ( + input_tokens is not None + and output_tokens is not None + and total_tokens is not None + and total_tokens != input_tokens + output_tokens + ): + errors.append( + f"{path}.total_tokens.value: must equal input_tokens + output_tokens" + ) + + +def _validate_receipt(receipt: dict[str, Any], errors: list[str]) -> None: + recorded_at = _parse_date_time(receipt.get("recorded_at")) + cell = receipt.get("cell") + route = cell.get("route") if isinstance(cell, dict) else None + if isinstance(cell, dict) and cell.get("arm_id") != route: + errors.append("$.cell.arm_id: must equal $.cell.route") + + run = receipt.get("run") + started = None + completed = None + validity = None + invalidation_reasons: list[Any] = [] + if isinstance(run, dict): + started = _parse_date_time(run.get("started_at")) + completed = _parse_available_timestamp(run.get("completed_at")) + validity = run.get("validity") + reasons = run.get("invalidation_reasons") + if isinstance(reasons, list): + invalidation_reasons = reasons + if run.get("status") == "completed" and completed is None: + errors.append( + "$.run.completed_at: completed run requires an available timestamp" + ) + if started is not None and completed is not None and completed < started: + errors.append("$.run.completed_at.value: must not precede started_at") + if ( + recorded_at is not None + and completed is not None + and recorded_at < completed + ): + errors.append("$.recorded_at: must not precede run completion") + if validity == "valid" and invalidation_reasons: + errors.append( + "$.run.invalidation_reasons: valid run must have no invalidation reasons" + ) + + requested = receipt.get("requested_configuration") + if isinstance(requested, dict) and route in ROUTES: + if requested.get("reasoning_setting") != route: + errors.append( + "$.requested_configuration.reasoning_setting: must equal cell route" + ) + expected = ( + (True, "automatic_delegation", 4) + if route == "ultra" + else (False, "single_agent", 1) + ) + actual = ( + requested.get("multi_agent_enabled"), + requested.get("orchestration"), + requested.get("expected_agent_count"), + ) + if actual != expected: + errors.append( + f"$.requested_configuration: {route} configuration contract drift" + ) + + verified = receipt.get("verified_configuration") + if isinstance(verified, dict): + _validate_unavailable(verified, "$.verified_configuration", errors) + if verified.get("availability") == "unavailable" and validity == "valid": + errors.append( + "$.verified_configuration: effective route unavailable; run cannot be valid" + ) + if verified.get("availability") == "available": + mismatch = False + if isinstance(requested, dict): + mismatch = verified.get("model_id") != requested.get( + "model_id" + ) or verified.get("reasoning_setting") != requested.get( + "reasoning_setting" + ) + if mismatch: + errors.append( + "$.verified_configuration: effective route differs from requested route" + ) + if validity == "valid": + errors.append( + "$.run.validity: effective route mismatch cannot be valid" + ) + + reroutes = receipt.get("reroutes") + if isinstance(reroutes, list) and reroutes: + if validity == "valid": + errors.append( + "$.run.validity: model-reroute must invalidate or make the run inconclusive" + ) + if "model-reroute" not in invalidation_reasons: + errors.append("$.run.invalidation_reasons: model-reroute must be recorded") + for index, reroute in enumerate(reroutes): + if not isinstance(reroute, dict): + continue + occurred = _parse_date_time(reroute.get("occurred_at")) + if started is not None and occurred is not None and occurred < started: + errors.append(f"$.reroutes[{index}].occurred_at: precedes run start") + if completed is not None and occurred is not None and occurred > completed: + errors.append( + f"$.reroutes[{index}].occurred_at: follows run completion" + ) + + units = receipt.get("execution_units") + unit_ids = _require_unique_ids(units, "unit_id", "$.execution_units", errors) + roles: list[Any] = [] + if isinstance(units, list): + roles = [unit.get("role") for unit in units if isinstance(unit, dict)] + if units and roles.count("root") != 1: + errors.append("$.execution_units: must contain exactly one root unit") + for index, unit in enumerate(units): + if not isinstance(unit, dict): + continue + parent = unit.get("parent_unit_id") + if unit.get("role") == "root" and parent is not None: + errors.append( + f"$.execution_units[{index}].parent_unit_id: root must use null" + ) + if unit.get("role") == "delegate" and ( + not isinstance(parent, str) or parent not in unit_ids + ): + errors.append( + f"$.execution_units[{index}].parent_unit_id: delegate parent must exist" + ) + _validate_unavailable( + unit.get("thread_id"), + f"$.execution_units[{index}].thread_id", + errors, + ) + _validate_unavailable( + unit.get("started_at"), + f"$.execution_units[{index}].started_at", + errors, + ) + unit_started = _parse_available_timestamp(unit.get("started_at")) + unit_completed = _parse_available_timestamp(unit.get("completed_at")) + if ( + unit.get("final_state") in {"completed", "failed", "interrupted"} + and unit_completed is None + ): + errors.append( + f"$.execution_units[{index}].completed_at: terminal unit requires timestamp" + ) + if ( + unit_started is not None + and unit_completed is not None + and unit_completed < unit_started + ): + errors.append( + f"$.execution_units[{index}].completed_at.value: precedes unit start" + ) + if ( + started is not None + and unit_started is not None + and unit_started < started + ): + errors.append( + f"$.execution_units[{index}].started_at: precedes run start" + ) + if ( + completed is not None + and unit_completed is not None + and unit_completed > completed + ): + errors.append( + f"$.execution_units[{index}].completed_at.value: follows run completion" + ) + _validate_token_arithmetic( + unit.get("telemetry"), f"$.execution_units[{index}].telemetry", errors + ) + + delegation = receipt.get("delegation") + if isinstance(delegation, dict): + _validate_unavailable(delegation, "$.delegation", errors) + available = delegation.get("availability") == "available" + if route in {"xhigh", "max"} and available: + concurrent = delegation.get("max_concurrent_agents") + concurrent_ok = concurrent == {"availability": "available", "value": 1} + if ( + delegation.get("enabled") is not False + or delegation.get("observed_agent_count") != 1 + or not concurrent_ok + or "delegate" in roles + ): + errors.append( + f"$.delegation: {route} must not fabricate automatic delegation" + ) + if route == "ultra" and validity == "valid": + if ( + not available + or delegation.get("enabled") is not True + or not isinstance(delegation.get("observed_agent_count"), int) + or delegation.get("observed_agent_count", 0) < 2 + or "delegate" not in roles + ): + errors.append( + "$.delegation: a valid ultra run requires observed automatic delegation" + ) + elif isinstance(units, list) and len(units) != delegation.get( + "observed_agent_count" + ): + errors.append( + "$.delegation.observed_agent_count: must equal recorded execution units" + ) + if isinstance(requested, dict) and delegation.get( + "observed_agent_count" + ) != requested.get("expected_agent_count"): + errors.append( + "$.delegation.observed_agent_count: valid run must match preregistered expectation" + ) + if available: + observed = delegation.get("observed_agent_count") + concurrent = _parse_available_int(delegation.get("max_concurrent_agents")) + if ( + isinstance(observed, int) + and concurrent is not None + and concurrent > observed + ): + errors.append( + "$.delegation.max_concurrent_agents: cannot exceed observed_agent_count" + ) + + telemetry = receipt.get("telemetry") + _validate_token_arithmetic(telemetry, "$.telemetry", errors) + if isinstance(telemetry, dict): + method = telemetry.get("token_aggregation_method") + token_values = [ + _available_value(telemetry.get(name)) + for name in ("input_tokens", "output_tokens", "total_tokens") + ] + if method == "unavailable" and any(value is not None for value in token_values): + errors.append( + "$.telemetry.token_aggregation_method: cannot be unavailable when token telemetry is available" + ) + total = _available_value(telemetry.get("total_tokens")) + if method == "sum_execution_units": + unit_totals = [ + _available_value(unit.get("telemetry", {}).get("total_tokens")) + for unit in units or [] + if isinstance(unit, dict) + ] + if not unit_totals or any(value is None for value in unit_totals): + errors.append( + "$.telemetry.token_aggregation_method: sum_execution_units requires every unit total" + ) + elif total != sum(value for value in unit_totals if value is not None): + errors.append( + "$.telemetry.total_tokens.value: must equal summed execution-unit totals" + ) + elapsed = _available_value(telemetry.get("elapsed_wall_clock_time")) + if ( + elapsed is not None + and started is not None + and completed is not None + and abs(elapsed - (completed - started).total_seconds()) > 1.0 + ): + errors.append( + "$.telemetry.elapsed_wall_clock_time.value: differs from run timestamps by more than one second" + ) + + quality = receipt.get("quality_observations") + if isinstance(quality, dict): + for name in ("duplicate_findings", "synthesis_omissions"): + assessment = quality.get(name) + path = f"$.quality_observations.{name}" + _validate_unavailable(assessment, path, errors) + if ( + isinstance(assessment, dict) + and assessment.get("availability") == "available" + ): + details = assessment.get("details") + if assessment.get("assessment") == "observed" and details == []: + errors.append( + f"{path}.details: observed requires at least one detail" + ) + if assessment.get("assessment") == "none_observed" and details not in ( + None, + [], + ): + errors.append( + f"{path}.details: none_observed requires an empty array" + ) + + outcome = receipt.get("outcome") + adjudication = receipt.get("adjudication") + validation_records = receipt.get("validation_performed") + if isinstance(outcome, dict): + _validate_unavailable( + outcome.get("rubric_score"), "$.outcome.rubric_score", errors + ) + if validity == "valid": + if outcome.get("task_success") != "pass": + errors.append("$.outcome.task_success: valid run must pass") + if outcome.get("validation_failures_after_adversarial_review") != 0: + errors.append( + "$.outcome.validation_failures_after_adversarial_review: valid run requires zero" + ) + if isinstance(validation_records, list): + failure_counts = { + phase: sum( + 1 + for record in validation_records + if isinstance(record, dict) + and record.get("phase") == phase + and record.get("status") == "failed" + ) + for phase in ("pre_adversarial", "post_adversarial") + } + expected_fields = { + "pre_adversarial": "validation_failures_before_adversarial_review", + "post_adversarial": "validation_failures_after_adversarial_review", + } + for phase, field in expected_fields.items(): + if outcome.get(field) != failure_counts[phase]: + errors.append( + f"$.outcome.{field}: must equal failed {phase} validation records" + ) + if isinstance(cell, dict) and isinstance(adjudication, dict): + if cell.get("evaluation_kind") == "scored" and validity == "valid": + if adjudication.get("status") not in { + "blinded_complete", + "unblinded_complete", + }: + errors.append( + "$.adjudication.status: valid scored run requires completion" + ) + if adjudication.get("decision") != "pass": + errors.append("$.adjudication.decision: valid scored run requires pass") + if cell.get("evaluation_kind") == "attestation" and adjudication.get( + "status" + ) not in {"not_required", "not_performed"}: + errors.append("$.adjudication.status: attestation must not be scored") + + deviations = receipt.get("deviations") + _require_unique_ids(deviations, "deviation_id", "$.deviations", errors) + if isinstance(deviations, list) and validity == "valid": + if any( + isinstance(deviation, dict) and deviation.get("invalidates_run") is True + for deviation in deviations + ): + errors.append( + "$.run.validity: cannot be valid when a deviation invalidates_run" + ) + + raw_artifacts = receipt.get("raw_artifacts") + _require_unique_ids(raw_artifacts, "artifact_id", "$.raw_artifacts", errors) + if isinstance(raw_artifacts, list): + for index, artifact in enumerate(raw_artifacts): + if not isinstance(artifact, dict): + continue + if ( + artifact.get("contains_sensitive_data") is True + and artifact.get("publication_status") == "public_safe" + ): + errors.append( + f"$.raw_artifacts[{index}].publication_status: sensitive artifact cannot be public_safe" + ) + + +def validate_document(document: Any, schema: Any) -> list[str]: + """Validate an in-memory manifest or receipt and return sorted errors.""" + + errors: list[str] = [] + if not isinstance(schema, dict): + return ["$: schema must be an object"] + if schema.get("$schema") != META_SCHEMA: + errors.append(f"$schema: must equal {META_SCHEMA!r}") + schema_id = document.get("schema_id") if isinstance(document, dict) else None + config = SCHEMA_CONFIG.get(schema_id) + if config is None: + errors.append("$.schema_id: unsupported route-evaluation schema identity") + else: + if document.get("schema_version") != config["version"]: + errors.append( + f"$.schema_version: expected {config['version']!r} for {schema_id}" + ) + if schema.get("$id") != config["uri"]: + errors.append(f"$id: unexpected schema URI for {schema_id}") + + _validate_schema_subset(document, schema, schema, "$", errors) + if isinstance(document, dict): + if schema_id == "route_evaluation_manifest.v0.1": + _validate_manifest(document, errors) + elif schema_id == "route_evaluation_receipt.v0.2": + _validate_receipt(document, errors) + return sorted(set(errors)) + + +def _reject_constant(value: str) -> Any: + raise RouteEvaluationValidationError( + f"non-standard JSON numeric constant is prohibited: {value}" + ) + + +def _parse_float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise RouteEvaluationValidationError(f"overflowed JSON number: {value}") + return result + + +def _object_without_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise RouteEvaluationValidationError(f"duplicate JSON object key: {key!r}") + result[key] = value + return result + + +def load_json_strict(path: Path) -> Any: + try: + return json.loads( + path.read_text(encoding="utf-8"), + parse_constant=_reject_constant, + parse_float=_parse_float, + object_pairs_hook=_object_without_duplicates, + ) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise RouteEvaluationValidationError(f"could not parse {path}: {exc}") from exc + + +def _schema_for_document(document: Any, explicit_schema: Path | None) -> dict[str, Any]: + if explicit_schema is not None: + schema = load_json_strict(explicit_schema) + else: + schema_id = document.get("schema_id") if isinstance(document, dict) else None + config = SCHEMA_CONFIG.get(schema_id) + if config is None: + raise RouteEvaluationValidationError( + "document has no supported route-evaluation schema_id" + ) + schema = load_json_strict(config["path"]) + if not isinstance(schema, dict): + raise RouteEvaluationValidationError("schema document must be an object") + return schema + + +def validate_path(path: Path, schema_path: Path | None = None) -> tuple[str, list[str]]: + document = load_json_strict(path) + schema = _schema_for_document(document, schema_path) + schema_id = document.get("schema_id") if isinstance(document, dict) else "unknown" + return str(schema_id), validate_document(document, schema) + + +def _parse_args(argv: Sequence[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("document", type=Path) + parser.add_argument("--schema", type=Path, default=None) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parse_args(argv) + try: + schema_id, errors = validate_path(args.document, args.schema) + except RouteEvaluationValidationError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + if errors: + print(f"INVALID {schema_id}", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print(f"VALID {schema_id}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())