From ca511818f49931f49565172d086269dfc4298bd7 Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Wed, 12 Aug 2026 22:00:19 +0200 Subject: [PATCH 1/3] test(cyborg): reproduce the public CAGE-2 protocol --- ...-cage2-protocol-reproduction-guardrails.md | 293 ++ docs/index.md | 1 + docs/researcher-command.md | 65 + mkdocs.yml | 1 + src/raes_adapters/cli.py | 80 + src/raes_adapters/cyborg/driver.py | 37 +- src/raes_adapters/cyborg/evaluator.py | 73 + src/raes_adapters/cyborg/orchestrator.py | 24 + .../cyborg/participant_runtime.py | 26 +- src/raes_adapters/cyborg/provisioner.py | 33 +- src/raes_adapters/cyborg/reproduction.py | 2369 +++++++++++++++++ src/raes_adapters/cyborg/researcher.py | 130 +- src/raes_adapters/cyborg/scenario.py | 58 +- src/raes_adapters/cyborg/target.py | 5 + tests/test_cyborg_execution.py | 119 +- tests/test_cyborg_reproduction.py | 564 ++++ tests/test_cyborg_scenario2.py | 31 +- tests/test_researcher_cli.py | 44 +- 18 files changed, 3939 insertions(+), 14 deletions(-) create mode 100644 docs/decisions/cyborg-cage2-protocol-reproduction-guardrails.md create mode 100644 src/raes_adapters/cyborg/reproduction.py create mode 100644 tests/test_cyborg_reproduction.py diff --git a/docs/decisions/cyborg-cage2-protocol-reproduction-guardrails.md b/docs/decisions/cyborg-cage2-protocol-reproduction-guardrails.md new file mode 100644 index 0000000..e2df759 --- /dev/null +++ b/docs/decisions/cyborg-cage2-protocol-reproduction-guardrails.md @@ -0,0 +1,293 @@ +# CAGE-2 protocol-reproduction guardrails + +GitHub issue #22 is the authority for this requirement-free deliverable. This +note fixes the ownership, evidence, security, persistence, statistical, and +claim boundaries for CAGE Challenge 2 protocol reproduction. It +defines no RAES contract, adapter protocol, reusable study schema, or +implementation plan. + +## Generated evidence, not repository content + +The adapter writes a reproduction bundle only to the fresh, explicit output +directory selected by the operator. Generated study data is not checked into +this repository, packaged in the distribution, or bound to a downstream +repository or filesystem location. Research projects and archives independently +own retention and publication after validating an exported bundle. + +Each completed output is immutable. Changed inputs, methods, tolerances, +agents, or results require a fresh output directory; a rerun never rewrites an +accepted result. The issue-named JSON files are operational indexes and +projections for the generated study, not new portable schema families. They do +not carry adapter-defined schema names or versions; the frozen revision and +declaration digest identify the one study selection. Embedded RAES artifacts +remain exact published models and are validated by their owners. + +Generated evidence is not `ControlPlaneStore` state, an adapter database, a +replay cache, package data, or a release artifact. The adapter records no +retention destination or path reference. + +## Freeze distinct source facts before execution + +Do not flatten the cited sources into one alleged upstream protocol. The +checked-in qualified `evaluation.py` row records 100 episodes, trial lengths +30/50/100, the three Red variants, and no seed binding. The paper reports 1,000 +episodes for every Red-agent/trial-length pairing, while the qualified README +evidence records the validation seed statement `random.seed(153)`. The frozen +oracle must identify which source supports each matrix or method field and must +retain the disagreements and `loss-evaluation-seed-unbound`. + +In particular, a seed value is not a seed policy. `protocol.json` must freeze +the stream owner, API, initialization scope (study, condition, attempt, or +episode), ordering, and reset behavior supported by the source. Repeating +per-episode `RunControls(seed=153)` is not equivalent to one batch-level +`random.seed(153)` call. Parallel execution is permitted only when the frozen +stream/order semantics remain unchanged. Simulator, Python, NumPy, wrapper, +Red, Green, Blue-policy, and evaluator streams stay distinct; unsupported or +unbound streams weaken the result. + +Every selected Blue baseline requires immutable implementation identity, +source/artifact digests, admitted participant manifest/selection/configuration, +and any model/configuration bytes it needs. The current packaged Blue Sleep +selection is the only complete external Blue implementation in this repository. +The qualified `BlueLoadAgent` fallback has no bound model artifact and must not +be presented as the published policy or silently replaced by Sleep. A baseline +without a complete public closure is predeclared as excluded, with the +exclusion retained in the schedule and source ledger. + +`source-ledger.json` is a frozen projection joining the existing qualification, +mapping ledger, loss disclosures, pack/scenario identities, and additional +paper/leaderboard sources needed by this study. It must cite and digest those +owners rather than copy `cage2-source-ledger.jsonl` into a second mapping +authority. Source bytes are not fetched during aggregation, and a URL without +a revision/content digest is not an immutable input. + +## Compose the canonical owners + +| Concern | Canonical incumbent and required boundary | +| --- | --- | +| Source admission | `qualification.json`, `CAGE2_SOURCE_26CE1C1`, `load_qualification()`, `load_source_ledger()`, `load_loss_disclosures()`, `validate_all()`, `verify_selected_cyborg_source()`, and the qualified source-tree/file/module-origin checks. Extend the evidence selection for genuinely new primary sources; do not add a second qualification record or permissive source loader. | +| Authored scenario and pack | `src/raes_adapters/cyborg/scenario/cage2-scenario2.sdl.yaml`, its byte-identical packaged `cage2-research` snapshot, the RAES module lock and canonical digest APIs, `raes-env-packs` validation, and the associated-artifact manifest/set digest. Use the same authored bytes for both execution lanes; do not author a reproduction-only SDL. | +| Experiment and participant controls | Published `ExperimentSpecModel`, `ExperimentTaskModel`, episode/red-variant/stochastic-control models, participant manifest/selection/configuration models and validators, `RunControls`, and the researcher CLI's exact digest/binding admission. Matrix membership is frozen data, not target defaults or environment configuration. | +| Target and execution | `create_cyborg_target()`, `_normalized_config()`, `_validate_selection()`, `RuntimeManager`, `RuntimeControlPlane`, `CyborgProvisioner`, `CyborgOrchestrator`, `CyborgParticipantRuntime`, `ReferenceTimeRuntime`, and `SourceInstalledCyborgDriver`. Preserve one admitted Blue action to one serialized aggregate turn, quarantine after an unprojectable mutation, and verified cleanup. | +| Evaluation and evidence | `CyborgEvaluator`, its committed per-step reward evidence and cumulative Blue-score `ExperimentDerivedMeasureModel`, RAES objective/truth/evidence/capture models, and `validate_experiment_run_against_task()`. Upstream score, RAES objective truth, participant outcome, conformance, and equivalence remain separate. | +| Run/study projection | `raes_adapters.cyborg.researcher`, `_researcher_support.build_archival_run()`, the published run/study models, and their cross-artifact validators. Canonical RAES records are emitted for evidence-complete runs; the operational run inventory retains every attempt, including those that cannot be sealed as a successful RAES run. | +| Diagnostics and failures | RAES `Diagnostic`/`DiagnosticModel`, `diagnostic_model()`, `diagnostic_payload()`, the CLI's stable `_CommandFailure` exit mapping, `redact_native_value()`, and `bounded_context_label()`. Add no exception hierarchy or reproduction error envelope. | +| Persistence | Exclusive mode-`0700` output reservation, safe run labels, `raes_operations.run_artifacts.atomic_write_json_artifact()`, relative artifact names, SHA-256 inventory entries, and inventory-last sealing. The atomic writer prevents partial files but does not prevent replacement; exclusive directory creation and frozen-revision policy supply that guarantee. | +| Conformance | The exact published `BackendConformanceReport`, report projector/writer, CybORG local diagnostics, declared weaknesses, and `_conformance_support` evidence-closure helpers. Conformance may be cited but is never copied into an equivalence case family or used as a score-similarity result. | +| Reference path | `raes_processor.reference.ReferenceProcessor.realize()`, the live manifest, published planning/runtime models, and `RuntimeControlPlane` gates. Report exactly the authored/contract/control surfaces this path exercises; do not relabel the reference processor as a CybORG simulator or synthesize native state and outcomes. | +| Repository workflow | The single `pyproject.toml`/`uv.lock`, existing `raes-adapters` console entry point, `_verification_envs()`, tests, clean-install distribution probes, strict docs/policy gates, `.github/workflows/ci.yml`, `PR Gate`, and canonical Nox graph. Extend these incumbents instead of adding a second command package, workflow, or verification graph. | + +The current CLI's native batch is useful for admission, execution, projection, +and sealing, but its fail-fast list comprehension and success-only run records +are not the issue #22 scheduler. Reuse its lower-level owners and persistence +rules; do not call a failed batch complete or duplicate the entire CLI. The +current hermetic conformance suite's seed 153 is likewise conformance evidence, +not a native evaluation run. + +## Immutable schedule and attempt lifecycle + +Separate a logical schedule slot from an execution attempt. The frozen slot +identifies the condition, expected ordinal, source-backed controls, and +predeclared eligibility. Every attempt has a never-reused run id and points to +one slot. A retry receives a new run id and an explicit predecessor/slot link; +the original terminal record remains immutable. + +Every scheduled slot and attempt ends in exactly one issue-defined terminal +disposition: + +- `valid`: execution, evidence validation, and required cleanup completed, and + the attempt is eligible under the frozen inclusion rule; +- `invalid`: execution returned but protocol, evidence, artifact, or validation + requirements were not satisfied; +- `failed`: execution, timeout, process, projection, persistence, or required + cleanup failed before valid evidence could be sealed; or +- `excluded`: a predeclared source/apparatus/eligibility rule says the slot is + not run or not eligible. + +These dispositions are not participant outcomes or score quality. Failure and +exclusion reasons use closed, stable codes and evidence references, never +native exception text. The retry limit and deterministic attempt-selection rule +(for example, first valid attempt in attempt order) are frozen before execution +so retries cannot become score-based optional stopping. Aggregates retain counts +for every disposition and never impute, delete, or overwrite an attempt. + +Because the source execution unit is one retained native session per condition, +a failed condition session terminally fails that attempt for every slot in the +condition. The one predeclared retry restores the condition's ordered-stream +checkpoint and allocates a new predecessor-linked run id to every slot. If the +retry also fails, its terminal records remain and the scheduler still attempts +every later independent condition; it never turns one condition failure into +fabricated attempts for the rest of the matrix. + +Reserve the per-attempt directory before native construction and seal its +inventory last. An interrupted or unsealed attempt is failed, not absent. A +parent process crash must be recoverable by comparing the immutable schedule to +the retained attempt directories; recovery creates new attempt ids and does not +resume or append to an old directory. + +The source evaluator retains one native session per condition and resets it +between episodes, so the adapter does the same without reseeding the +study-scoped random stream. Each slot is still a distinct RAES experiment run: +its portable snapshot begins from the pristine realized baseline and never +inherits lifecycle, action, observation, or evaluation history from the prior +slot. Native-session continuity and portable evidence isolation are separate +invariants. + +## One evidence closure and one aggregation implementation + +The eligible scalar score is the validated cumulative Blue reward already +projected by `CyborgEvaluator` from committed per-step RAES evidence. Do not +recompute it from a native reward vector, stdout, a private evaluator object, or +free-form summary. The aggregate reader accepts only sealed per-attempt +inventories, validates embedded RAES records, verifies file digests and +slot/attempt joins, and selects attempts through the frozen rule. + +Each run retains the exact transitive RAES evidence closure referenced by its +published derived measures. Unreferenced runtime telemetry is not research +evidence for this protocol and is omitted rather than bulk-archived; the +offline verifier rejects both a missing referenced record and an unrelated +extra record. This keeps the public bundle tied to the declared measurements +without weakening per-step component and score traceability. + +`protocol.json` must predeclare the estimator, two-sided 95% confidence-interval +method, sample-variance convention, finite-value policy, critical-value method, +rounding/display rule, minimum eligible count, retry/inclusion treatment, +tolerances, and exact/bounded/failed reproduction criteria. A paper table that +prints intervals is not evidence of an unstated formula. If the published +method cannot be established, the study method is explicitly adapter-study +methodology and the tier/claim is weakened rather than reverse-engineered from +reported numbers. + +One pure aggregation function owns both initial `aggregates.json` production +and clean-environment recomputation. It reads only immutable retained run +evidence plus the frozen protocol, uses a deterministic ordering and JSON +projection, and reports eligible `n` plus every terminal-disposition count per +condition. Tests must recompute after shuffled directory enumeration and reject +missing, duplicate, non-finite, digest-drifted, cross-condition, or post-freeze +inputs. Do not maintain separate “runner” and “validator” formulas. + +Published and observed tables remain separately identified. Numerical equality +or tolerance membership may support only the predeclared exact/bounded/failed +reproduction classification. It does not establish contract conformance, +state/observation equivalence, backend identity, deterministic replay, or +scientific reproducibility. + +## Equivalence tiers and reference-backend separation + +`tiers.json` contains exactly the issue's six review tiers: `authored-source`, +`contract`, `execution-control`, `state/observation`, `outcome/evaluation`, and +`disclosure`. `disclosure` is an issue-local evidence-completeness tier; do not +add it to the existing source-ledger loss vocabulary merely to reuse its parser. + +For each tier, keep CybORG and the RAES reference path as separately identified +subjects with one `pass`, `fail`, or `weakened` result, the predeclared rule, +and machine-resolvable evidence/loss references. Unsupported reference-runtime +facts produce `weakened`; they are never filled with CybORG observations, +published scores, zeros, or synthetic success. A failure in one subject does +not overwrite the other, and there is no bundle-wide boolean `passed`. + +Known losses constrain attainable results. The native-observation boundary, +Remove misreport, wrapper-cutoff ambiguity, evaluator seed gap, and unbound +Blue policy remain visible even when a run succeeds. Acceptance requires honest +tier results, not six passes. The report's final claim is the strongest claim +supported by the weakest applicable evidence closure and must repeat the +explicit non-claims from issue #22. + +## Security and whole-repository path + +The design passes every applicable layer below before evidence is published: + +| Layer | Required treatment | +| --- | --- | +| Authentication and authorization | This is a local, offline command and adds no controller, route, daemon, remote caller, or credential flow. Participant authority still comes from admitted participant selections and runtime/control-plane gates. Any later service must use `create_control_plane_app()` with `ControlPlaneSecurityConfig.strict_defaults`; no adapter endpoint is permitted. | +| Secret and policy-material surface | The public study requires no secret. Do not read secret stores or `.env`, add token options, resolve private provider artifacts, or expose credentials, learned sessions, governed entropy, private model weights, or secret references. Only reviewed public baseline bytes may be selected. | +| CLI and configuration shape | Keep the closed `argparse` choices, safe-label rules, exact matrix/source ids, and `_normalized_config()` / `_validate_selection()` authority. Operational output path and bounded worker count cannot change source identity, matrix membership, seed scope, participant implementation, profile/corpus roots, driver class, or claim criteria. Unknown keys and hidden defaults fail before native construction. | +| Pack, source, and static parsers | Run `raes-env-packs` validation and digest joins; strict duplicate-key JSON; RAES SDL parsing, lock/import verification, instantiation, compilation, and canonical digest; published experiment/participant models; qualification/source-ledger checks; and confined source/module-origin verification. No permissive YAML/JSON parser, floating ref, arbitrary import path, runtime fetch, or working-directory discovery is admission. | +| Runtime validation | Keep `RuntimeTarget` manifest/component/signature checks, `RuntimeManager`/`RuntimeControlPlane` plan and snapshot-transition gates, workflow/time/participant/action/observation/evaluation validators, terminal separation, session quarantine, and verified cleanup. Backend checks add only the selected CybORG mapping. | +| Native and disclosure boundary | Native state, hidden truth, observations, sessions, action ids/classes, reward mappings/vectors, raw logs, object representations, random states, and full tracebacks stay private. Build allowlisted RAES projections before serialization; hashing or redacting an arbitrary native dump does not make it portable. | +| Error envelopes | Use validated RAES diagnostics and the existing stable command exit classes/messages. Do not serialize Pydantic rejected values, exception strings/types/causes, stdout/stderr, host paths, argv/environment mappings, or traceback text. Cleanup failure is retained independently from the primary failure. | +| OS/process exposure | Resolve all inputs beneath explicit roots; reject absolute/traversing paths, existing targets, symlinks, and unsafe run ids; use mode-`0700` directories and argument-vector invocation. If attempts use subprocess isolation, reuse the qualification tool's explicit working directory, environment allowlist, cleared `PYTHONPATH`, safe-path behavior, timeout, bounded output handling, and no shell interpolation or runtime network access. Native stdout/stderr is suppressed, not retained. | +| Logging and observability | Portable observability is validated RAES evidence, stable diagnostics, terminal dispositions, counts, digests, provenance, limitations, and relative inventories. Logging is limited to safe operation names, public ids/pointers, counts, and dispositions; never log inputs, paths, actions, rewards, native/random state, policy material, or exceptions. | +| Persistence and publication | Validate before atomic write, reserve roots exclusively, retain failed attempts, use only relative references, seal inventories last, and verify all digests before aggregation. Public export requires leakage scans over actual persisted bytes. No overwrite/force/resume/append path exists. | +| Packaging and repository policy | Keep one distribution and lock. Generated run evidence remains outside the checkout and distribution. Keep individual exported records bounded rather than creating a monolithic native/result dump, and keep the existing CI/Nox/policy/docs/distribution graph authoritative. | + +`environment.json` records bounded platform, Python, distribution, qualified +source, adapter, RAES, pack, study/protocol, and method identities. It is not an +environment-variable or argv dump and must not include hostnames, usernames, +absolute paths, caches, installed-package metadata unrelated to the study, or +provider logs. + +## Extension seam and workflow boundary + +The extension seam is one immutable reproduction selection: frozen revision, +source/pack/scenario/participant identities, an ordered matrix of condition and +slot ids, seed-allocation policy, failure/attempt-selection policy, statistical +method, tolerances, and claim rules. Backend-local executor callables remain in +`raes_adapters.cyborg`; this is not a new `raes_adapters.base` protocol. A new +public Blue baseline or future source revision adds a new admitted selection and +fresh output without editing an accepted protocol or hard-coding another +branch for trial length or Red policy. + +Tests use reduced cardinalities and injected/hostile drivers but exercise the +same scheduler, terminal-disposition, retry, sealing, aggregation, tier, and +leakage code. They do not relabel hermetic evidence as native. The complete +native matrix is an explicit qualified-source acceptance run, not a PR-unit-test +loop or the existing weekly hermetic conformance suite. If CI later runs it, use +the existing workflow and label it as non-PR native reproduction; a new required +job must be added to `PR Gate`. + +Before acceptance, run relevant targeted tests, the qualified native command in +the admitted environment, clean-environment aggregate recomputation, and the +canonical graph: + +```shell +uv tool run --from 'nox[uv]==2026.4.10' nox -s verify -- --skip-requirement +``` + +The committed pack currently remains `status: built` with +`reference_triangle: false`. Reproduction evidence may support a later reviewed +status change, but implementation must not mark the pack golden or mutate that +publication claim automatically. + +## Gotchas and anti-patterns + +- Do not conflate the 100-episode checked-in evaluator default, the paper's + 1,000-episode evaluation, and the frozen adapter study. +- Do not turn `random.seed(153)` into 1,000 identical per-episode resets or call + it complete stochastic control. +- Do not use the current external Blue Sleep selection as a substitute for + `BlueLoadAgent`, submitted agents, or missing model weights. +- Do not drop failed slots, overwrite retries, choose the best attempt, or tune + exclusions/tolerances/CI methods after seeing scores. +- Do not aggregate unsealed summaries or native output; use validated per-run + derived measures and inventories. +- Do not equate score similarity with conformance, objective truth, semantic + equivalence, or backend identity. +- Do not turn tier results into conformance cases, source-ledger dispositions, + manifest capabilities, or one aggregate pass flag. +- Do not let the reference path fabricate CybORG observations, stochastic + transitions, rewards, terminal causes, or cleanup facts. +- Do not duplicate the source ledger, RAES models/validators, evaluator score + calculation, CLI failure hierarchy, artifact writer, inventory logic, + conformance projector, or CI workflow. +- Do not check generated study output into this repository or encode a + downstream repository or filesystem location in adapter code or metadata. + +## Non-goals and implementation boundaries + +- No deterministic replay, backend identity, exact native state/observation + equality, or scientific reproducibility claim beyond retained evidence. +- No training, leaderboard service, submitted-agent reconstruction, private + model acquisition, web UI, API, daemon, scheduler service, or authentication + system. +- No new RAES/environment-pack SDL, schema, vocabulary, profile, fixture + corpus, evidence type, metric meaning, backend protocol, diagnostic envelope, + exception hierarchy, repository/store, or policy gate. +- No runtime clone/download, floating source, editable install, unpublished + behavioral patch, arbitrary plugin/import, or second distribution/lock. +- No native hidden truth, observation payload, action id/class, reward vector, + raw log, credential, object representation, host path, environment/argv dump, + or full traceback in the public bundle. +- No change to the qualified source defects, loss disclosures, upstream score + calculation, authored Scenario2 semantics, or existing conformance claim merely + to make the reproduction pass. diff --git a/docs/index.md b/docs/index.md index 49c7ea2..2661ba6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,6 +29,7 @@ semantic and protocol authority. - [CybORG/CAGE-2 provisioner and backend-manifest guardrails](decisions/cyborg-cage2-provisioner-manifest-guardrails.md) - [CybORG conformance-composition guardrails](decisions/cyborg-conformance-guardrails.md) - [CybORG researcher run-and-evidence command guardrails](decisions/cyborg-researcher-command-guardrails.md) +- [CybORG/CAGE-2 protocol-reproduction guardrails](decisions/cyborg-cage2-protocol-reproduction-guardrails.md) - [CybORG/CAGE-2 downstream environment-pack guardrails](decisions/cyborg-cage2-example-pack-guardrails.md) - [NASim researcher run-and-evidence command guardrails](decisions/nasim-researcher-command-guardrails.md) - [CyberBattleSim qualification guardrails](decisions/cyberbattlesim-qualification-guardrails.md) diff --git a/docs/researcher-command.md b/docs/researcher-command.md index 3a448c9..3a63e66 100644 --- a/docs/researcher-command.md +++ b/docs/researcher-command.md @@ -102,6 +102,71 @@ records, bounded diagnostics, summaries, machine/software inventory, and the final relative inventory. Study mode also retains a validated RAES collection record; it adds no scientific or equivalence claim. +## Reproduce the public CAGE-2 evaluation protocol + +The frozen issue-22 path is separate from the two-seed example above. It +predeclares and executes the complete 3 trial-length × 3 Red-policy × 1,000 +episode matrix, using one study-scoped Python random stream initialized with +seed 153. It compares the observed RAES-mediated cumulative Blue reward against +the published CCS Sleeper table and retains every scheduled slot, terminal +disposition, confidence-interval input, and evidence join: + +```shell +raes-adapters reproduce \ + --phase declare \ + --source-root \ + --output cage2-reproduction-declaration + +raes-adapters reproduce \ + --phase run \ + --source-root \ + --output cage2-study-output +``` + +The run phase requires the separately installed qualified CybORG source and +does not fetch it. The declaration freezes source disagreements as losses: the +checked-in evaluator says 100 episodes and binds no seed, whereas the public +validation description says 1,000 episodes and `random.seed(153)`. The observed +Blue implementation is a public Sleep-policy behavioral reconstruction; it is +not represented as the unavailable submitted-agent artifact. + +Each valid run contains compressed, deterministic JSON with the complete +referenced RAES evidence records, derived measures, objective and proposition +results, and archival run record. Compression changes storage only. Every file +and transitive inventory is SHA-256 sealed, every compressed record is parsed +and model-validated offline, and no native observations, state, reward vectors, +logs, or random state are published. + +Within a condition, the source-native session is retained and reset so the +single seeded stream continues exactly as declared. Portable RAES snapshots are +not retained across those resets: each scheduled slot starts a fresh archival +run and cannot inherit another slot's action or evidence history. + +Recompute the aggregates and six separately-subjected evidence tiers in an +installed environment that has no CybORG source and needs no network access: + +```shell +raes-adapters reproduce \ + --phase verify \ + --bundle \ + --output cage2-recomputed +``` + +The result supports an exact, bounded, or failed outcome-reproduction finding +according to the predeclared method. It deliberately does not infer state or +observation equivalence, backend identity, deterministic replay, conformance, +or submitted-agent identity from score similarity. Those limits make the +export usable as honest apparatus and outcome-reproduction evidence rather +than a proof-of-concept smoke run. The adapter does not prescribe where a +research project retains or publishes that export. + +Because the public submitted CCS Sleeper artifact is unavailable, a matching +result is specifically a behavioral-baseline outcome reproduction: it shows +that the RAES-mediated Scenario 2 apparatus reproduces the published sleeping +baseline distribution within the frozen rule. It is not a replication of an +unavailable submitted implementation, and `tiers.json` keeps that distinction +in the strongest supported claim. + ## Exit status | Exit | Meaning | diff --git a/mkdocs.yml b/mkdocs.yml index 4b60949..aea5571 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -49,6 +49,7 @@ nav: - CybORG reward, objective, and outcome projection guardrails: decisions/cyborg-evaluation-projection-guardrails.md - CybORG conformance-composition guardrails: decisions/cyborg-conformance-guardrails.md - CybORG researcher run-and-evidence command guardrails: decisions/cyborg-researcher-command-guardrails.md + - CybORG/CAGE-2 protocol-reproduction guardrails: decisions/cyborg-cage2-protocol-reproduction-guardrails.md - CybORG/CAGE-2 downstream environment-pack guardrails: decisions/cyborg-cage2-example-pack-guardrails.md - NASim researcher run-and-evidence command guardrails: decisions/nasim-researcher-command-guardrails.md - CyberBattleSim qualification guardrails: decisions/cyberbattlesim-qualification-guardrails.md diff --git a/src/raes_adapters/cli.py b/src/raes_adapters/cli.py index e0f1a1d..a79ffac 100644 --- a/src/raes_adapters/cli.py +++ b/src/raes_adapters/cli.py @@ -79,6 +79,7 @@ run_cyborg_conformance_suite, verify_selected_cyborg_source, ) +from raes_adapters.cyborg import reproduction as cyborg_reproduction from raes_adapters.cyborg import researcher as cyborg_researcher from raes_adapters.nasim import load_qualification as load_nasim_qualification from raes_adapters.nasim import researcher as nasim_researcher @@ -310,6 +311,12 @@ def _parser() -> _Parser: run_parser.add_argument("--suite", choices=("pr", "full"), default="pr") run_parser.add_argument("--output", type=_relative_output, required=True) _add_admission_arguments(run_parser) + + reproduce_parser = commands.add_parser("reproduce") + reproduce_parser.add_argument("--phase", choices=("declare", "run", "verify"), required=True) + reproduce_parser.add_argument("--output", type=_relative_output, required=True) + reproduce_parser.add_argument("--source-root", type=Path) + reproduce_parser.add_argument("--bundle", type=Path) return parser @@ -1683,6 +1690,8 @@ def _conformance_controls_absent(args: argparse.Namespace) -> bool: def _dispatch(args: argparse.Namespace) -> int: """Dispatch one parsed command through its closed execution path.""" + if args.command == "reproduce": + return _reproduce(args) if args.command == "inspect": print(json.dumps(_adapter(args).inspection_payload(), sort_keys=True)) return 0 @@ -1692,6 +1701,77 @@ def _dispatch(args: argparse.Namespace) -> int: return _run_command(adapter, args) +def _reproduce(args: argparse.Namespace) -> int: + """Dispatch the frozen CAGE-2 study and offline recomputation paths.""" + + source_root = args.source_root + bundle = args.bundle + if args.phase in {"declare", "run"}: + if source_root is None or bundle is not None: + raise _CommandFailure( + EXIT_VALIDATION, + _CONTROLS_INVALID_CODE, + _CONTROLS_INVALID_MESSAGE, + ) + elif bundle is None or source_root is not None: + raise _CommandFailure( + EXIT_VALIDATION, + _CONTROLS_INVALID_CODE, + _CONTROLS_INVALID_MESSAGE, + ) + invocation_root = Path.cwd().resolve() + output = (invocation_root / args.output).resolve() + try: + relative_output = output.relative_to(invocation_root) + except ValueError: + relative_output = None + if relative_output is None or not relative_output.parts: + raise _CommandFailure( + EXIT_OUTPUT, + "researcher.output.unavailable", + "output root is unavailable", + ) + result: dict[str, object] + try: + if args.phase == "declare": + cyborg_reproduction.write_declaration(source_root, output) + result = { + "disposition": "declared", + "frozen_revision": cyborg_reproduction.FROZEN_SELECTION.frozen_revision, + "inventory": _INVENTORY_NAME, + } + elif args.phase == "run": + cyborg_reproduction.run_full_study(source_root, output) + verified = cyborg_reproduction.verify_bundle(output) + result = {"disposition": "completed", "inventory": _INVENTORY_NAME, **verified} + else: + cyborg_reproduction.recompute_bundle(bundle, output) + result = { + "disposition": "verified", + "inventory": _INVENTORY_NAME, + } + except FileExistsError: + raise _CommandFailure( + EXIT_OUTPUT, + "researcher.output.unavailable", + "output root is unavailable", + ) from None + except ValueError: + raise _CommandFailure( + EXIT_VALIDATION, + "researcher.validation.reproduction-invalid", + "reproduction evidence is invalid", + ) from None + except Exception: + raise _CommandFailure( + EXIT_RUNTIME, + _RUNTIME_FAILURE_CODE, + "reproduction execution failed", + ) from None + print(json.dumps(result, sort_keys=True)) + return 0 + + def _run_command(adapter: _BackendAdapter, args: argparse.Namespace) -> int: """Dispatch the ``run`` command through its conformance or native path.""" diff --git a/src/raes_adapters/cyborg/driver.py b/src/raes_adapters/cyborg/driver.py index 1c3cc71..37576de 100644 --- a/src/raes_adapters/cyborg/driver.py +++ b/src/raes_adapters/cyborg/driver.py @@ -227,7 +227,7 @@ def _add_action_cost( if not math.isclose(red_difference, 0.0, rel_tol=0.0, abs_tol=1e-9): raise ValueError action_cost = totals[_BLUE] - component_totals[_BLUE] - if action_cost > 0.0: + if action_cost > 0.0 and not math.isclose(action_cost, 0.0, rel_tol=0.0, abs_tol=1e-9): raise ValueError if external_address == _RESTORE and not math.isclose( action_cost, -1.0, rel_tol=0.0, abs_tol=1e-9 @@ -363,6 +363,31 @@ def __init__(self, *, expected_version: str) -> None: self._runtime_workspace: tempfile.TemporaryDirectory[str] | None = None self._cyborg_type: _NativeCyborgType | None = None self._random_states: dict[int, object] = {} + self._ordered_stream_state: object | None = None + + def begin_ordered_stream(self, seed: int) -> None: + """Initialize one study-scoped Python random stream without global effects.""" + + if type(seed) is not int or not 0 <= seed <= 0xFFFFFFFF: + raise ValueError("ordered stream seed is outside the supported range") + self._ordered_stream_state = random.Random(seed).getstate() + + def ordered_stream_checkpoint(self) -> object: + """Return the current opaque stream checkpoint for bounded orchestration.""" + + if self._ordered_stream_state is None: + raise RuntimeError("ordered stream is not initialized") + return self._ordered_stream_state + + def restore_ordered_stream(self, checkpoint: object) -> None: + """Restore a checkpoint previously returned by this driver.""" + + validator = random.Random() + try: + validator.setstate(cast(tuple[object, ...], checkpoint)) + except Exception: + raise ValueError("ordered stream checkpoint is invalid") from None + self._ordered_stream_state = checkpoint def construct( self, @@ -418,6 +443,8 @@ def _construct( try: if seed is not None: random.seed(seed) + elif self._ordered_stream_state is not None: + random.setstate(cast(tuple[object, ...], self._ordered_stream_state)) native = ( cyborg_type(str(scenario_path), "sim") if agents is None @@ -441,11 +468,14 @@ def _construct( def cleanup(self, handle: object) -> bool: """Shut down an owned CybORG backend without inspecting native output.""" + retained_state = self._random_states.get(id(handle)) try: cast(_NativeCyborg, handle).shutdown() except Exception: return False self._random_states.pop(id(handle), None) + if retained_state is not None and self._ordered_stream_state is not None: + self._ordered_stream_state = retained_state return True def reset(self, handle: object, *, seed: int | None) -> bool: @@ -457,6 +487,11 @@ def reset(self, handle: object, *, seed: int | None) -> bool: if seed is not None: random.seed(seed) cast(_NativeCyborg, handle).set_seed(seed) + else: + stream_state = self._random_states.get(id(handle)) + if stream_state is None: + raise ValueError + random.setstate(cast(tuple[object, ...], stream_state)) cast(_NativeCyborg, handle).reset() self._random_states[id(handle)] = random.getstate() except Exception: diff --git a/src/raes_adapters/cyborg/evaluator.py b/src/raes_adapters/cyborg/evaluator.py index dd1f457..0d88074 100644 --- a/src/raes_adapters/cyborg/evaluator.py +++ b/src/raes_adapters/cyborg/evaluator.py @@ -779,6 +779,79 @@ def _measures( if not facts or not records: return tuple(measures) blue_total = sum(dict(turn.rewards)[_BLUE] for turn in facts) + blue_components: dict[str, float] = dict.fromkeys(_SUPPORTED_COMPONENTS, 0.0) + component_refs: dict[str, list[ExperimentEvidenceRecordReferenceModel]] = { + name: [] for name in _SUPPORTED_COMPONENTS + } + record_by_id = {record.evidence_record_id: record for record in records} + for turn in facts: + for component in turn.components: + if ( + component.participant_address == _BLUE + and component.component in _SUPPORTED_COMPONENTS + ): + blue_components[component.component] += component.value + record_id = _identity( + turn.run_id, + turn.action_instance_id, + turn.logical_step, + component.participant_address, + component.target_address, + component.component, + ) + evidence_id = "evidence-record.cyborg-cage2." + record_id + if evidence_id in record_by_id: + component_refs[component.component].append( + ExperimentEvidenceRecordReferenceModel( + ref_kind="evidence-record", + ref_id=evidence_id, + ref_version=_RECORD_VERSION, + ) + ) + for component_name in sorted(_SUPPORTED_COMPONENTS): + refs_for_component = component_refs[component_name] + if refs_for_component: + measures.append( + ExperimentDerivedMeasureModel( + schema_version="experiment-derived-measure/v1", + derived_measure_id=( + f"measure.cyborg-cage2.cumulative-blue-{component_name}." + + _identity(facts[-1].run_id, facts[-1].episode_id, len(facts)) + ), + measure_version="1.0.0", + measure_kind="metric", + metric_ref=ExperimentReferenceModel( + ref_kind="metric-definition", + ref_id=f"cage2-cumulative-blue-{component_name}-reward", + ref_version="1.0.0", + ), + method=ExperimentDerivedMeasureMethodModel( + method_id=f"cyborg-cage2-sum-blue-{component_name}-rewards", + method_version="1.0.0", + name=f"Cumulative committed Blue {component_name} reward", + description=( + "Sum the named committed per-step Blue component in logical " + "step order." + ), + ), + source_evidence_refs=refs_for_component, + generated_at=now, + value_status="reported", + value=blue_components[component_name], + uncertainty="No uncertainty interval is inferred from one source run.", + limitations=[ + "This component is not a conformance result, objective outcome, " + "or replication-equivalence claim." + ], + provenance_refs=[ + ExperimentReferenceModel( + ref_kind="run", + ref_id=facts[-1].run_id, + ref_version="1.0.0", + ) + ], + ) + ) refs = [ ExperimentEvidenceRecordReferenceModel( ref_kind="evidence-record", diff --git a/src/raes_adapters/cyborg/orchestrator.py b/src/raes_adapters/cyborg/orchestrator.py index 9a706bc..69e997d 100644 --- a/src/raes_adapters/cyborg/orchestrator.py +++ b/src/raes_adapters/cyborg/orchestrator.py @@ -62,6 +62,7 @@ def __init__(self) -> None: self._lock = threading.RLock() self._policy: _ExecutionPolicy | None = None self._source_terminal = False + self._run_id: str | None = None def activate(self, policy: _ExecutionPolicy) -> None: """Activate one admitted execution policy.""" @@ -69,6 +70,7 @@ def activate(self, policy: _ExecutionPolicy) -> None: with self._lock: self._policy = policy self._source_terminal = False + self._run_id = None def clear(self) -> None: """Clear all active execution controls.""" @@ -76,6 +78,7 @@ def clear(self) -> None: with self._lock: self._policy = None self._source_terminal = False + self._run_id = None def policy(self) -> _ExecutionPolicy | None: """Return the currently admitted policy, if any.""" @@ -101,6 +104,22 @@ def reset_terminal(self) -> None: with self._lock: self._source_terminal = False + def begin_run(self, run_id: str) -> None: + """Bind one safe portable run identity to the active logical episode.""" + + if not 1 <= len(run_id) <= 64 or not run_id.replace("-", "").isalnum(): + raise ValueError("run id is not a safe label") + with self._lock: + if self._policy is None: + raise RuntimeError("execution policy is unavailable") + self._run_id = run_id + + def run_id(self) -> str | None: + """Return the currently bound portable run identity.""" + + with self._lock: + return self._run_id + class CyborgOrchestrator(object): """Admit one CAGE-2 workflow and bind its declared control policy.""" @@ -209,6 +228,11 @@ def status(self) -> dict[str, object]: return {"running": self._control.policy() is not None, "results": len(self._results)} + def begin_run(self, run_id: str) -> None: + """Bind the next episode's portable run identity without native effects.""" + + self._control.begin_run(run_id) + def results(self) -> dict[str, dict[str, object]]: """Return defensive copies of portable workflow results.""" diff --git a/src/raes_adapters/cyborg/participant_runtime.py b/src/raes_adapters/cyborg/participant_runtime.py index ca3d30c..029463c 100644 --- a/src/raes_adapters/cyborg/participant_runtime.py +++ b/src/raes_adapters/cyborg/participant_runtime.py @@ -401,6 +401,30 @@ def _clear_observations(self) -> None: self._observations = {} self._pending_observations = {} + def begin_independent_run(self) -> bool: + """Reset the retained native session at a new experiment-run boundary. + + A reproduction condition intentionally reuses one native CybORG session + so its study-scoped random stream continues across source resets. RAES + experiment runs are nevertheless independent archival units: carrying + the preceding run's portable lifecycle and action history into the next + run would both contaminate its evidence and make validation quadratic. + + The caller therefore resumes from the pristine, already-realized + snapshot captured before participant initialization. This method owns + only the corresponding backend-private reset and mirror cleanup. + """ + + with self._provisioner.execution_transaction(): + if not self._provisioner.reset_execution(): + return False + self._results = {} + self._history = {} + self._episode_counter = {} + self._clear_observations() + self._control.reset_terminal() + return True + @staticmethod def _reset_failure(snapshot: RuntimeSnapshot) -> ApplyResult: """Return the bounded aggregate-reset failure.""" @@ -566,7 +590,7 @@ def _native_turn( try: projected = self._provisioner.execute_turn( request.validated_selection, - run_id=f"{policy.workflow_address}-run", + run_id=self._control.run_id() or f"{policy.workflow_address}-run", episode_id=episode_id, action_instance_id=request.action_instance_id, logical_step=tick + 1, diff --git a/src/raes_adapters/cyborg/provisioner.py b/src/raes_adapters/cyborg/provisioner.py index 995a67e..49dd6d5 100644 --- a/src/raes_adapters/cyborg/provisioner.py +++ b/src/raes_adapters/cyborg/provisioner.py @@ -6,7 +6,7 @@ import threading from collections.abc import Iterator from contextlib import contextmanager -from typing import NamedTuple +from typing import NamedTuple, cast from raes_backend_protocols.protocols import Provisioner # type: ignore[import-untyped] from raes_contracts.contracts import ( # type: ignore[import-untyped] @@ -64,6 +64,7 @@ def __init__( profile_id: str, source_commit: str, seed: int | None, + reseed_on_reset: bool = True, scenario_binding: CyborgScenarioBinding | None = None, ) -> None: self._driver = driver @@ -71,6 +72,7 @@ def __init__( self._profile_id = profile_id self._source_commit = source_commit self._seed = seed + self._reseed_on_reset = reseed_on_reset self._scenario_binding = scenario_binding self._active: object | None = None self._active_available = False @@ -137,9 +139,12 @@ def _replace_backend( """Construct the desired backend before retiring the current handle.""" descriptor = self._descriptor(plan) + checkpoint = self._ordered_stream_checkpoint() try: candidate = self._driver.construct(descriptor, seed=self._seed) except Exception: + if checkpoint is not None: + self._restore_ordered_stream(checkpoint) return _failure( snapshot, _diagnostic( @@ -152,8 +157,12 @@ def _replace_backend( self._active_available = False if not self._cleanup_handle(candidate): self._pending_cleanup.append(candidate) + if checkpoint is not None: + self._restore_ordered_stream(checkpoint) return _failure(snapshot, _cleanup_failed_diagnostic()) + if checkpoint is not None: + self._restore_ordered_stream(checkpoint) self._active = candidate self._active_available = True self._active_descriptor = descriptor @@ -162,6 +171,25 @@ def _replace_backend( self._clear_evaluation() return _success(snapshot, reconciliation, self._realization_envelope) + def _ordered_stream_checkpoint(self) -> object | None: + """Capture an optional driver study stream around provisional realization.""" + + checkpoint = getattr(self._driver, "ordered_stream_checkpoint", None) + if not callable(checkpoint): + return None + try: + return cast(object, checkpoint()) + except RuntimeError: + return None + + def _restore_ordered_stream(self, checkpoint: object) -> None: + """Restore a captured driver stream or fail the provisional realization.""" + + restore = getattr(self._driver, "restore_ordered_stream", None) + if not callable(restore): + raise RuntimeError("ordered study stream is unavailable") + restore(checkpoint) + @contextmanager def execution_transaction(self) -> Iterator[None]: """Serialize one complete native-and-portable session transition.""" @@ -314,7 +342,8 @@ def reset_execution(self) -> bool: if self._active is None or not self._active_available or not callable(reset): return False try: - succeeded = reset(self._active, seed=self._seed) is True + reset_seed = self._seed if self._reseed_on_reset else None + succeeded = reset(self._active, seed=reset_seed) is True except Exception: succeeded = False if not succeeded: diff --git a/src/raes_adapters/cyborg/reproduction.py b/src/raes_adapters/cyborg/reproduction.py new file mode 100644 index 0000000..e38c218 --- /dev/null +++ b/src/raes_adapters/cyborg/reproduction.py @@ -0,0 +1,2369 @@ +"""Frozen CAGE-2 outcome-reproduction study and offline derivation. + +This module is deliberately backend-local. RAES models remain the authority +for experiment evidence and runs; the dictionaries here are immutable indexes +for the issue-22 research export rather than another portable contract family. +""" + +from __future__ import annotations + +import gzip +import hashlib +import json +import math +import os +import platform +import statistics +import tempfile +from collections import Counter +from collections.abc import Callable, Mapping, Sequence +from contextlib import redirect_stderr, redirect_stdout, suppress +from dataclasses import dataclass, replace +from importlib import metadata +from pathlib import Path +from statistics import NormalDist +from typing import cast + +from raes_contracts.contracts import ( # type: ignore[import-untyped] + ExperimentArtifactRefModel, + ExperimentDerivedMeasureModel, + ExperimentEvidenceRecordModel, + ExperimentRunModel, + ExperimentSpecModel, + ExperimentTaskModel, + ParticipantConfigurationResultModel, + ParticipantImplementationManifestModel, + ParticipantImplementationSelectionModel, + validate_experiment_run_against_task, +) +from raes_contracts.participant_configuration import ( # type: ignore[import-untyped] + validate_participant_configuration_selection, +) +from raes_operations.run_artifacts import ( # type: ignore[import-untyped] + atomic_write_json_artifact, +) + +from .driver import CyborgDriver, SourceInstalledCyborgDriver +from .researcher import ( + EpisodeEvidence, + RunControls, + archival_run, + execute_episode_series, +) + +_INVALID_JSON = "invalid JSON artifact" +_DISPOSITIONS = ("excluded", "failed", "invalid", "valid") +_RED_VARIANTS = ("b-line", "meander", "sleep") +_TRIAL_LENGTHS = (30, 50, 100) +_TIERS = ( + "authored-source", + "contract", + "execution-control", + "state/observation", + "outcome/evaluation", + "disclosure", +) +_SCORE_METRIC_ID = "cage2-cumulative-blue-reward" +_SOURCE_COMMIT = "26ce1c1253fa9e2e73f25e6a7f2da32860c11257" +_PAPER_SHA256 = "45a6e564da6dff67453ec75585e847228a7f407641f250f4abe28d1f29406f29" +_PRIVATE_KEY_MARKERS = ( + " ".join(("BEGIN", "OPENSSH", "PRIVATE", "KEY")), + " ".join(("BEGIN", "PRIVATE", "KEY")), +) +_FORBIDDEN_TEXT = ( + "Traceback (most recent call last):", + *_PRIVATE_KEY_MARKERS, + "/home/", +) +_FORBIDDEN_FIELDS = frozenset( + { + "action_id", + "action_mask", + "hidden_truth", + "native_observation", + "native_state", + "random_state", + "reward_vector", + } +) +_MAX_PUBLIC_FILE_BYTES = 500 * 1024 +_MAX_DECOMPRESSED_JSON_BYTES = 64 * 1024 * 1024 + + +@dataclass(frozen=True) +class ReproductionSelection: + """One immutable backend-local study selection.""" + + frozen_revision: str + episodes_per_condition: int + seed: int + trial_lengths: tuple[int, ...] + red_variants: tuple[str, ...] + retry_limit: int + + def with_cardinality_for_test(self, episodes: int) -> ReproductionSelection: + """Return a reduced-cardinality selection for the identical test path.""" + + if type(episodes) is not int or episodes < 1: + raise ValueError("test cardinality is invalid") + return replace( + self, + frozen_revision=f"{self.frozen_revision}-test-{episodes}", + episodes_per_condition=episodes, + ) + + +FROZEN_SELECTION = ReproductionSelection( + frozen_revision="cage2-26ce1c1-ccs-sleeper-v1", + episodes_per_condition=1000, + seed=153, + trial_lengths=_TRIAL_LENGTHS, + red_variants=_RED_VARIANTS, + retry_limit=1, +) + +_PUBLISHED_CONDITION_MEANS: dict[tuple[int, str], tuple[float, int]] = { + (30, "b-line"): (-218.65, 2), + (30, "meander"): (-39.31, 2), + (30, "sleep"): (0.0, 0), + (50, "b-line"): (-480.17, 2), + (50, "meander"): (-267.6, 1), + (50, "sleep"): (0.0, 0), + (100, "b-line"): (-1134.03, 2), + (100, "meander"): (-972.43, 2), + (100, "sleep"): (0.0, 0), +} + +_INPUT_PATHS: dict[str, str] = { + "authored-scenario": "src/raes_adapters/cyborg/scenario/cage2-scenario2.sdl.yaml", + "module-lock": "src/raes_adapters/cyborg/scenario/raes.lock.json", + "pack-manifest": ( + "src/raes_adapters/cyborg/examples/cage2-research/pack.content-manifest.json" + ), + "experiment-authoring-input": ( + "src/raes_adapters/cyborg/examples/cage2-research/experiment/cage2-research.spec.exp.json" + ), + "experiment-task": ( + "src/raes_adapters/cyborg/examples/cage2-research/experiment/cage2-research.task.exp.json" + ), + "blue-manifest": ( + "src/raes_adapters/cyborg/examples/cage2-research/participant/" + "cyborg-blue-sleep-policy.manifest.json" + ), + "blue-selection": ( + "src/raes_adapters/cyborg/examples/cage2-research/participant/" + "cyborg-blue-sleep-policy.selection.json" + ), + "blue-configuration": ( + "src/raes_adapters/cyborg/examples/cage2-research/participant/" + "cyborg-blue-sleep-policy.configuration.json" + ), + "qualification": "src/raes_adapters/cyborg/qualification.json", + "mapping-ledger": "src/raes_adapters/cyborg/mapping/cage2-source-ledger.jsonl", + "loss-disclosures": "src/raes_adapters/cyborg/mapping/cage2-loss-disclosures.md", +} +_FROZEN_ARTIFACT_FACTS: dict[str, tuple[str, int]] = { + "authored-scenario": ( + "14da37a9e4c4e09136fe7d6fc10a75d515ac29243bfe2b98cf81839847b3a544", + 27339, + ), + "blue-configuration": ( + "c31948db14751cf80dd66726c34d7471786eabf3d934658f1d40248fb065f918", + 997, + ), + "blue-manifest": ( + "73cf1016f04ead0741755dd98dd1a1e4cc2927b14e948246294a7bd1a28cd26b", + 2271, + ), + "blue-selection": ( + "e5a8dfb2134a708bac9b479fd23661948d6c617f7bc0b8855683445645ec3791", + 1166, + ), + "experiment-authoring-input": ( + "7a0140d85ae0114d76b4248197d391661d3596dffcd82464022e31b83b1caa0e", + 3102, + ), + "experiment-task": ( + "e04872ff2c8308741cd149b7403c968b0cf64d9a46c6f5ba55c8c73c21f000ae", + 2621, + ), + "loss-disclosures": ( + "cb0ebf48ae406140d4ca8657498107e825879c4002e0ae7394cabde1196fbf58", + 3166, + ), + "mapping-ledger": ( + "e3ab5e419ecc1aed4b7277188fd67a0a12c2228dbd0b5e67396fa955affadb8f", + 37320, + ), + "module-lock": ( + "783160e17db156bc5d7e7dedd5844d3f38767a483e1b9b5988ed7149e196916e", + 56, + ), + "pack-manifest": ( + "27d6a1bc57ca89194a27d873179654b424f25c9c65a00c4e19ad32f87b3c9c8e", + 7169, + ), + "qualification": ( + "8dd70165e8f0dbf4a804cb287058dd7a5fddaefc50144d3e4c21b04530a958a4", + 19427, + ), +} +_FROZEN_CONTRACT_DIGESTS = { + "experiment_spec": "f0ed630c86d997e889fb9e57d76383c5b0fa8fda8ae61c2b83b9572d1dd8e88b", + "experiment_task": "d5f317677eb481fcb7f60d7e2eae4a9ce93253b4e87a59f06ba85543b6efdafc", + "participant_manifest": "bb8bd43edff6d011808b8c09b72bcf63e759cc761d2e1807e259cdbd790cf43e", + "participant_selection": "fa84715499b4e91814b3da7898718dd2a61ffc239c7c4e010bdcd1ea8f335740", + "participant_configuration": ( + "a4c3e209b784123250d72a454f369f093a11987ad3782e690ca9e17c6e342d0c" + ), +} + + +def canonical_json_bytes(payload: object) -> bytes: + """Return the sole issue-local canonical JSON byte representation.""" + + return ( + json.dumps(payload, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n" + ).encode() + + +def sha256_payload(payload: object) -> str: + """Hash one issue-local payload canonically.""" + + return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _reject_constant(_value: str) -> object: + raise ValueError(_INVALID_JSON) + + +def _closed_pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(_INVALID_JSON) + result[key] = value + return result + + +def _load_strict_json_value(path: Path) -> object: + """Load duplicate-key-free finite JSON, including deterministic gzip JSON.""" + + try: + raw = path.read_bytes() + if path.name.endswith(".json.gz"): + raw = gzip.decompress(raw) + if len(raw) > _MAX_DECOMPRESSED_JSON_BYTES: + raise ValueError(_INVALID_JSON) + value = json.loads( + raw, + object_pairs_hook=_closed_pairs, + parse_constant=_reject_constant, + ) + except (OSError, UnicodeError, ValueError, gzip.BadGzipFile) as error: + raise ValueError(_INVALID_JSON) from error + return value + + +def load_strict_json(path: Path) -> dict[str, object]: + """Load a duplicate-key-free finite JSON object.""" + + value = _load_strict_json_value(path) + if not isinstance(value, dict): + raise ValueError(_INVALID_JSON) + return cast(dict[str, object], value) + + +def _condition_id(trial_length: int, red_variant: str) -> str: + return f"steps-{trial_length:03d}--red-{red_variant}" + + +def _schedule(selection: ReproductionSelection) -> list[dict[str, object]]: + schedule: list[dict[str, object]] = [] + ordinal = 0 + for trial_length in selection.trial_lengths: + for red_variant in selection.red_variants: + condition_id = _condition_id(trial_length, red_variant) + for episode in range(1, selection.episodes_per_condition + 1): + ordinal += 1 + slot_id = f"cage2-slot-{ordinal:05d}" + schedule.append( + { + "slot_id": slot_id, + "condition_id": condition_id, + "trial_length": trial_length, + "red_variant": red_variant, + "episode_ordinal": episode, + "study_stream_ordinal": ordinal, + "eligibility": "scheduled", + "run_id": f"{slot_id}-attempt-01", + "attempt_sequence": 1, + } + ) + return schedule + + +def _schedule_partitions(selection: ReproductionSelection) -> list[dict[str, object]]: + schedule = _schedule(selection) + partitions: list[dict[str, object]] = [] + offset = 0 + for trial_length in selection.trial_lengths: + for red_variant in selection.red_variants: + entries = schedule[offset : offset + selection.episodes_per_condition] + condition_id = _condition_id(trial_length, red_variant) + partitions.append( + { + "condition_id": condition_id, + "path": f"plans/{condition_id}.json", + "slot_count": len(entries), + "first_slot_id": entries[0]["slot_id"], + "last_slot_id": entries[-1]["slot_id"], + "sha256": sha256_payload(entries), + } + ) + offset += selection.episodes_per_condition + return partitions + + +def _artifact_refs(repo_root: Path) -> list[dict[str, object]]: + refs: list[dict[str, object]] = [] + for artifact_id, relative in sorted(_INPUT_PATHS.items()): + path = repo_root / relative + if not path.is_file() or path.is_symlink(): + raise ValueError("frozen input is unavailable") + digest = _sha256_file(path) + size = path.stat().st_size + if (digest, size) != _FROZEN_ARTIFACT_FACTS[artifact_id]: + raise ValueError("frozen input identity is invalid") + refs.append( + { + "artifact_id": artifact_id, + "path": relative, + "sha256": digest, + "size_bytes": size, + } + ) + return refs + + +def _contract_payloads(repo_root: Path) -> dict[str, object]: + contracts: dict[str, object] = { + "experiment_spec": load_strict_json(repo_root / _INPUT_PATHS["experiment-authoring-input"]), + "experiment_task": load_strict_json(repo_root / _INPUT_PATHS["experiment-task"]), + "participant_manifest": load_strict_json(repo_root / _INPUT_PATHS["blue-manifest"]), + "participant_selection": load_strict_json(repo_root / _INPUT_PATHS["blue-selection"]), + "participant_configuration": load_strict_json( + repo_root / _INPUT_PATHS["blue-configuration"] + ), + } + if { + name: sha256_payload(payload) for name, payload in contracts.items() + } != _FROZEN_CONTRACT_DIGESTS: + raise ValueError("frozen contract identity is invalid") + return contracts + + +def _expected_artifact_refs() -> list[dict[str, object]]: + return [ + { + "artifact_id": artifact_id, + "path": _INPUT_PATHS[artifact_id], + "sha256": _FROZEN_ARTIFACT_FACTS[artifact_id][0], + "size_bytes": _FROZEN_ARTIFACT_FACTS[artifact_id][1], + } + for artifact_id in sorted(_INPUT_PATHS) + ] + + +def _source_ledger(artifacts: Sequence[Mapping[str, object]]) -> dict[str, object]: + return { + "source": { + "repository": "https://github.com/cage-challenge/cage-challenge-2", + "commit": _SOURCE_COMMIT, + "profile_id": "cage2-cyborg-2.1-source-26ce1c1", + }, + "protocol_source_facts": { + "qualified_evaluator": { + "artifact_ref": "qualification", + "path": "CybORG/CybORG/Evaluation/evaluation.py", + "episode_count": 100, + "trial_lengths": [30, 50, 100], + "red_variants": ["B_lineAgent", "RedMeanderAgent", "SleepAgent"], + "seed_binding": None, + }, + "public_validation": { + "artifact_ref": "qualification", + "path": "README.md", + "episode_count": 1000, + "seed_statement": "random.seed(153)", + "confidence_interval_statement": "95-percent-confidence-intervals", + }, + "paper": { + "identifier": "arXiv:2309.07388v1", + "url": "https://arxiv.org/pdf/2309.07388v1", + "sha256": _PAPER_SHA256, + }, + }, + "artifact_refs": [dict(item) for item in artifacts], + "loss_refs": [ + "loss-blue-policy-artifact-unbound", + "loss-evaluation-seed-unbound", + "loss-native-observation-boundary", + "loss-remove-success-misreport", + "loss-wrapper-termination-cutoff", + ], + "exclusions": [ + "BlueLoadAgent-without-model-bytes", + "private-or-unpublished-submitted-agent-artifacts", + ], + } + + +def _published_results() -> dict[str, object]: + conditions = [] + for trial_length in _TRIAL_LENGTHS: + for red_variant in _RED_VARIANTS: + value, decimals = _PUBLISHED_CONDITION_MEANS[(trial_length, red_variant)] + conditions.append( + { + "condition_id": _condition_id(trial_length, red_variant), + "mean": value, + "display_decimals": decimals, + "bounded_margin": max(1.0, abs(value) * 0.01), + } + ) + return { + "result_identity": "cage2-readme-leaderboard-ccs-sleeper", + "agent_label": "CCS Sleeper", + "technique_label": "Sleeping Agent", + "agent_artifact_identity": "unavailable", + "conditions": conditions, + "primary_total": -3112.2, + "primary_total_interval_95": [-3118.87, -3105.53], + "reported_interval_half_width": 6.67, + } + + +def _declaration( + selection: ReproductionSelection, + artifacts: list[dict[str, object]], + contracts: Mapping[str, object], +) -> dict[str, object]: + return { + "frozen_revision": selection.frozen_revision, + "source_profile": "cage2-cyborg-2.1-source-26ce1c1", + "source_commit": _SOURCE_COMMIT, + "baseline": { + "observed_implementation": "cyborg-blue-sleep-policy", + "observed_behavior": "participant.action-contract.sleep", + "published_result_identity": "cage2-readme-leaderboard-ccs-sleeper", + "identity_relation": "behavioral-reconstruction-not-agent-artifact-identity", + "loss_ref": "loss-blue-policy-artifact-unbound", + }, + "artifacts": artifacts, + "published_contracts": dict(contracts), + "trial_lengths": list(selection.trial_lengths), + "red_variants": list(selection.red_variants), + "episodes_per_condition": selection.episodes_per_condition, + "schedule_partitions": _schedule_partitions(selection), + "seed_policy": { + "api": "random.seed", + "initialization_scope": "study", + "owner": "python-random", + "reset_behavior": "continue-stream-across-episode-reset", + "seed": selection.seed, + "serialization": "trial-length-then-red-variant-then-episode", + }, + "attempt_policy": { + "retry_limit": selection.retry_limit, + "selection": "first-valid-attempt-in-attempt-sequence", + "terminal_dispositions": list(_DISPOSITIONS), + "missing_scores": "never-imputed", + }, + "aggregation": { + "score_metric_id": _SCORE_METRIC_ID, + "condition_estimator": "arithmetic-mean", + "sample_variance_divisor": "n-1", + "interval": "two-sided-normal-approximation-95-percent", + "critical_value": NormalDist().inv_cdf(0.975), + "primary_estimand": "sum-of-nine-condition-means", + "primary_standard_error": "sqrt-sum-of-independent-condition-mean-variances", + "finite_values_required": True, + "minimum_eligible_per_condition": selection.episodes_per_condition, + "rounding": "binary64-calculation-json-number-output-published-display-only", + "method_authority": "adapter-study-predeclared-method-not-attributed-to-paper", + }, + "comparison": { + "published": _published_results(), + "exact": "all-condition-means-equal-after-published-display-rounding", + "bounded": ( + "all-condition-means-within-predeclared-margins-and-primary-95-percent-" + "intervals-mutually-cover-the-point-estimates" + ), + "failed": "neither-exact-nor-bounded", + }, + "tier_policy": list(_TIERS), + "explicit_non_claims": [ + "deterministic-replay", + "backend-identity", + "submitted-agent-artifact-identity", + "exact-native-state-equivalence", + "state-or-observation-equivalence-from-score", + "conformance-from-score", + "scientific-reproducibility-beyond-retained-evidence", + ], + } + + +def build_declaration( + repo_root: Path, + *, + selection: ReproductionSelection = FROZEN_SELECTION, +) -> tuple[dict[str, object], dict[str, object]]: + """Build the frozen protocol and its source-owner index.""" + + root = repo_root.resolve() + artifacts = _artifact_refs(root) + contracts = _contract_payloads(root) + declaration = _declaration(selection, artifacts, contracts) + protocol: dict[str, object] = { + "declaration_sha256": sha256_payload(declaration), + "declaration": declaration, + } + source_ledger = _source_ledger(artifacts) + validate_protocol(protocol, selection=selection) + _validate_source_ledger(source_ledger, protocol) + return protocol, source_ledger + + +def _require_exact_keys(value: Mapping[str, object], keys: set[str], label: str) -> None: + if set(value) != keys: + raise ValueError(f"{label} has unknown or missing fields") + + +def _validate_schedule(value: object, selection: ReproductionSelection) -> None: + if not isinstance(value, list) or len(value) != ( + len(selection.trial_lengths) + * len(selection.red_variants) + * selection.episodes_per_condition + ): + raise ValueError("attempt schedule is invalid") + run_ids = [item["run_id"] for item in value] + slot_ids = [item["slot_id"] for item in value] + if len(set(run_ids)) != len(run_ids) or len(set(slot_ids)) != len(slot_ids): + raise ValueError("attempt identities are not unique") + expected = _schedule(selection) + if value != expected: + raise ValueError("attempt schedule is invalid") + + +def validate_protocol( + payload: Mapping[str, object], + *, + selection: ReproductionSelection | None = None, +) -> None: + """Validate the closed frozen protocol and its content digest.""" + + _require_exact_keys( + payload, + {"declaration_sha256", "declaration"}, + "protocol", + ) + declaration = payload["declaration"] + if not isinstance(declaration, dict): + raise ValueError("protocol declaration is invalid") + if payload["declaration_sha256"] != sha256_payload(declaration): + raise ValueError("protocol declaration digest is invalid") + artifacts = declaration.get("artifacts") + if not isinstance(artifacts, list) or artifacts != _expected_artifact_refs(): + raise ValueError("protocol artifacts are invalid") + if selection is None: + selection = _selection_from_declaration(declaration) + contracts = declaration.get("published_contracts") + if not isinstance(contracts, dict): + raise ValueError("published contracts are invalid") + if { + name: sha256_payload(contract) for name, contract in contracts.items() + } != _FROZEN_CONTRACT_DIGESTS: + raise ValueError("published contract identity is invalid") + try: + spec = ExperimentSpecModel.model_validate(contracts["experiment_spec"]) + task = ExperimentTaskModel.model_validate(contracts["experiment_task"]) + manifest = ParticipantImplementationManifestModel.model_validate( + contracts["participant_manifest"] + ) + participant_selection = ParticipantImplementationSelectionModel.model_validate( + contracts["participant_selection"] + ) + configuration = ParticipantConfigurationResultModel.model_validate( + contracts["participant_configuration"] + ) + validate_participant_configuration_selection(participant_selection, configuration) + if ( + spec.task_ref.ref_id != task.task_id + or participant_selection.implementation_identity != manifest.identity + ): + raise ValueError + except Exception as error: + raise ValueError("published contracts are invalid") from error + expected = _declaration( + selection, + cast(list[dict[str, object]], artifacts), + contracts, + ) + if declaration.get("seed_policy") != expected["seed_policy"]: + raise ValueError("seed policy is invalid") + if declaration.get("attempt_policy") != expected["attempt_policy"]: + raise ValueError("attempt policy is invalid") + if declaration.get("aggregation") != expected["aggregation"]: + raise ValueError("aggregation method is invalid") + if declaration.get("comparison") != expected["comparison"]: + raise ValueError("comparison method is invalid") + for key, value in expected.items(): + if key not in {"artifacts", "published_contracts"} and declaration.get(key) != value: + raise ValueError("protocol declaration is invalid") + _validate_schedule(_schedule(selection), selection) + + +def _validate_source_ledger(payload: Mapping[str, object], protocol: Mapping[str, object]) -> None: + declaration = protocol["declaration"] + if not isinstance(declaration, dict): + raise ValueError("source ledger artifact join is invalid") + artifacts = declaration.get("artifacts") + if not isinstance(artifacts, list) or dict(payload) != _source_ledger( + cast(list[dict[str, object]], artifacts) + ): + raise ValueError("source ledger is invalid") + + +def _score_measure(run_id: str, score: float) -> dict[str, object]: + if not math.isfinite(score): + raise ValueError("score measure is invalid") + payload = ExperimentDerivedMeasureModel( + schema_version="experiment-derived-measure/v1", + derived_measure_id=f"derived-measure.{run_id}.cumulative-blue-reward", + measure_version="1.0.0", + measure_kind="score", + metric_ref={ + "ref_kind": "metric-definition", + "ref_id": _SCORE_METRIC_ID, + "ref_version": "1.0.0", + }, + method={ + "method_id": "cyborg-cumulative-blue-reward-v1", + "method_version": "1.0.0", + "name": "Sum committed per-step Blue rewards", + "parameters": [], + }, + source_evidence_refs=[ + { + "ref_kind": "evidence-record", + "ref_id": f"evidence-record.{run_id}.test", + "ref_version": "1.0.0", + } + ], + generated_at="2026-01-01T00:00:00Z", + value_status="reported", + value=score, + limitations=["Test projection; production uses CyborgEvaluator output."], + provenance_refs=[ + { + "ref_kind": "profile", + "ref_id": "cage2-cyborg-2.1-source-26ce1c1", + "ref_version": _SOURCE_COMMIT, + } + ], + ).model_dump(mode="json") + return cast(dict[str, object], payload) + + +def attempt_record_for_test( + protocol: Mapping[str, object], slot: Mapping[str, object], *, score: float +) -> dict[str, object]: + """Construct reduced-cardinality canonical score input for unit tests.""" + + return { + "declaration_sha256": protocol["declaration_sha256"], + "slot_id": slot["slot_id"], + "run_id": slot["run_id"], + "predecessor_run_id": None, + "attempt_sequence": slot["attempt_sequence"], + "condition_id": slot["condition_id"], + "trial_length": slot["trial_length"], + "red_variant": slot["red_variant"], + "episode_ordinal": slot["episode_ordinal"], + "disposition": "valid", + "diagnostic_code": None, + "cleanup_verified": True, + "score_measure": _score_measure(cast(str, slot["run_id"]), score), + "evidence_refs": [f"runs/{slot['run_id']}/evidence.json"], + } + + +def retry_record_for_test(original: Mapping[str, object], *, score: float) -> dict[str, object]: + """Construct a valid second attempt for retry-policy tests.""" + + retry = dict(original) + run_id = f"{original['slot_id']}-attempt-02" + retry.update( + run_id=run_id, + predecessor_run_id=original["run_id"], + attempt_sequence=2, + disposition="valid", + diagnostic_code=None, + cleanup_verified=True, + score_measure=_score_measure(run_id, score), + evidence_refs=[f"runs/{run_id}/evidence.json"], + ) + return retry + + +def _validate_score_measure(value: object) -> float: + try: + measure = ExperimentDerivedMeasureModel.model_validate(value) + except Exception as error: + raise ValueError("score measure is invalid") from error + score = measure.value + if ( + measure.measure_kind != "score" + or measure.metric_ref.ref_id != _SCORE_METRIC_ID + or measure.value_status != "reported" + or isinstance(score, bool) + or not isinstance(score, (int, float)) + or not math.isfinite(float(score)) + ): + raise ValueError("score measure is invalid") + return float(score) + + +def _validate_attempt( + value: Mapping[str, object], protocol: Mapping[str, object], slot: Mapping[str, object] +) -> None: + expected_keys = { + "declaration_sha256", + "slot_id", + "run_id", + "predecessor_run_id", + "attempt_sequence", + "condition_id", + "trial_length", + "red_variant", + "episode_ordinal", + "disposition", + "diagnostic_code", + "cleanup_verified", + "score_measure", + "evidence_refs", + } + _require_exact_keys(value, expected_keys, "attempt record") + if value["declaration_sha256"] != protocol["declaration_sha256"]: + raise ValueError("attempt protocol join is invalid") + for key in ( + "slot_id", + "condition_id", + "trial_length", + "red_variant", + "episode_ordinal", + ): + if value[key] != slot[key]: + raise ValueError("attempt slot join is invalid") + sequence = value["attempt_sequence"] + declaration = protocol["declaration"] + if not isinstance(declaration, dict): + raise ValueError("attempt protocol join is invalid") + retry_limit = cast(dict[str, object], declaration["attempt_policy"])["retry_limit"] + if ( + type(sequence) is not int + or type(retry_limit) is not int + or not 1 <= sequence <= retry_limit + 1 + ): + raise ValueError("attempt sequence is invalid") + expected_run_id = f"{slot['slot_id']}-attempt-{sequence:02d}" + predecessor = None if sequence == 1 else f"{slot['slot_id']}-attempt-{sequence - 1:02d}" + if value["run_id"] != expected_run_id or value["predecessor_run_id"] != predecessor: + raise ValueError("attempt identities are invalid") + disposition = value["disposition"] + if disposition not in _DISPOSITIONS: + raise ValueError("attempt disposition is invalid") + if disposition == "valid": + if value["cleanup_verified"] is not True: + raise ValueError("valid attempt cleanup is invalid") + _validate_score_measure(value["score_measure"]) + elif value["score_measure"] is not None: + raise ValueError("ineligible attempt has a score measure") + + +def _selection_from_declaration(declaration: Mapping[str, object]) -> ReproductionSelection: + try: + selection = ReproductionSelection( + frozen_revision=cast(str, declaration["frozen_revision"]), + episodes_per_condition=cast(int, declaration["episodes_per_condition"]), + seed=cast(int, cast(dict[str, object], declaration["seed_policy"])["seed"]), + trial_lengths=tuple(cast(list[int], declaration["trial_lengths"])), + red_variants=tuple(cast(list[str], declaration["red_variants"])), + retry_limit=cast( + int, cast(dict[str, object], declaration["attempt_policy"])["retry_limit"] + ), + ) + if declaration["schedule_partitions"] != _schedule_partitions(selection): + raise ValueError + except (KeyError, TypeError, ValueError): + raise ValueError("protocol schedule is invalid") from None + return selection + + +def _slot_index(protocol: Mapping[str, object]) -> dict[str, dict[str, object]]: + declaration = protocol["declaration"] + if not isinstance(declaration, dict): + raise ValueError("protocol schedule is invalid") + selection = _selection_from_declaration(declaration) + schedule = _schedule(selection) + if declaration["schedule_partitions"] != _schedule_partitions(selection): + raise ValueError("protocol schedule is invalid") + return {cast(str, item["slot_id"]): item for item in schedule if isinstance(item, dict)} + + +def schedule_for_test(protocol: Mapping[str, object]) -> list[dict[str, object]]: + """Return the validated generated schedule for reduced-cardinality tests.""" + + return list(_slot_index(protocol).values()) + + +def select_eligible_attempts( + protocol: Mapping[str, object], attempts: Sequence[Mapping[str, object]] +) -> list[dict[str, object]]: + """Validate every attempt and select the first valid attempt per slot.""" + + slots = _slot_index(protocol) + by_slot: dict[str, list[dict[str, object]]] = {key: [] for key in slots} + run_ids: set[str] = set() + for raw in attempts: + record = dict(raw) + slot_id = record.get("slot_id") + if not isinstance(slot_id, str) or slot_id not in slots: + raise ValueError("attempt slot join is invalid") + _validate_attempt(record, protocol, slots[slot_id]) + run_id = cast(str, record["run_id"]) + if run_id in run_ids: + raise ValueError("scheduled attempts are duplicated") + run_ids.add(run_id) + by_slot[slot_id].append(record) + if any(not records for records in by_slot.values()): + raise ValueError("scheduled attempts are missing") + selected: list[dict[str, object]] = [] + for slot_id in slots: + ordered = sorted(by_slot[slot_id], key=lambda item: cast(int, item["attempt_sequence"])) + if [item["attempt_sequence"] for item in ordered] != list(range(1, len(ordered) + 1)): + raise ValueError("attempt sequence is invalid") + valid = next((item for item in ordered if item["disposition"] == "valid"), None) + if valid is not None: + selected.append(valid) + return selected + + +def _interval(values: Sequence[float], critical_value: float) -> dict[str, object]: + count = len(values) + if count == 0: + return {"eligible_n": 0, "mean": None, "sample_variance": None, "interval_95": None} + mean = math.fsum(values) / count + if count == 1: + return { + "eligible_n": count, + "mean": mean, + "sample_variance": None, + "interval_95": None, + } + variance = statistics.variance(values, xbar=mean) + half_width = critical_value * math.sqrt(variance / count) + return { + "eligible_n": count, + "mean": mean, + "sample_variance": variance, + "interval_95": [mean - half_width, mean + half_width], + } + + +def _finite_float(value: object) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise ValueError("finite numeric value is required") + return float(value) + + +def _comparison( + declaration: Mapping[str, object], conditions: Sequence[Mapping[str, object]] +) -> dict[str, object]: + published = cast( + dict[str, object], cast(dict[str, object], declaration["comparison"])["published"] + ) + published_by_id = { + cast(str, item["condition_id"]): item + for item in cast(list[dict[str, object]], published["conditions"]) + } + minimum = cast(dict[str, object], declaration["aggregation"])["minimum_eligible_per_condition"] + complete = all(item["eligible_n"] == minimum for item in conditions) + exact = complete + bounded = complete + for observed in conditions: + oracle = published_by_id[cast(str, observed["condition_id"])] + mean = observed["mean"] + if not isinstance(mean, (int, float)) or isinstance(mean, bool): + exact = bounded = False + continue + decimals = cast(int, oracle["display_decimals"]) + exact = exact and round(float(mean), decimals) == _finite_float(oracle["mean"]) + bounded = bounded and abs(float(mean) - _finite_float(oracle["mean"])) <= _finite_float( + oracle["bounded_margin"] + ) + total_mean = math.fsum(cast(float, item["mean"]) for item in conditions) if complete else None + total_variance = None + total_interval = None + if complete and all(item["sample_variance"] is not None for item in conditions): + total_variance = math.fsum( + cast(float, item["sample_variance"]) / cast(int, item["eligible_n"]) + for item in conditions + ) + critical = _finite_float( + cast(dict[str, object], declaration["aggregation"])["critical_value"] + ) + half_width = critical * math.sqrt(total_variance) + total_interval = [ + cast(float, total_mean) - half_width, + cast(float, total_mean) + half_width, + ] + published_interval = cast(list[float], published["primary_total_interval_95"]) + bounded = bounded and ( + total_interval[0] <= _finite_float(published["primary_total"]) <= total_interval[1] + and published_interval[0] <= cast(float, total_mean) <= published_interval[1] + ) + else: + bounded = False + classification = ( + "exact" if exact else "bounded" if bounded else "failed" if complete else "unavailable" + ) + return { + "classification": classification, + "published_result_identity": published["result_identity"], + "observed_primary_total": total_mean, + "observed_primary_variance": total_variance, + "observed_primary_interval_95": total_interval, + "published_primary_total": published["primary_total"], + "published_primary_interval_95": published["primary_total_interval_95"], + } + + +def compute_aggregates( + protocol: Mapping[str, object], attempts: Sequence[Mapping[str, object]] +) -> dict[str, object]: + """Derive the sole deterministic aggregate projection from attempt records.""" + + selected = select_eligible_attempts(protocol, attempts) + slots = _slot_index(protocol) + selected_by_condition: dict[str, list[float]] = {} + for item in selected: + selected_by_condition.setdefault(cast(str, item["condition_id"]), []).append( + _validate_score_measure(item["score_measure"]) + ) + declaration = cast(dict[str, object], protocol["declaration"]) + aggregation = cast(dict[str, object], declaration["aggregation"]) + critical = _finite_float(aggregation["critical_value"]) + conditions: list[dict[str, object]] = [] + for trial_length in cast(list[int], declaration["trial_lengths"]): + for red_variant in cast(list[str], declaration["red_variants"]): + condition_id = _condition_id(trial_length, red_variant) + summary = _interval(sorted(selected_by_condition.get(condition_id, [])), critical) + conditions.append( + { + "condition_id": condition_id, + "trial_length": trial_length, + "red_variant": red_variant, + **summary, + } + ) + counts = Counter(cast(str, item["disposition"]) for item in attempts) + payload: dict[str, object] = { + "declaration_sha256": protocol["declaration_sha256"], + "scheduled_slot_count": len(slots), + "attempt_count": len(attempts), + "selected_valid_count": len(selected), + "disposition_counts": {key: counts.get(key, 0) for key in _DISPOSITIONS}, + "conditions": conditions, + "comparison": _comparison(declaration, conditions), + "method": aggregation, + } + return payload + + +def build_tiers( + protocol: Mapping[str, object], + aggregates: Mapping[str, object], + reference: Mapping[str, object], +) -> dict[str, object]: + """Project the six independent subjects without a bundle-wide pass bit.""" + + comparison = cast(dict[str, object], aggregates["comparison"]) + outcome_class = comparison["classification"] + cyborg_results = { + "authored-source": ("pass", ["protocol.json", "source-ledger.json"]), + "contract": ("pass", ["protocol.json", "runs/"]), + "execution-control": ( + "weakened", + ["source-ledger.json#loss-evaluation-seed-unbound", "protocol.json#seed_policy"], + ), + "state/observation": ( + "weakened", + ["source-ledger.json#loss-native-observation-boundary"], + ), + "outcome/evaluation": ( + "pass" + if outcome_class in {"exact", "bounded"} + else "fail" + if outcome_class == "failed" + else "weakened", + ["aggregates.json#comparison"], + ), + "disclosure": ("pass", ["source-ledger.json#loss_refs", "validation-report.md"]), + } + reference_keys = { + "authored-source": "authored_source", + "contract": "contract", + "execution-control": "execution_control", + "state/observation": "state_observation", + "outcome/evaluation": "outcome_evaluation", + "disclosure": "disclosure", + } + tier_rows = [] + for tier in _TIERS: + cyborg_result, cyborg_evidence = cyborg_results[tier] + reference_result = reference.get(reference_keys[tier]) + if reference_result not in {"pass", "fail", "weakened"}: + raise ValueError("reference tier result is invalid") + tier_rows.append( + { + "tier": tier, + "subjects": { + "cyborg": { + "result": cyborg_result, + "rule": "issue-22-frozen-tier-rule", + "evidence_refs": cyborg_evidence, + }, + "raes-reference": { + "result": reference_result, + "rule": "reference-processor-supported-surface-only", + "evidence_refs": list(cast(Sequence[str], reference["evidence_refs"])), + }, + }, + } + ) + if outcome_class in {"exact", "bounded"}: + strongest = ( + "behavioral-baseline-outcome-reproduction-without-submitted-agent-or-state-equivalence" + ) + research_20 = "usable-behavioral-baseline-outcome-reproduction-evidence" + elif outcome_class == "failed": + strongest = "failed-outcome-reproduction-with-retained-negative-result" + research_20 = "usable-negative-reproduction-result" + else: + strongest = "insufficient-outcome-evidence" + research_20 = "insufficient-for-reproduction-result" + return { + "declaration_sha256": protocol["declaration_sha256"], + "tiers": tier_rows, + "strongest_supported_claim": strongest, + "research_consumers": { + "OpenRAE/research#14": ( + "apparatus-evidence-with-explicit-control-and-observation-losses" + ), + "OpenRAE/research#20": research_20, + }, + "explicit_non_claims": cast(dict[str, object], protocol["declaration"])[ + "explicit_non_claims" + ], + } + + +def _scan_value(value: object) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key in _FORBIDDEN_FIELDS: + raise ValueError("forbidden native material") + _scan_value(item) + elif isinstance(value, list): + for item in value: + _scan_value(item) + elif isinstance(value, str) and any(token in value for token in _FORBIDDEN_TEXT): + raise ValueError("forbidden native material") + + +def scan_public_tree(root: Path) -> None: + """Reject native, host, traceback, symlink, and special-file material.""" + + if root.is_symlink() or not root.is_dir(): + raise ValueError("public tree is invalid") + for path in root.rglob("*"): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise ValueError("public tree is invalid") + if not path.is_file(): + continue + raw = path.read_bytes() + if any(token.encode() in raw for token in _FORBIDDEN_TEXT): + raise ValueError("forbidden native material") + if path.suffix == ".json" or path.name.endswith(".json.gz"): + try: + if path.name.endswith(".json.gz"): + raw = gzip.decompress(raw) + if len(raw) > _MAX_DECOMPRESSED_JSON_BYTES: + raise ValueError(_INVALID_JSON) + value = json.loads( + raw, + object_pairs_hook=_closed_pairs, + parse_constant=_reject_constant, + ) + except (UnicodeError, ValueError, gzip.BadGzipFile) as error: + raise ValueError(_INVALID_JSON) from error + _scan_value(value) + + +def _reserve_directory(path: Path) -> Path: + """Reserve one fresh mode-0700 output root without parent discovery.""" + + resolved = path.resolve() + os.mkdir(resolved, 0o700) + return resolved + + +def _write_json(path: Path, payload: object) -> None: + atomic_write_json_artifact(path, payload) + + +def _write_gzip_json(path: Path, payload: object) -> None: + """Atomically write deterministic compressed canonical JSON.""" + + compressed = gzip.compress(canonical_json_bytes(payload), compresslevel=9, mtime=0) + if len(compressed) >= _MAX_PUBLIC_FILE_BYTES: + raise ValueError("compressed public artifact exceeds the file-size limit") + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(dir=path.parent, suffix=".tmp") + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(compressed) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + with suppress(OSError): + os.unlink(temporary) + raise + + +def _write_text(path: Path, value: str) -> None: + """Atomically write one bounded UTF-8 public text artifact.""" + + content = value.encode("utf-8") + if len(content) >= _MAX_PUBLIC_FILE_BYTES: + raise ValueError("public text artifact exceeds the file-size limit") + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(dir=path.parent, suffix=".tmp") + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except BaseException: + with suppress(OSError): + os.unlink(temporary) + raise + + +def _media_type(path: Path) -> str: + if path.suffix == ".gz": + return "application/gzip" + return "application/json" if path.suffix == ".json" else "text/markdown" + + +def _inventory_entry(root: Path, path: Path) -> dict[str, object]: + if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root.resolve()): + raise ValueError("inventory member is invalid") + content = path.read_bytes() + if len(content) >= _MAX_PUBLIC_FILE_BYTES: + raise ValueError("public artifact exceeds the file-size limit") + return { + "path": path.relative_to(root).as_posix(), + "media_type": _media_type(path), + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + } + + +def _seal_inventory(root: Path, members: Sequence[Path]) -> dict[str, object]: + inventory: dict[str, object] = { + "artifacts": [ + _inventory_entry(root, path) + for path in sorted(members, key=lambda item: item.relative_to(root).as_posix()) + ] + } + _write_json(root / "inventory.json", inventory) + return inventory + + +def _verify_inventory(root: Path, *, expected_members: set[str] | None = None) -> dict[str, object]: + inventory = load_strict_json(root / "inventory.json") + if set(inventory) != {"artifacts"} or not isinstance(inventory["artifacts"], list): + raise ValueError("inventory is invalid") + seen: set[str] = set() + for item in inventory["artifacts"]: + if not isinstance(item, dict) or set(item) != { + "path", + "media_type", + "sha256", + "size_bytes", + }: + raise ValueError("inventory is invalid") + relative = item["path"] + if not isinstance(relative, str): + raise ValueError("inventory is invalid") + source = root / relative + candidate = source.resolve() + if ( + relative in seen + or Path(relative).is_absolute() + or ".." in Path(relative).parts + or not candidate.is_relative_to(root.resolve()) + or source.is_symlink() + or not candidate.is_file() + ): + raise ValueError("inventory is invalid") + seen.add(relative) + expected = _inventory_entry(root, candidate) + if item != expected: + raise ValueError("inventory digest is invalid") + direct_files = { + path.name for path in root.iterdir() if path.is_file() and path.name != "inventory.json" + } + inventoried_direct_files = { + Path(relative).name for relative in seen if len(Path(relative).parts) == 1 + } + if direct_files != inventoried_direct_files: + raise ValueError("inventory membership is incomplete") + if expected_members is not None and seen != expected_members: + raise ValueError("inventory membership is incomplete") + return inventory + + +def _write_schedule_partitions( + root: Path, protocol: Mapping[str, object], selection: ReproductionSelection +) -> list[Path]: + plans = root / "plans" + os.mkdir(plans, 0o700) + by_condition: dict[str, list[dict[str, object]]] = {} + for item in _schedule(selection): + by_condition.setdefault(cast(str, item["condition_id"]), []).append(item) + declaration = cast(dict[str, object], protocol["declaration"]) + written: list[Path] = [] + for declared in cast(list[dict[str, object]], declaration["schedule_partitions"]): + path = root / cast(str, declared["path"]) + entries = by_condition[cast(str, declared["condition_id"])] + if sha256_payload(entries) != declared["sha256"]: + raise ValueError("schedule partition digest is invalid") + _write_json(path, entries) + written.append(path) + return written + + +def write_declaration( + repo_root: Path, + output: Path, + *, + selection: ReproductionSelection = FROZEN_SELECTION, +) -> Path: + """Write one immutable declaration stage with partitioned schedule files.""" + + root = _reserve_directory(output) + protocol, source_ledger = build_declaration(repo_root, selection=selection) + protocol_path = root / "protocol.json" + ledger_path = root / "source-ledger.json" + _write_json(protocol_path, protocol) + _write_json(ledger_path, source_ledger) + plans = _write_schedule_partitions(root, protocol, selection) + _seal_inventory(root, [protocol_path, ledger_path, *plans]) + return root + + +def _load_study_inputs( + repo_root: Path, +) -> tuple[ + object, + ExperimentTaskModel, + ParticipantImplementationManifestModel, + ParticipantImplementationSelectionModel, + ParticipantConfigurationResultModel, +]: + from raes import parse_sdl # type: ignore[import-untyped] + + scenario = parse_sdl( + (repo_root / _INPUT_PATHS["authored-scenario"]).read_text(encoding="utf-8") + ) + task = ExperimentTaskModel.model_validate( + load_strict_json(repo_root / _INPUT_PATHS["experiment-task"]) + ) + manifest = ParticipantImplementationManifestModel.model_validate( + load_strict_json(repo_root / _INPUT_PATHS["blue-manifest"]) + ) + selection = ParticipantImplementationSelectionModel.model_validate( + load_strict_json(repo_root / _INPUT_PATHS["blue-selection"]) + ) + configuration = ParticipantConfigurationResultModel.model_validate( + load_strict_json(repo_root / _INPUT_PATHS["blue-configuration"]) + ) + return scenario, task, manifest, selection, configuration + + +def capture_reference_path(repo_root: Path) -> dict[str, object]: + """Exercise only the authored and contract surfaces of the reference processor.""" + + from raes import parse_sdl + from raes_processor.reference import ReferenceProcessor # type: ignore[import-untyped] + + from .manifest import create_cyborg_manifest + + scenario = parse_sdl( + (repo_root / _INPUT_PATHS["authored-scenario"]).read_text(encoding="utf-8") + ) + result = ReferenceProcessor.realize( + scenario, + create_cyborg_manifest(), + target_name="cyborg-cage2", + ) + errors = sorted({item.code for item in result.diagnostics if item.is_error}) + return { + "processor": "raes-processor-reference", + "scenario_name": result.scenario_name, + "target_manifest": "cyborg-cage2", + "authored_source": "pass" if not errors else "fail", + "contract": "pass" if not errors else "fail", + "execution_control": "weakened", + "state_observation": "weakened", + "outcome_evaluation": "weakened", + "disclosure": "pass", + "planning_error_codes": errors, + "planned_operation_counts": { + "provisioning": len(result.execution_plan.provisioning.operations), + "orchestration": len(result.execution_plan.orchestration.operations), + "evaluation": len(result.execution_plan.evaluation.operations), + }, + "evidence_refs": [ + "reference.json", + "protocol.json#explicit_non_claims", + ], + "limitations": [ + "The reference processor realizes authored contracts and a target plan only.", + "It does not execute CybORG transitions, observations, rewards, or terminal behavior.", + ], + } + + +def _validate_reference(reference: Mapping[str, object]) -> None: + """Validate the bounded reference-processor evidence surface.""" + + _require_exact_keys( + reference, + { + "processor", + "scenario_name", + "target_manifest", + "authored_source", + "contract", + "execution_control", + "state_observation", + "outcome_evaluation", + "disclosure", + "planning_error_codes", + "planned_operation_counts", + "evidence_refs", + "limitations", + }, + "reference evidence", + ) + if ( + reference["processor"] != "raes-processor-reference" + or reference["scenario_name"] != "cage2-research" + or reference["target_manifest"] != "cyborg-cage2" + or reference["execution_control"] != "weakened" + or reference["state_observation"] != "weakened" + or reference["outcome_evaluation"] != "weakened" + or reference["evidence_refs"] != ["reference.json", "protocol.json#explicit_non_claims"] + ): + raise ValueError("reference evidence identity is invalid") + if any(reference[key] not in {"pass", "fail"} for key in ("authored_source", "contract")): + raise ValueError("reference evidence result is invalid") + if reference["disclosure"] != "pass": + raise ValueError("reference evidence result is invalid") + codes = reference["planning_error_codes"] + if not isinstance(codes, list) or any(not isinstance(code, str) or not code for code in codes): + raise ValueError("reference diagnostics are invalid") + counts = reference["planned_operation_counts"] + if ( + not isinstance(counts, dict) + or set(counts) != {"provisioning", "orchestration", "evaluation"} + or any(type(value) is not int or value < 0 for value in counts.values()) + ): + raise ValueError("reference operation counts are invalid") + limitations = reference["limitations"] + if not isinstance(limitations, list) or limitations != [ + "The reference processor realizes authored contracts and a target plan only.", + "It does not execute CybORG transitions, observations, rewards, or terminal behavior.", + ]: + raise ValueError("reference limitations are invalid") + + +def _evidence_step(record: ExperimentEvidenceRecordModel) -> int: + for ref in record.source_refs: + if ref.ref_id.startswith("logical-step:"): + return int(ref.ref_id.removeprefix("logical-step:")) + raise ValueError("evidence logical-step join is missing") + + +def _write_evidence_chunks( + run_root: Path, records: Sequence[ExperimentEvidenceRecordModel] +) -> list[dict[str, object]]: + """Write the exact referenced record closure in bounded deterministic chunks.""" + + payloads = [record.model_dump(mode="json") for record in records] + ordered_steps = [_evidence_step(record) for record in records] + if not payloads or ordered_steps != sorted(ordered_steps): + raise ValueError("episode evidence order is invalid") + evidence_chunks: list[dict[str, object]] = [] + pending: list[tuple[list[dict[str, object]], int, int]] = [ + (payloads, min(ordered_steps), max(ordered_steps)) + ] + chunk_number = 0 + while pending: + chunk_records, first_step, last_step = pending.pop(0) + compressed = gzip.compress(canonical_json_bytes(chunk_records), compresslevel=9, mtime=0) + if len(compressed) >= _MAX_PUBLIC_FILE_BYTES: + if len(chunk_records) < 2: + raise ValueError("one evidence record exceeds the file-size limit") + midpoint = len(chunk_records) // 2 + left = chunk_records[:midpoint] + right = chunk_records[midpoint:] + pending[0:0] = [ + ( + left, + first_step, + _evidence_step(ExperimentEvidenceRecordModel.model_validate(left[-1])), + ), + ( + right, + _evidence_step(ExperimentEvidenceRecordModel.model_validate(right[0])), + last_step, + ), + ] + continue + chunk_number += 1 + path = run_root / f"evidence-{chunk_number:03d}.json.gz" + _write_gzip_json(path, chunk_records) + evidence_chunks.append( + { + "path": path.relative_to(run_root).as_posix(), + "sha256": _sha256_file(path), + "record_count": len(chunk_records), + "first_logical_step": first_step, + "last_logical_step": last_step, + } + ) + return evidence_chunks + + +def _write_episode_evidence( + run_root: Path, + artifact_uri: str, + controls: RunControls, + episode: EpisodeEvidence, + task: ExperimentTaskModel, + scenario_digest: str, +) -> dict[str, object]: + evidence_records = episode.evidence_records + measures = episode.derived_measures + referenced = {ref.ref_id for measure in measures for ref in measure.source_evidence_refs} + retained = [record for record in evidence_records if record.evidence_record_id in referenced] + retained_ids = {item.evidence_record_id for item in retained} + if not retained or referenced != retained_ids: + raise ValueError("episode has no retained evidence closure") + evidence_chunks = _write_evidence_chunks(run_root, retained) + episode_summary = run_root / "episode-summary.json.gz" + _write_gzip_json( + episode_summary, + { + "derived_measures": [item.model_dump(mode="json") for item in measures], + "proposition_truth": episode.proposition_truth_results, + "objective_results": episode.objective_results, + "diagnostics": [item.model_dump(mode="json") for item in episode.diagnostics], + }, + ) + evidence_index = run_root / "evidence-index.json" + _write_json( + evidence_index, + { + "record_count": len(retained), + "chunks": evidence_chunks, + "episode_summary": { + "path": episode_summary.relative_to(run_root).as_posix(), + "sha256": _sha256_file(episode_summary), + }, + }, + ) + index_bytes = evidence_index.read_bytes() + artifact = ExperimentArtifactRefModel( + artifact_id=f"portable-evidence-{controls.run_id}", + role="observation", + media_type="application/json", + uri=artifact_uri, + checksum={"algorithm": "sha256", "value": hashlib.sha256(index_bytes).hexdigest()}, + size_bytes=len(index_bytes), + created_at=retained[0].captured_at, + source="cyborg-cage2 evaluator projection", + satisfies_refs=[ + {"ref_kind": "evidence", "ref_id": "operational-service-state"}, + {"ref_kind": "evidence", "ref_id": "source-ledger:reward-components"}, + ], + sensitivity="redacted", + ) + run = archival_run( + controls=controls, + scenario_digest=scenario_digest, + task=task, + episode=replace(episode, evidence_records=tuple(retained)), + evidence_artifact=artifact, + ) + validate_experiment_run_against_task(task, run) + _write_gzip_json(run_root / "run.json.gz", run.model_dump(mode="json")) + score = next( + ( + item + for item in measures + if item.metric_ref.ref_id == _SCORE_METRIC_ID + and item.measure_kind == "score" + and item.value_status == "reported" + ), + None, + ) + if score is None: + raise ValueError("episode score measure is missing") + _validate_score_measure(score.model_dump(mode="json")) + return { + "score_measure": score.model_dump(mode="json"), + "evidence_refs": [ + "evidence-index.json", + "episode-summary.json.gz", + "run.json.gz", + ], + } + + +def _attempt_record( + protocol: Mapping[str, object], + slot: Mapping[str, object], + *, + attempt_sequence: int, + disposition: str, + cleanup_verified: bool, + score_measure: object = None, + evidence_refs: Sequence[str] = (), + diagnostic_code: str | None = None, +) -> dict[str, object]: + slot_id = cast(str, slot["slot_id"]) + run_id = f"{slot_id}-attempt-{attempt_sequence:02d}" + record = { + "declaration_sha256": protocol["declaration_sha256"], + "slot_id": slot_id, + "run_id": run_id, + "predecessor_run_id": ( + None if attempt_sequence == 1 else f"{slot_id}-attempt-{attempt_sequence - 1:02d}" + ), + "attempt_sequence": attempt_sequence, + "condition_id": slot["condition_id"], + "trial_length": slot["trial_length"], + "red_variant": slot["red_variant"], + "episode_ordinal": slot["episode_ordinal"], + "disposition": disposition, + "diagnostic_code": diagnostic_code, + "cleanup_verified": cleanup_verified, + "score_measure": score_measure, + "evidence_refs": list(evidence_refs), + } + _validate_attempt(record, protocol, slot) + return record + + +def _slot_for_attempt(slot: Mapping[str, object], attempt_sequence: int) -> dict[str, object]: + """Bind one frozen logical slot to a never-reused execution identity.""" + + attempt_slot = dict(slot) + attempt_slot["run_id"] = f"{slot['slot_id']}-attempt-{attempt_sequence:02d}" + attempt_slot["attempt_sequence"] = attempt_sequence + return attempt_slot + + +def _condition_controls( + slots: Sequence[Mapping[str, object]], + protocol: Mapping[str, object], + manifest: ParticipantImplementationManifestModel, + selection: ParticipantImplementationSelectionModel, + configuration: ParticipantConfigurationResultModel, +) -> tuple[RunControls, ...]: + declaration = cast(dict[str, object], protocol["declaration"]) + seed = cast(int, cast(dict[str, object], declaration["seed_policy"])["seed"]) + return tuple( + RunControls( + run_id=cast(str, slot["run_id"]), + seed=seed, + max_steps=cast(int, slot["trial_length"]), + red_variant=cast(str, slot["red_variant"]), + blue_manifest=manifest, + blue_selection=selection, + blue_configuration=configuration, + ) + for slot in slots + ) + + +def _write_environment(protocol: Mapping[str, object]) -> dict[str, object]: + def version(name: str) -> str: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return "not-installed" + + declaration = cast(dict[str, object], protocol["declaration"]) + artifacts = cast(list[dict[str, object]], declaration["artifacts"]) + pack_manifest = next(item for item in artifacts if item["artifact_id"] == "pack-manifest") + contracts = cast(dict[str, object], declaration["published_contracts"]) + task = ExperimentTaskModel.model_validate(contracts["experiment_task"]) + aggregation = cast(dict[str, object], declaration["aggregation"]) + return { + "python": platform.python_version(), + "implementation": platform.python_implementation(), + "operating_system": platform.system(), + "architecture": platform.machine(), + "distributions": { + "raes": version("raes"), + "raes-adapters": version("raes-adapters"), + "CybORG": version("CybORG"), + "gym": version("gym"), + "numpy": version("numpy"), + }, + "source_profile": declaration["source_profile"], + "source_commit": declaration["source_commit"], + "frozen_revision": declaration["frozen_revision"], + "declaration_sha256": protocol["declaration_sha256"], + "pack": { + "content_manifest_sha256": pack_manifest["sha256"], + "scenario_ref_digest": task.scenario_ref.ref_digest, + }, + "method": { + "score_metric_id": aggregation["score_metric_id"], + "condition_estimator": aggregation["condition_estimator"], + "interval": aggregation["interval"], + "seed_policy_sha256": sha256_payload(declaration["seed_policy"]), + }, + "limitations": [ + "No hostname, username, absolute path, argv, environment mapping, or " + "provider log is retained.", + "The qualified source installation is separately acquired and is not bundled.", + ], + } + + +def _validate_environment( + environment: Mapping[str, object], protocol: Mapping[str, object] +) -> None: + """Validate bounded apparatus provenance and its frozen-protocol joins.""" + + expected = _write_environment(protocol) + _require_exact_keys(environment, set(expected), "environment") + for key in ( + "python", + "implementation", + "operating_system", + "architecture", + ): + if not isinstance(environment[key], str) or not environment[key]: + raise ValueError("environment identity is invalid") + distributions = environment["distributions"] + if not isinstance(distributions, dict) or set(distributions) != set( + cast(dict[str, object], expected["distributions"]) + ): + raise ValueError("environment distributions are invalid") + if any(not isinstance(value, str) or not value for value in distributions.values()): + raise ValueError("environment distributions are invalid") + for key in ( + "source_profile", + "source_commit", + "frozen_revision", + "declaration_sha256", + "pack", + "method", + "limitations", + ): + if environment[key] != expected[key]: + raise ValueError("environment protocol join is invalid") + + +def _validation_report_text(tiers: Mapping[str, object]) -> str: + return ( + "# CAGE-2 protocol reproduction validation\n\n" + f"Frozen claim: `{tiers['strongest_supported_claim']}`.\n\n" + "Recompute from a clean installed environment without CybORG or network access:\n\n" + "```shell\n" + "raes-adapters reproduce --phase verify --bundle " + "--output cage2-recomputed\n" + "```\n\n" + "The output is evidence for OpenRAE/research#14 and OpenRAE/research#20. " + "It does not claim deterministic replay, backend identity, submitted-agent " + "artifact identity, state/observation equivalence from score, or scientific " + "reproducibility beyond the retained evidence.\n" + ) + + +def _write_validation_report(root: Path, tiers: Mapping[str, object]) -> Path: + path = root / "validation-report.md" + _write_text(path, _validation_report_text(tiers)) + return path + + +def _condition_inventory(condition_root: Path) -> Path: + members = [condition_root / "cleanup.json"] + members.extend(sorted(condition_root.glob("*/inventory.json"))) + _seal_inventory(condition_root, members) + return condition_root / "inventory.json" + + +def run_full_study( + repo_root: Path, + output: Path, + *, + selection: ReproductionSelection = FROZEN_SELECTION, + driver: object | None = None, + progress: Callable[[str, int, int], None] | None = None, +) -> Path: + """Execute, retain, derive, and seal the complete frozen native study.""" + + source_root = repo_root.resolve() + root = _reserve_directory(output) + protocol, source_ledger = build_declaration(source_root, selection=selection) + protocol_path = root / "protocol.json" + ledger_path = root / "source-ledger.json" + _write_json(protocol_path, protocol) + _write_json(ledger_path, source_ledger) + plan_paths = _write_schedule_partitions(root, protocol, selection) + scenario, task, manifest, participant_selection, configuration = _load_study_inputs(source_root) + reference = capture_reference_path(source_root) + reference_path = root / "reference.json" + _write_json(reference_path, reference) + environment_path = root / "environment.json" + _write_json(environment_path, _write_environment(protocol)) + runtime_driver = cast( + CyborgDriver, + driver if driver is not None else SourceInstalledCyborgDriver(expected_version="2.1"), + ) + begin_stream_value = getattr(runtime_driver, "begin_ordered_stream", None) + checkpoint_stream_value = getattr(runtime_driver, "ordered_stream_checkpoint", None) + restore_stream_value = getattr(runtime_driver, "restore_ordered_stream", None) + if not all( + callable(item) + for item in (begin_stream_value, checkpoint_stream_value, restore_stream_value) + ): + raise ValueError("study driver has no recoverable ordered stream") + begin_stream = cast(Callable[[int], None], begin_stream_value) + checkpoint_stream = cast(Callable[[], object], checkpoint_stream_value) + restore_stream = cast(Callable[[object], None], restore_stream_value) + begin_stream(selection.seed) + runs_root = root / "runs" + os.mkdir(runs_root, 0o700) + all_attempts: list[dict[str, object]] = [] + condition_inventories: list[Path] = [] + schedule = _schedule(selection) + scenario_digest = task.scenario_ref.ref_digest + if not isinstance(scenario_digest, str): + raise ValueError("task scenario digest is unavailable") + for partition in _schedule_partitions(selection): + condition_id = cast(str, partition["condition_id"]) + condition_root = runs_root / condition_id + os.mkdir(condition_root, 0o700) + slots = [item for item in schedule if item["condition_id"] == condition_id] + condition_checkpoint = checkpoint_stream() + cleanup_attempts: list[dict[str, object]] = [] + condition_completed = False + for attempt_sequence in range(1, selection.retry_limit + 2): + if attempt_sequence > 1: + restore_stream(condition_checkpoint) + attempt_slots = [_slot_for_attempt(slot, attempt_sequence) for slot in slots] + staged: dict[str, dict[str, object]] = {} + run_roots: dict[str, Path] = {} + for attempt_slot in attempt_slots: + run_id = cast(str, attempt_slot["run_id"]) + run_root = condition_root / run_id + os.mkdir(run_root, 0o700) + run_roots[run_id] = run_root + + def consume( + controls: RunControls, + episode: EpisodeEvidence, + *, + _staged: dict[str, dict[str, object]] = staged, + _run_roots: dict[str, Path] = run_roots, + _condition_id: str = condition_id, + _slot_count: int = len(slots), + ) -> None: + _staged[controls.run_id] = _write_episode_evidence( + _run_roots[controls.run_id], + (_run_roots[controls.run_id] / "evidence-index.json") + .relative_to(root) + .as_posix(), + controls, + episode, + task, + scenario_digest, + ) + if progress is not None: + progress(_condition_id, len(_staged), _slot_count) + + controls = _condition_controls( + attempt_slots, + protocol, + manifest, + participant_selection, + configuration, + ) + condition_success = False + try: + with ( + Path(os.devnull).open("w", encoding="utf-8") as sink, + redirect_stdout(sink), + redirect_stderr(sink), + ): + execute_episode_series( + scenario, + controls, + driver=runtime_driver, + episode_consumer=consume, + retain_evidence=False, + ) + condition_success = len(staged) == len(slots) + except Exception: + condition_success = False + cleanup_attempts.append( + { + "attempt_sequence": attempt_sequence, + "run_count": len(attempt_slots), + "verified": condition_success, + } + ) + for slot, attempt_slot in zip(slots, attempt_slots, strict=True): + run_id = cast(str, attempt_slot["run_id"]) + run_root = run_roots[run_id] + if condition_success: + stage = staged[run_id] + record = _attempt_record( + protocol, + slot, + attempt_sequence=attempt_sequence, + disposition="valid", + cleanup_verified=True, + score_measure=stage["score_measure"], + evidence_refs=cast(list[str], stage["evidence_refs"]), + ) + else: + for artifact in run_root.iterdir(): + if artifact.is_symlink() or not artifact.is_file(): + raise ValueError("failed attempt artifact boundary is invalid") + artifact.unlink() + record = _attempt_record( + protocol, + slot, + attempt_sequence=attempt_sequence, + disposition="failed", + cleanup_verified=False, + diagnostic_code="cyborg-reproduction.condition-session-failed", + ) + attempt_path = run_root / "attempt.json.gz" + _write_gzip_json(attempt_path, record) + members = [item for item in run_root.rglob("*") if item.is_file()] + _seal_inventory(run_root, members) + all_attempts.append(record) + if condition_success: + condition_completed = True + break + _write_json( + condition_root / "cleanup.json", + { + "condition_id": condition_id, + "completed": condition_completed, + "attempts": cleanup_attempts, + }, + ) + condition_inventories.append(_condition_inventory(condition_root)) + aggregates = compute_aggregates(protocol, all_attempts) + aggregates_path = root / "aggregates.json" + _write_json(aggregates_path, aggregates) + tiers = build_tiers(protocol, aggregates, reference) + tiers_path = root / "tiers.json" + _write_json(tiers_path, tiers) + report_path = _write_validation_report(root, tiers) + scan_public_tree(root) + root_members = [ + protocol_path, + ledger_path, + reference_path, + environment_path, + aggregates_path, + tiers_path, + report_path, + *plan_paths, + *condition_inventories, + ] + _seal_inventory(root, root_members) + return root + + +def compact_bundle_evidence( + root: Path, + *, + progress: Callable[[int, int], None] | None = None, +) -> Path: + """Rewrite a sealed pre-closure bundle to its exact referenced evidence set. + + This deterministic migration exists for a native study that began under the + earlier retain-all implementation. It never changes attempts, measures, + scores, controls, or provenance; it removes only evidence records that no + retained derived measure references, recomputes the deterministic result + projections, then reseals and validates the complete bundle. + """ + + bundle = root.resolve() + protocol = load_strict_json(bundle / "protocol.json") + selection = _selection_from_declaration(cast(dict[str, object], protocol["declaration"])) + validate_protocol(protocol, selection=selection) + _verify_inventory(bundle) + contracts = cast( + dict[str, object], cast(dict[str, object], protocol["declaration"])["published_contracts"] + ) + task = ExperimentTaskModel.model_validate(contracts["experiment_task"]) + slots = _slot_index(protocol) + root_inventory = load_strict_json(bundle / "inventory.json") + inventory_artifacts = cast(list[dict[str, object]], root_inventory["artifacts"]) + root_members = [bundle / cast(str, item["path"]) for item in inventory_artifacts] + condition_roots = sorted(path for path in (bundle / "runs").iterdir() if path.is_dir()) + run_roots = [ + run_root + for condition_root in condition_roots + for run_root in sorted(path for path in condition_root.iterdir() if path.is_dir()) + ] + total = len(run_roots) + completed = 0 + all_attempts: list[dict[str, object]] = [] + for condition_root in condition_roots: + condition_run_roots = sorted(path for path in condition_root.iterdir() if path.is_dir()) + _verify_inventory( + condition_root, + expected_members={"cleanup.json"} + | {f"{run_root.name}/inventory.json" for run_root in condition_run_roots}, + ) + for run_root in condition_run_roots: + _verify_inventory(run_root) + attempt = load_strict_json(run_root / "attempt.json.gz") + slot_id = attempt.get("slot_id") + if not isinstance(slot_id, str) or slot_id not in slots: + raise ValueError("attempt slot join is invalid") + _validate_attempt(attempt, protocol, slots[slot_id]) + all_attempts.append(attempt) + if attempt["disposition"] == "valid": + index = load_strict_json(run_root / "evidence-index.json") + summary_ref = cast(dict[str, object], index["episode_summary"]) + summary_path = run_root / cast(str, summary_ref["path"]) + if _sha256_file(summary_path) != summary_ref["sha256"]: + raise ValueError("episode summary digest is invalid") + summary = load_strict_json(summary_path) + measures = [ + ExperimentDerivedMeasureModel.model_validate(item) + for item in cast(list[dict[str, object]], summary["derived_measures"]) + ] + referenced = { + ref.ref_id for measure in measures for ref in measure.source_evidence_refs + } + records: list[ExperimentEvidenceRecordModel] = [] + for chunk in cast(list[dict[str, object]], index["chunks"]): + chunk_path = run_root / cast(str, chunk["path"]) + if _sha256_file(chunk_path) != chunk["sha256"]: + raise ValueError("evidence chunk digest is invalid") + values = _load_strict_json_value(chunk_path) + if not isinstance(values, list) or len(values) != chunk["record_count"]: + raise ValueError("evidence chunk is invalid") + chunk_records = [ + ExperimentEvidenceRecordModel.model_validate(item) for item in values + ] + steps = [_evidence_step(record) for record in chunk_records] + if ( + not steps + or steps != sorted(steps) + or min(steps) != chunk["first_logical_step"] + or max(steps) != chunk["last_logical_step"] + ): + raise ValueError("evidence chunk ordering is invalid") + records.extend(chunk_records) + if len(records) != index["record_count"]: + raise ValueError("evidence record closure is invalid") + retained = [record for record in records if record.evidence_record_id in referenced] + if {record.evidence_record_id for record in retained} != referenced: + raise ValueError("derived-measure evidence closure is invalid") + for path in run_root.glob("evidence-*.json.gz"): + path.unlink() + chunks = _write_evidence_chunks(run_root, retained) + _write_json( + run_root / "evidence-index.json", + { + "record_count": len(retained), + "chunks": chunks, + "episode_summary": { + "path": summary_path.relative_to(run_root).as_posix(), + "sha256": _sha256_file(summary_path), + }, + }, + ) + index_bytes = (run_root / "evidence-index.json").read_bytes() + run_payload = load_strict_json(run_root / "run.json.gz") + artifacts = cast(list[dict[str, object]], run_payload["evidence_artifacts"]) + if len(artifacts) != 1: + raise ValueError("run evidence artifacts are invalid") + artifacts[0]["checksum"] = { + "algorithm": "sha256", + "value": hashlib.sha256(index_bytes).hexdigest(), + } + artifacts[0]["size_bytes"] = len(index_bytes) + traceability = cast(dict[str, object], run_payload["traceability"]) + traceability["evidence_record_refs"] = [ + item + for item in cast(list[dict[str, object]], traceability["evidence_record_refs"]) + if item["ref_id"] in referenced + ] + run = ExperimentRunModel.model_validate(run_payload) + validate_experiment_run_against_task(task, run) + _write_gzip_json(run_root / "run.json.gz", run.model_dump(mode="json")) + _seal_inventory( + run_root, + [ + path + for path in run_root.iterdir() + if path.is_file() and path.name != "inventory.json" + ], + ) + completed += 1 + if progress is not None: + progress(completed, total) + _condition_inventory(condition_root) + aggregates = compute_aggregates(protocol, all_attempts) + _write_json(bundle / "aggregates.json", aggregates) + reference = load_strict_json(bundle / "reference.json") + _validate_reference(reference) + tiers = build_tiers(protocol, aggregates, reference) + _write_json(bundle / "tiers.json", tiers) + _write_validation_report(bundle, tiers) + _seal_inventory(bundle, root_members) + verify_bundle(bundle, selection=selection) + return bundle + + +def _verify_schedule_partitions(root: Path, protocol: Mapping[str, object]) -> None: + declaration = cast(dict[str, object], protocol["declaration"]) + expected = _slot_index(protocol) + observed: list[dict[str, object]] = [] + expected_paths = { + cast(str, partition["path"]) + for partition in cast(list[dict[str, object]], declaration["schedule_partitions"]) + } + plans_root = root / "plans" + actual_paths = { + path.relative_to(root).as_posix() + for path in plans_root.iterdir() + if path.is_file() and not path.is_symlink() + } + if actual_paths != expected_paths or any(not path.is_file() for path in plans_root.iterdir()): + raise ValueError("schedule partition membership is invalid") + for partition in cast(list[dict[str, object]], declaration["schedule_partitions"]): + path = root / cast(str, partition["path"]) + value = _load_strict_json_value(path) + if not isinstance(value, list) or sha256_payload(value) != partition["sha256"]: + raise ValueError("schedule partition digest is invalid") + observed.extend(cast(list[dict[str, object]], value)) + if observed != list(expected.values()): + raise ValueError("schedule partitions do not join the protocol") + + +def _valid_run_evidence( + bundle: Path, + run_root: Path, + attempt: Mapping[str, object], + task: ExperimentTaskModel, +) -> None: + """Validate the sealed portable evidence closure for one eligible attempt.""" + + index_path = run_root / "evidence-index.json" + index = load_strict_json(index_path) + _require_exact_keys(index, {"record_count", "chunks", "episode_summary"}, "evidence index") + chunks = index["chunks"] + if not isinstance(chunks, list) or not chunks: + raise ValueError("evidence index is invalid") + record_ids: list[str] = [] + for chunk in chunks: + if not isinstance(chunk, dict): + raise ValueError("evidence index is invalid") + _require_exact_keys( + chunk, + { + "path", + "sha256", + "record_count", + "first_logical_step", + "last_logical_step", + }, + "evidence chunk", + ) + relative = chunk["path"] + if not isinstance(relative, str) or Path(relative).name != relative: + raise ValueError("evidence chunk path is invalid") + path = run_root / relative + if _sha256_file(path) != chunk["sha256"]: + raise ValueError("evidence chunk digest is invalid") + values = _load_strict_json_value(path) + if not isinstance(values, list) or len(values) != chunk["record_count"]: + raise ValueError("evidence chunk is invalid") + records = [ExperimentEvidenceRecordModel.model_validate(item) for item in values] + steps = [_evidence_step(record) for record in records] + if ( + not steps + or steps != sorted(steps) + or min(steps) != chunk["first_logical_step"] + or max(steps) != chunk["last_logical_step"] + ): + raise ValueError("evidence chunk ordering is invalid") + record_ids.extend(record.evidence_record_id for record in records) + if len(record_ids) != index["record_count"] or len(record_ids) != len(set(record_ids)): + raise ValueError("evidence record closure is invalid") + + summary_ref = index["episode_summary"] + if not isinstance(summary_ref, dict): + raise ValueError("episode summary reference is invalid") + _require_exact_keys(summary_ref, {"path", "sha256"}, "episode summary reference") + summary_relative = summary_ref["path"] + if not isinstance(summary_relative, str) or Path(summary_relative).name != summary_relative: + raise ValueError("episode summary path is invalid") + summary_path = run_root / summary_relative + if _sha256_file(summary_path) != summary_ref["sha256"]: + raise ValueError("episode summary digest is invalid") + summary = load_strict_json(summary_path) + _require_exact_keys( + summary, + {"derived_measures", "proposition_truth", "objective_results", "diagnostics"}, + "episode summary", + ) + measure_values = summary["derived_measures"] + if not isinstance(measure_values, list) or not measure_values: + raise ValueError("derived measures are invalid") + measures = [ExperimentDerivedMeasureModel.model_validate(item) for item in measure_values] + known_records = set(record_ids) + referenced_records = { + ref.ref_id for measure in measures for ref in measure.source_evidence_refs + } + if referenced_records != known_records: + raise ValueError("derived-measure evidence closure is invalid") + score = next( + ( + item + for item in measures + if item.metric_ref.ref_id == _SCORE_METRIC_ID + and item.measure_kind == "score" + and item.value_status == "reported" + ), + None, + ) + if score is None or canonical_json_bytes(score.model_dump(mode="json")) != canonical_json_bytes( + attempt["score_measure"] + ): + raise ValueError("attempt score does not join the episode evidence") + + run_path = run_root / "run.json.gz" + run_payload = load_strict_json(run_path) + run = ExperimentRunModel.model_validate(run_payload) + validate_experiment_run_against_task(task, run) + traceability = cast(dict[str, object], run_payload["traceability"]) + trace_record_refs = cast(list[dict[str, object]], traceability["evidence_record_refs"]) + trace_measure_refs = cast(list[dict[str, object]], traceability["derived_measure_refs"]) + if {cast(str, item["ref_id"]) for item in trace_record_refs} != known_records: + raise ValueError("run evidence traceability is incomplete") + if {cast(str, item["ref_id"]) for item in trace_measure_refs} != { + item.derived_measure_id for item in measures + }: + raise ValueError("run measure traceability is incomplete") + artifact_relative = index_path.relative_to(bundle).as_posix() + artifacts = cast(list[dict[str, object]], run_payload["evidence_artifacts"]) + if len(artifacts) != 1 or artifacts[0].get("uri") != artifact_relative: + raise ValueError("run evidence artifact path is invalid") + checksum = artifacts[0].get("checksum") + if not isinstance(checksum, dict) or checksum.get("value") != _sha256_file(index_path): + raise ValueError("run evidence artifact digest is invalid") + result_summaries = cast(dict[str, dict[str, object]], run_payload["result_summaries"]) + result = result_summaries.get("cage2-cumulative-blue-reward-result") + if result is None or float(cast(float, result.get("value"))) != _validate_score_measure( + attempt["score_measure"] + ): + raise ValueError("run score summary is invalid") + + +def _load_attempts(root: Path, protocol: Mapping[str, object]) -> list[dict[str, object]]: + attempts: list[dict[str, object]] = [] + slots = _slot_index(protocol) + declaration = cast(dict[str, object], protocol["declaration"]) + contracts = cast(dict[str, object], declaration["published_contracts"]) + task = ExperimentTaskModel.model_validate(contracts["experiment_task"]) + runs_root = root / "runs" + expected_conditions = {cast(str, slot["condition_id"]) for slot in slots.values()} + condition_roots = sorted(path for path in runs_root.iterdir() if path.is_dir()) + if ( + any(path.is_symlink() or not path.is_dir() for path in runs_root.iterdir()) + or {path.name for path in condition_roots} != expected_conditions + ): + raise ValueError("run condition membership is invalid") + for condition_root in condition_roots: + run_roots = sorted(path for path in condition_root.iterdir() if path.is_dir()) + _verify_inventory( + condition_root, + expected_members={"cleanup.json"} + | {f"{run_root.name}/inventory.json" for run_root in run_roots}, + ) + for run_root in run_roots: + if run_root.is_symlink() or any(path.is_dir() for path in run_root.iterdir()): + raise ValueError("run artifact membership is invalid") + _verify_inventory(run_root) + attempt = load_strict_json(run_root / "attempt.json.gz") + slot_id = attempt.get("slot_id") + if not isinstance(slot_id, str) or slot_id not in slots: + raise ValueError("attempt slot join is invalid") + _validate_attempt(attempt, protocol, slots[slot_id]) + if attempt["disposition"] == "valid": + _valid_run_evidence(root, run_root, attempt, task) + attempts.append(attempt) + return attempts + + +def verify_bundle( + root: Path, + *, + selection: ReproductionSelection = FROZEN_SELECTION, +) -> dict[str, object]: + """Offline-verify every transitive inventory and recompute the frozen result.""" + + bundle = root.resolve() + expected_root_directories = {"plans", "runs"} + if ( + bundle.is_symlink() + or {path.name for path in bundle.iterdir() if path.is_dir()} != expected_root_directories + ): + raise ValueError("bundle directory membership is invalid") + protocol = load_strict_json(bundle / "protocol.json") + validate_protocol(protocol, selection=selection) + declaration = cast(dict[str, object], protocol["declaration"]) + conditions = {cast(str, slot["condition_id"]) for slot in _slot_index(protocol).values()} + _verify_inventory( + bundle, + expected_members={ + "protocol.json", + "source-ledger.json", + "reference.json", + "environment.json", + "aggregates.json", + "tiers.json", + "validation-report.md", + *{ + cast(str, partition["path"]) + for partition in cast(list[dict[str, object]], declaration["schedule_partitions"]) + }, + *{f"runs/{condition}/inventory.json" for condition in conditions}, + }, + ) + ledger = load_strict_json(bundle / "source-ledger.json") + _validate_source_ledger(ledger, protocol) + _validate_environment(load_strict_json(bundle / "environment.json"), protocol) + _verify_schedule_partitions(bundle, protocol) + attempts = _load_attempts(bundle, protocol) + aggregates = compute_aggregates(protocol, attempts) + if canonical_json_bytes(aggregates) != canonical_json_bytes( + load_strict_json(bundle / "aggregates.json") + ): + raise ValueError("aggregate recomputation does not match") + reference = load_strict_json(bundle / "reference.json") + _validate_reference(reference) + tiers = build_tiers(protocol, aggregates, reference) + if canonical_json_bytes(tiers) != canonical_json_bytes(load_strict_json(bundle / "tiers.json")): + raise ValueError("tier recomputation does not match") + try: + report = (bundle / "validation-report.md").read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise ValueError("validation report is invalid") from error + if report != _validation_report_text(tiers): + raise ValueError("validation report does not match") + scan_public_tree(bundle) + return { + "declaration_sha256": protocol["declaration_sha256"], + "scheduled_slot_count": aggregates["scheduled_slot_count"], + "attempt_count": aggregates["attempt_count"], + "classification": cast(dict[str, object], aggregates["comparison"])["classification"], + "strongest_supported_claim": tiers["strongest_supported_claim"], + } + + +def recompute_bundle( + bundle: Path, + output: Path, + *, + selection: ReproductionSelection = FROZEN_SELECTION, +) -> Path: + """Verify a frozen bundle and write a fresh clean-environment result.""" + + result = verify_bundle(bundle, selection=selection) + root = _reserve_directory(output) + result_path = root / "verification.json" + _write_json(result_path, result) + _seal_inventory(root, [result_path]) + return root + + +__all__ = [ + "FROZEN_SELECTION", + "ReproductionSelection", + "attempt_record_for_test", + "build_declaration", + "build_tiers", + "canonical_json_bytes", + "compute_aggregates", + "compact_bundle_evidence", + "load_strict_json", + "capture_reference_path", + "recompute_bundle", + "retry_record_for_test", + "scan_public_tree", + "schedule_for_test", + "select_eligible_attempts", + "sha256_payload", + "run_full_study", + "validate_protocol", + "verify_bundle", + "write_declaration", +] diff --git a/src/raes_adapters/cyborg/researcher.py b/src/raes_adapters/cyborg/researcher.py index f0d83f0..2a0c31e 100644 --- a/src/raes_adapters/cyborg/researcher.py +++ b/src/raes_adapters/cyborg/researcher.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass from raes_contracts.contracts import ( # type: ignore[import-untyped] @@ -332,6 +332,10 @@ def _start_runtime( diagnostics.extend(diagnostic_model(item) for item in started.diagnostics) if not started.success or target.participant_runtime is None or target.evaluator is None: raise RuntimeError("researcher orchestration admission failed") + begin_run = getattr(target.orchestrator, "begin_run", None) + if not callable(begin_run): + raise RuntimeError("researcher run identity admission failed") + begin_run(controls.run_id) evaluation = planned.evaluation if not evaluation.resources: evaluation = cage2_evaluation_plan() @@ -459,6 +463,129 @@ def execute_episode( ) +def _begin_independent_run( + target: RuntimeTarget, + baseline: RuntimeSnapshot, + controls: RunControls, + diagnostics: list[DiagnosticModel], +) -> RuntimeSnapshot: + """Reset one retained native session and start a fresh portable run.""" + + if target.participant_runtime is None or target.orchestrator is None: + raise RuntimeError("researcher reset runtime is unavailable") + begin = getattr(target.participant_runtime, "begin_independent_run", None) + if not callable(begin) or begin() is not True: + raise RuntimeError("researcher coordinated reset failed") + begin_run = getattr(target.orchestrator, "begin_run", None) + if not callable(begin_run): + raise RuntimeError("researcher run identity admission failed") + begin_run(controls.run_id) + return _initialize_participants(target, baseline, controls, diagnostics) + + +def execute_episode_series( + scenario: object, + controls: Sequence[RunControls], + *, + driver: CyborgDriver, + episode_consumer: Callable[[RunControls, EpisodeEvidence], None] | None = None, + retain_evidence: bool = True, +) -> tuple[EpisodeEvidence, ...]: + """Execute one ordered condition series on a retained native session. + + The supplied driver must already own the frozen study-scoped stream. The + target therefore constructs without a per-episode seed and coordinated + resets continue, rather than restart, that stream. + """ + + if not controls: + raise ValueError("researcher condition series is empty") + first = controls[0] + if any( + item.seed != first.seed + or item.max_steps != first.max_steps + or item.red_variant != first.red_variant + for item in controls + ): + raise ValueError("researcher condition series controls are inconsistent") + target = create_cyborg_target( + scenario=scenario, + seed=None, + reseed_on_reset=False, + driver=driver, + ) + baseline, startup_diagnostics, evaluation_plan = _start_runtime(target, scenario, first) + snapshot = _initialize_participants(target, baseline, first, startup_diagnostics) + captured: list[ + tuple[ + int, + tuple[ExperimentEvidenceRecordModel, ...], + tuple[ExperimentDerivedMeasureModel, ...], + dict[str, dict[str, object]], + dict[str, dict[str, object]], + tuple[DiagnosticModel, ...], + ] + ] = [] + cleanup_verified = False + try: + for index, item in enumerate(controls): + diagnostics = list(startup_diagnostics) if index == 0 else [] + if index: + snapshot = _begin_independent_run(target, baseline, item, diagnostics) + snapshot, completed_steps = _execute_steps(target, snapshot, item, diagnostics) + snapshot, records, measures = _evaluate_episode( + target, snapshot, diagnostics, evaluation_plan + ) + captured.append( + ( + completed_steps, + records, + measures, + { + address: dict(payload) + for address, payload in snapshot.proposition_truth_results.items() + }, + { + address: dict(payload) + for address, payload in snapshot.evaluation_results.items() + }, + tuple(diagnostics), + ) + ) + if episode_consumer is not None: + episode_consumer( + item, + EpisodeEvidence( + completed_steps=completed_steps, + evidence_records=records, + derived_measures=measures, + proposition_truth_results=captured[-1][3], + objective_results=captured[-1][4], + diagnostics=tuple(diagnostics), + cleanup_verified=False, + ), + ) + if not retain_evidence: + captured.pop() + finally: + cleanup = RuntimeManager(target, initial_snapshot=snapshot).destroy() + cleanup_verified = cleanup.success + if not cleanup_verified: + raise RuntimeError("researcher cleanup failed") + return tuple( + EpisodeEvidence( + completed_steps=completed, + evidence_records=records, + derived_measures=measures, + proposition_truth_results=truth, + objective_results=objectives, + diagnostics=diagnostics, + cleanup_verified=True, + ) + for completed, records, measures, truth, objectives, diagnostics in captured + ) + + __all__ = [ "EpisodeEvidence", "RunControls", @@ -466,4 +593,5 @@ def execute_episode( "archival_run", "blue_implementation_provenance", "execute_episode", + "execute_episode_series", ] diff --git a/src/raes_adapters/cyborg/scenario.py b/src/raes_adapters/cyborg/scenario.py index 0402a24..15701f2 100644 --- a/src/raes_adapters/cyborg/scenario.py +++ b/src/raes_adapters/cyborg/scenario.py @@ -44,6 +44,42 @@ "user-3": ("User3", "linux_user_host1"), "user-4": ("User4", "linux_user_host2"), } +_SCENARIO2_NATIVE_SUBNET_HOST_ORDER = { + "Enterprise": ("Enterprise0", "Enterprise1", "Enterprise2", "Defender"), + "Operational": ("Op_Server0", "Op_Host0", "Op_Host1", "Op_Host2"), + "User": ("User0", "User1", "User2", "User3", "User4"), +} +_SCENARIO2_BLUE_SESSION_HOST_ORDER = ( + "User0", + "User1", + "User2", + "User3", + "User4", + "Enterprise0", + "Enterprise1", + "Enterprise2", + "Defender", + "Op_Server0", + "Op_Host0", + "Op_Host1", + "Op_Host2", +) +_SCENARIO2_RED_NATIVE_ACTION_ORDER = ( + "Sleep", + "DiscoverRemoteSystems", + "DiscoverNetworkServices", + "ExploitRemoteService", + "BlueKeep", + "EternalBlue", + "FTPDirectoryTraversal", + "HarakaRCE", + "HTTPRFI", + "HTTPSRFI", + "SQLInjection", + "PrivilegeEscalate", + "Impact", + "SSHBruteForce", +) _SCENARIO2_BLUE_PARTICIPANT = "participant.behavior.blue" _SCENARIO2_GREEN_PARTICIPANT = "participant.behavior.green" _SCENARIO2_RED_PARTICIPANT = "participant.behavior.red" @@ -661,6 +697,14 @@ def _translate_scenario2( members = subnet["Hosts"] assert isinstance(members, list) members.append(native_name) + for subnet_name, source_order in _SCENARIO2_NATIVE_SUBNET_HOST_ORDER.items(): + subnet = subnets[subnet_name] + assert isinstance(subnet, dict) + members = subnet["Hosts"] + assert isinstance(members, list) + if set(members) != set(source_order): + raise ValueError("CybORG Scenario2 subnet membership is invalid.") + subnet["Hosts"] = list(source_order) return { "Agents": _scenario2_agent_declarations(scenario_binding), "Subnets": subnets, @@ -749,11 +793,16 @@ def _scenario2_native_actions( participants = {participant.address: participant for participant in binding.participants} participant = participants[participant_address] - return [ + actions = [ native for contract in participant.action_contract_addresses for native in _SCENARIO2_NATIVE_ACTIONS_BY_CONTRACT[contract] ] + if participant_address == _SCENARIO2_RED_PARTICIPANT: + if set(actions) != set(_SCENARIO2_RED_NATIVE_ACTION_ORDER): + raise ValueError("CybORG Scenario2 Red action closure is invalid.") + return list(_SCENARIO2_RED_NATIVE_ACTION_ORDER) + return actions def _scenario2_participant( @@ -829,6 +878,13 @@ def _scenario2_agent_declarations( for hostname in green_known_hosts } blue_accounts = _scenario2_account_sessions(binding, _SCENARIO2_BLUE_PARTICIPANT) + blue_account_by_host = dict(blue_accounts) + if set(blue_account_by_host) != set(_SCENARIO2_BLUE_SESSION_HOST_ORDER): + raise ValueError("CybORG Scenario2 Blue session closure is invalid.") + blue_accounts = [ + (hostname, blue_account_by_host[hostname]) + for hostname in _SCENARIO2_BLUE_SESSION_HOST_ORDER + ] blue_clients: list[dict[str, object]] = [ { "hostname": native, diff --git a/src/raes_adapters/cyborg/target.py b/src/raes_adapters/cyborg/target.py index 370943d..3e72e85 100644 --- a/src/raes_adapters/cyborg/target.py +++ b/src/raes_adapters/cyborg/target.py @@ -30,6 +30,7 @@ "driver", "mapping_ledger_resource", "qualification_profile_id", + "reseed_on_reset", "scenario", "seed", "simulator_version", @@ -62,6 +63,7 @@ def _normalized_config(config: dict[str, object]) -> dict[str, object]: CAGE2_SOURCE_26CE1C1.ledger_resource, ), "seed": config.get("seed"), + "reseed_on_reset": config.get("reseed_on_reset", True), "driver": config.get("driver"), "scenario": config.get("scenario"), } @@ -99,6 +101,8 @@ def _validate_selection( seed = config["seed"] if seed is not None and (type(seed) is not int or not 0 <= seed <= 0xFFFFFFFF): raise ValueError("CybORG seed must be an unsigned 32-bit integer.") + if type(config["reseed_on_reset"]) is not bool: + raise ValueError("CybORG reset seed policy must be boolean.") driver = config["driver"] if driver is not None and ( not callable(getattr(driver, "construct", None)) @@ -142,6 +146,7 @@ def create_cyborg_components( profile_id=str(normalized["qualification_profile_id"]), source_commit=str(normalized["source_commit"]), seed=normalized["seed"] if isinstance(normalized["seed"], int) else None, + reseed_on_reset=cast(bool, normalized["reseed_on_reset"]), scenario_binding=( bind_scenario_profile( normalized["scenario"], diff --git a/tests/test_cyborg_execution.py b/tests/test_cyborg_execution.py index e2c781a..03b9892 100644 --- a/tests/test_cyborg_execution.py +++ b/tests/test_cyborg_execution.py @@ -354,8 +354,13 @@ def _prepared_target( *, max_steps: int, red_variant: str = "sleep", + reseed_on_reset: bool = True, ) -> tuple[Any, RuntimeSnapshot]: - target = create_cyborg_target(driver=driver, seed=7) + target = create_cyborg_target( + driver=driver, + seed=7, + reseed_on_reset=reseed_on_reset, + ) assert target.orchestrator is not None assert target.participant_runtime is not None assert target.time_runtime is not None @@ -627,11 +632,13 @@ def test_evaluator_joins_supported_objective_to_typed_evidence_and_measure() -> measures = evaluator.derived_measures() assert records assert all(isinstance(item, ExperimentEvidenceRecordModel) for item in records) - assert len(measures) == 1 - assert isinstance(measures[0], ExperimentDerivedMeasureModel) - assert measures[0].value == -0.1 - assert measures[0].measure_kind == "score" - assert "not a conformance" in measures[0].limitations[0] + assert all(isinstance(item, ExperimentDerivedMeasureModel) for item in measures) + measure_values = {item.metric_ref.ref_id: item.value for item in measures} + assert measure_values["cage2-cumulative-blue-confidentiality-reward"] == -0.1 + assert measures[-1].value == -0.1 + assert measures[-1].measure_kind == "score" + assert measures[-1].metric_ref.ref_id == "cage2-cumulative-blue-reward" + assert "not a conformance" in measures[-1].limitations[0] joined = str([item.model_dump(mode="json") for item in records]) for expected in ( f"{_WORKFLOW}-run", @@ -1020,6 +1027,33 @@ def test_coordinated_reset_resets_the_aggregate_once() -> None: f"{_GREEN}-episode-2", f"{_RED}-episode-2", } + + +def test_study_scoped_reset_does_not_reseed_each_episode() -> None: + driver = FakeExecutionDriver() + target, snapshot = _prepared_target( + driver, + max_steps=1, + reseed_on_reset=False, + ) + assert target.participant_runtime is not None + exhausted = target.participant_runtime.admit_action(_request(), snapshot) + assert exhausted.success + + reset = target.participant_runtime.reset_many( + tuple( + ParticipantEpisodeResetRequest( + participant_address=address, + episode_id=f"{address}-episode-2", + ) + for address in (_BLUE, _GREEN, _RED) + ), + exhausted.snapshot, + ) + + assert reset.success + assert driver.resets == 1 + assert driver.seed is None assert { value["previous_episode_id"] for value in reset.snapshot.participant_episode_results.values() @@ -1318,6 +1352,48 @@ def get_last_action(self, agent: str) -> object: ) +def test_source_driver_ordered_stream_continues_across_unseeded_resets() -> None: + class NativeHandle: + def __init__(self) -> None: + self.reset_draws: list[float] = [] + + def reset(self) -> object: + self.reset_draws.append(random.random()) + return object() + + def set_seed(self, seed: int) -> None: + random.seed(seed) + + def shutdown(self) -> None: + return None + + source_driver = SourceInstalledCyborgDriver(expected_version="2.1") + source_driver.begin_ordered_stream(153) + initial = source_driver.ordered_stream_checkpoint() + native = NativeHandle() + source_driver._random_states[id(native)] = initial + random.seed(99) + caller_state = random.getstate() + + assert source_driver.reset(native, seed=None) + assert source_driver.reset(native, seed=None) + + expected = random.Random(153) + assert native.reset_draws == [expected.random(), expected.random()] + assert random.getstate() == caller_state + + +def test_source_driver_ordered_stream_checkpoint_can_discard_provisional_construction() -> None: + source_driver = SourceInstalledCyborgDriver(expected_version="2.1") + source_driver.begin_ordered_stream(153) + checkpoint = source_driver.ordered_stream_checkpoint() + source_driver.restore_ordered_stream(random.Random(7).getstate()) + + source_driver.restore_ordered_stream(checkpoint) + + assert source_driver.ordered_stream_checkpoint() == checkpoint + + def test_source_driver_projects_actual_dict_observation_shape( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1449,6 +1525,37 @@ def get_reward_breakdown(self, agent: str) -> dict[str, object]: ) +def test_source_driver_accepts_bounded_reward_reconciliation_roundoff() -> None: + class NativeHandle: + def get_rewards(self) -> dict[str, float]: + return {"Blue": -0.3, "Green": 0.0, "Red": 0.3} + + def get_reward_breakdown(self, agent: str) -> dict[str, object]: + sign = -1.0 if agent == "Blue" else 1.0 + return { + "User0": SimpleNamespace(confidentiality=sign * 0.1, availability=0.0), + "User1": SimpleNamespace(confidentiality=sign * 0.2, availability=0.0), + } + + projected = SourceInstalledCyborgDriver._project_evaluation_turn( + NativeHandle(), + _NativeEvaluationContext( + external_address=_SLEEP, + host_addresses={ + "User0": "provision.node.user-0", + "User1": "provision.node.user-1", + }, + run_id="run-roundoff", + episode_id="episode-roundoff", + action_instance_id="action-roundoff", + logical_step=1, + terminal_cause="logical-step-limit", + ), + ) + + assert all(item.component != "action-cost" for item in projected.components) + + @pytest.mark.parametrize( ("rewards", "blue_breakdown", "red_breakdown", "external_address"), [ diff --git a/tests/test_cyborg_reproduction.py b/tests/test_cyborg_reproduction.py new file mode 100644 index 0000000..5bb7029 --- /dev/null +++ b/tests/test_cyborg_reproduction.py @@ -0,0 +1,564 @@ +from __future__ import annotations + +import copy +import json +import math +from pathlib import Path + +import pytest + +from raes_adapters import cli +from raes_adapters.cyborg import reproduction +from raes_adapters.cyborg.driver import ( + _NativeEvaluationContext, + _NativeEvaluationTurn, + _NativeParticipantOccurrence, + _NativeRewardComponent, + _NativeTurnResult, +) + +PROJECT_ROOT = Path(__file__).parents[1] + + +class StudyDriver: + """Bounded native seam for the real scheduler and persistence path.""" + + def __init__(self) -> None: + self.handles: list[object] = [] + self.red_variant = "sleep" + self.stream = object() + + def begin_ordered_stream(self, seed: int) -> None: + assert seed == 153 + + def ordered_stream_checkpoint(self) -> object: + return self.stream + + def restore_ordered_stream(self, checkpoint: object) -> None: + assert checkpoint is self.stream + + def construct(self, descriptor: object, *, seed: int | None) -> object: + del descriptor + assert seed is None + handle = object() + self.handles.append(handle) + return handle + + def construct_execution( + self, + descriptor: object, + *, + seed: int | None, + red_variant: str, + ) -> object: + self.red_variant = red_variant + return self.construct(descriptor, seed=seed) + + def reset(self, handle: object, *, seed: int | None) -> bool: + return handle in self.handles and seed is None + + def step(self, handle: object, selection: object) -> _NativeTurnResult: + assert handle in self.handles + red = { + "b-line": "participant.action-contract.discover-network-services", + "meander": "participant.action-contract.discover-remote-systems", + "sleep": "participant.action-contract.sleep", + }[self.red_variant] + return _NativeTurnResult( + external_action_succeeded=True, + source_terminal=False, + occurrences=( + _NativeParticipantOccurrence( + "participant.behavior.blue", selection.action_contract_address + ), + _NativeParticipantOccurrence( + "participant.behavior.green", + "participant.action-contract.green-port-scan", + ), + _NativeParticipantOccurrence("participant.behavior.red", red), + ), + ) + + def project_evaluation( + self, handle: object, context: _NativeEvaluationContext + ) -> _NativeEvaluationTurn: + assert handle in self.handles + return _NativeEvaluationTurn( + run_id=context.run_id, + episode_id=context.episode_id, + action_instance_id=context.action_instance_id, + logical_step=context.logical_step, + terminal_cause=context.terminal_cause, + rewards=( + ("participant.behavior.blue", -1.0), + ("participant.behavior.green", 0.0), + ("participant.behavior.red", 1.0), + ), + components=( + _NativeRewardComponent( + "participant.behavior.blue", + "provision.node.op-server-0", + "availability", + 0.0, + "source-ledger:reward-components", + ), + ), + ) + + def cleanup(self, handle: object) -> bool: + self.handles.remove(handle) + return True + + +class FailingStudyDriver(StudyDriver): + """Fail after one complete episode so staged evidence must be discarded.""" + + def __init__(self) -> None: + super().__init__() + self.step_calls = 0 + + def step(self, handle: object, selection: object) -> _NativeTurnResult: + self.step_calls += 1 + if self.step_calls == 31: + raise RuntimeError("test native failure") + return super().step(handle, selection) + + +class ExhaustedRetryStudyDriver(StudyDriver): + """Exhaust the first condition's retries, then allow later conditions.""" + + def __init__(self) -> None: + super().__init__() + self.session_count = 0 + + def construct_execution( + self, + descriptor: object, + *, + seed: int | None, + red_variant: str, + ) -> object: + self.session_count += 1 + return super().construct_execution( + descriptor, + seed=seed, + red_variant=red_variant, + ) + + def step(self, handle: object, selection: object) -> _NativeTurnResult: + if self.session_count <= 2: + raise RuntimeError("test native failure") + return super().step(handle, selection) + + +def _selection(*, episodes: int = 2) -> reproduction.ReproductionSelection: + return reproduction.FROZEN_SELECTION.with_cardinality_for_test(episodes) + + +def _valid_attempts(protocol: dict[str, object], *, offset: float = 0.0) -> list[dict[str, object]]: + declaration = protocol["declaration"] + assert isinstance(declaration, dict) + schedule = reproduction.schedule_for_test(protocol) + return [ + reproduction.attempt_record_for_test( + protocol, + slot, + score=float(index) + offset, + ) + for index, slot in enumerate(schedule, start=1) + ] + + +def test_declaration_freezes_full_matrix_and_separates_source_facts() -> None: + protocol, source_ledger = reproduction.build_declaration(PROJECT_ROOT, selection=_selection()) + + reproduction.validate_protocol(protocol, selection=_selection()) + declaration = protocol["declaration"] + assert isinstance(declaration, dict) + schedule = reproduction.schedule_for_test(protocol) + assert len(schedule) == 18 + assert len({item["slot_id"] for item in schedule}) == 18 + assert len({item["run_id"] for item in schedule}) == 18 + assert {item["trial_length"] for item in schedule} == {30, 50, 100} + assert {item["red_variant"] for item in schedule} == { + "b-line", + "meander", + "sleep", + } + assert declaration["seed_policy"] == { + "api": "random.seed", + "initialization_scope": "study", + "owner": "python-random", + "reset_behavior": "continue-stream-across-episode-reset", + "seed": 153, + "serialization": "trial-length-then-red-variant-then-episode", + } + source_facts = source_ledger["protocol_source_facts"] + assert source_facts["qualified_evaluator"]["episode_count"] == 100 + assert source_facts["public_validation"]["episode_count"] == 1000 + assert source_facts["qualified_evaluator"]["seed_binding"] is None + assert source_facts["public_validation"]["seed_statement"] == "random.seed(153)" + assert "schema_version" not in protocol + assert "schema_version" not in source_ledger + + +@pytest.mark.parametrize( + "payload", + [ + '{"schema_version":"x","schema_version":"y"}', + '{"value":NaN}', + '{"value":Infinity}', + ], +) +def test_strict_json_rejects_duplicates_and_nonfinite_values(tmp_path: Path, payload: str) -> None: + path = tmp_path / "hostile.json" + path.write_text(payload, encoding="utf-8") + + with pytest.raises(ValueError, match="invalid JSON artifact"): + reproduction.load_strict_json(path) + + +def test_protocol_rejects_drift_duplicate_attempts_and_per_episode_seed_scope() -> None: + selection = _selection() + protocol, _ledger = reproduction.build_declaration(PROJECT_ROOT, selection=selection) + + drifted = copy.deepcopy(protocol) + drifted["declaration"]["seed_policy"]["initialization_scope"] = "episode" + drifted["declaration_sha256"] = reproduction.sha256_payload(drifted["declaration"]) + with pytest.raises(ValueError, match="seed policy"): + reproduction.validate_protocol(drifted, selection=selection) + + duplicate = copy.deepcopy(protocol) + duplicate["declaration"]["schedule_partitions"][1]["first_slot_id"] = duplicate["declaration"][ + "schedule_partitions" + ][0]["first_slot_id"] + duplicate["declaration_sha256"] = reproduction.sha256_payload(duplicate["declaration"]) + with pytest.raises(ValueError, match="protocol declaration"): + reproduction.validate_protocol(duplicate, selection=selection) + + +def test_aggregation_is_deterministic_and_counts_every_disposition() -> None: + selection = _selection() + protocol, _ledger = reproduction.build_declaration(PROJECT_ROOT, selection=selection) + attempts = _valid_attempts(protocol) + attempts[0]["disposition"] = "failed" + attempts[0]["score_measure"] = None + attempts[1]["disposition"] = "invalid" + attempts[1]["score_measure"] = None + attempts[2]["disposition"] = "excluded" + attempts[2]["score_measure"] = None + + first = reproduction.compute_aggregates(protocol, attempts) + second = reproduction.compute_aggregates(protocol, list(reversed(attempts))) + + assert reproduction.canonical_json_bytes(first) == reproduction.canonical_json_bytes(second) + totals = first["disposition_counts"] + assert totals == {"excluded": 1, "failed": 1, "invalid": 1, "valid": 15} + assert "schema_version" not in first + condition = first["conditions"][0] + assert condition["eligible_n"] in {0, 1, 2} + assert math.isfinite(condition["mean"]) if condition["mean"] is not None else True + + +def test_aggregate_rejects_missing_duplicate_cross_condition_and_nonfinite_scores() -> None: + selection = _selection() + protocol, _ledger = reproduction.build_declaration(PROJECT_ROOT, selection=selection) + attempts = _valid_attempts(protocol) + + with pytest.raises(ValueError, match="scheduled attempts"): + reproduction.compute_aggregates(protocol, attempts[:-1]) + + duplicated = [*attempts, copy.deepcopy(attempts[0])] + with pytest.raises(ValueError, match="scheduled attempts"): + reproduction.compute_aggregates(protocol, duplicated) + + crossed = copy.deepcopy(attempts) + crossed[0]["condition_id"] = crossed[2]["condition_id"] + with pytest.raises(ValueError, match="slot join"): + reproduction.compute_aggregates(protocol, crossed) + + nonfinite = copy.deepcopy(attempts) + nonfinite[0]["score_measure"]["value"] = float("nan") + with pytest.raises(ValueError, match="score measure"): + reproduction.compute_aggregates(protocol, nonfinite) + + +def test_tiers_keep_subjects_separate_and_do_not_turn_score_into_equivalence() -> None: + selection = _selection() + protocol, _ledger = reproduction.build_declaration(PROJECT_ROOT, selection=selection) + aggregates = reproduction.compute_aggregates(protocol, _valid_attempts(protocol)) + reference = { + "authored_source": "pass", + "contract": "pass", + "execution_control": "weakened", + "state_observation": "weakened", + "outcome_evaluation": "weakened", + "disclosure": "pass", + "evidence_refs": ["reference.json"], + } + + tiers = reproduction.build_tiers(protocol, aggregates, reference) + + assert "passed" not in tiers + assert "schema_version" not in tiers + assert [item["tier"] for item in tiers["tiers"]] == [ + "authored-source", + "contract", + "execution-control", + "state/observation", + "outcome/evaluation", + "disclosure", + ] + assert all(set(item["subjects"]) == {"cyborg", "raes-reference"} for item in tiers["tiers"]) + state = next(item for item in tiers["tiers"] if item["tier"] == "state/observation") + assert state["subjects"]["raes-reference"]["result"] == "weakened" + assert tiers["strongest_supported_claim"] != "state-or-semantic-equivalence" + assert "state" not in tiers["strongest_supported_claim"] + + bounded = copy.deepcopy(aggregates) + bounded["comparison"]["classification"] = "bounded" + bounded_tiers = reproduction.build_tiers(protocol, bounded, reference) + assert bounded_tiers["strongest_supported_claim"] == ( + "behavioral-baseline-outcome-reproduction-without-submitted-agent-or-state-equivalence" + ) + assert bounded_tiers["research_consumers"]["OpenRAE/research#20"] == ( + "usable-behavioral-baseline-outcome-reproduction-evidence" + ) + + +def test_retry_keeps_original_attempt_and_requires_a_new_identity() -> None: + selection = _selection() + protocol, _ledger = reproduction.build_declaration(PROJECT_ROOT, selection=selection) + attempts = _valid_attempts(protocol) + original = attempts[0] + original["disposition"] = "failed" + original["score_measure"] = None + retry = reproduction.retry_record_for_test(original, score=4.0) + + selected = reproduction.select_eligible_attempts(protocol, [*attempts, retry]) + + assert "schema_version" not in original + assert "schema_version" not in retry + assert retry["run_id"] != original["run_id"] + assert retry["predecessor_run_id"] == original["run_id"] + assert any(item["run_id"] == retry["run_id"] for item in selected) + assert any(item["run_id"] == original["run_id"] for item in [*attempts, retry]) + + +def test_public_selection_is_the_complete_9000_slot_study() -> None: + protocol, _ledger = reproduction.build_declaration(PROJECT_ROOT) + declaration = protocol["declaration"] + assert isinstance(declaration, dict) + assert sum(item["slot_count"] for item in declaration["schedule_partitions"]) == 9000 + assert len(declaration["schedule_partitions"]) == 9 + assert declaration["episodes_per_condition"] == 1000 + + +def test_environment_provenance_joins_pack_scenario_and_method() -> None: + protocol, _ledger = reproduction.build_declaration(PROJECT_ROOT, selection=_selection()) + + environment = reproduction._write_environment(protocol) + reproduction._validate_environment(environment, protocol) + + assert "schema_version" not in environment + assert set(environment["distributions"]) == { + "CybORG", + "gym", + "numpy", + "raes", + "raes-adapters", + } + assert environment["pack"]["scenario_ref_digest"].startswith("sha256:") + assert environment["method"]["score_metric_id"] == "cage2-cumulative-blue-reward" + + +def test_reference_evidence_rejects_unreviewed_claim_fields() -> None: + reference = reproduction.capture_reference_path(PROJECT_ROOT) + reproduction._validate_reference(reference) + assert "schema_version" not in reference + reference["equivalent"] = True + + with pytest.raises(ValueError, match="reference evidence"): + reproduction._validate_reference(reference) + + +def test_persisted_scan_rejects_native_and_traceback_material(tmp_path: Path) -> None: + root = tmp_path / "bundle" + root.mkdir() + (root / "bad.json").write_text( + json.dumps({"native_observation": "Traceback (most recent call last):"}), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="forbidden native material"): + reproduction.scan_public_tree(root) + + +def test_cli_declares_partitioned_full_protocol_without_native_source( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + + result = cli.main( + [ + "reproduce", + "--phase", + "declare", + "--source-root", + str(PROJECT_ROOT), + "--output", + "declaration", + ] + ) + + assert result == 0 + summary = json.loads(capsys.readouterr().out) + assert summary["disposition"] == "declared" + protocol = reproduction.load_strict_json(tmp_path / "declaration" / "protocol.json") + assert ( + sum(item["slot_count"] for item in protocol["declaration"]["schedule_partitions"]) == 9000 + ) + assert ( + max(path.stat().st_size for path in (tmp_path / "declaration").rglob("*") if path.is_file()) + < 500 * 1024 + ) + + +def test_reproduction_output_symlink_escape_is_rejected( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + outside = tmp_path.parent / f"{tmp_path.name}-outside" + outside.mkdir() + (tmp_path / "linked").symlink_to(outside, target_is_directory=True) + monkeypatch.chdir(tmp_path) + + result = cli.main( + [ + "reproduce", + "--phase", + "declare", + "--source-root", + str(PROJECT_ROOT), + "--output", + "linked/evidence", + ] + ) + + assert result == cli.EXIT_OUTPUT + assert not (outside / "evidence").exists() + assert "researcher.output.unavailable" in capsys.readouterr().err + + +def test_reduced_study_seals_every_attempt_and_recomputes_offline(tmp_path: Path) -> None: + selection = _selection(episodes=1) + bundle = tmp_path / "bundle" + + reproduction.run_full_study( + PROJECT_ROOT, + bundle, + selection=selection, + driver=StudyDriver(), + ) + result = reproduction.verify_bundle(bundle, selection=selection) + + assert result["scheduled_slot_count"] == 9 + assert result["attempt_count"] == 9 + assert result["classification"] == "failed" + attempts = list(bundle.glob("runs/*/*/attempt.json.gz")) + assert len(attempts) == 9 + assert all(reproduction.load_strict_json(path)["disposition"] == "valid" for path in attempts) + stale_tiers = reproduction.load_strict_json(bundle / "tiers.json") + stale_tiers["strongest_supported_claim"] = "stale-pre-migration-projection" + reproduction._write_json(bundle / "tiers.json", stale_tiers) + root_inventory = reproduction.load_strict_json(bundle / "inventory.json") + reproduction._seal_inventory( + bundle, + [bundle / item["path"] for item in root_inventory["artifacts"]], + ) + reproduction.compact_bundle_evidence(bundle) + assert reproduction.verify_bundle(bundle, selection=selection)["attempt_count"] == 9 + assert reproduction.load_strict_json(bundle / "tiers.json")["strongest_supported_claim"] == ( + "failed-outcome-reproduction-with-retained-negative-result" + ) + output = tmp_path / "recomputed" + reproduction.recompute_bundle(bundle, output, selection=selection) + assert (output / "verification.json").is_file() + + +def test_failed_condition_keeps_only_validated_terminal_attempts(tmp_path: Path) -> None: + selection = _selection(episodes=2) + bundle = tmp_path / "failed-bundle" + + reproduction.run_full_study( + PROJECT_ROOT, + bundle, + selection=selection, + driver=FailingStudyDriver(), + ) + result = reproduction.verify_bundle(bundle, selection=selection) + + assert result["classification"] == "failed" + attempts = list(bundle.glob("runs/*/*/attempt.json.gz")) + records = [reproduction.load_strict_json(path) for path in attempts] + assert len(records) == 20 + assert [record["disposition"] for record in records].count("failed") == 2 + assert [record["disposition"] for record in records].count("valid") == 18 + retries = [record for record in records if record["attempt_sequence"] == 2] + assert len(retries) == 2 + assert all(record["predecessor_run_id"].endswith("attempt-01") for record in retries) + assert all( + {path.name for path in attempt.parent.iterdir()} == {"attempt.json.gz", "inventory.json"} + for attempt in attempts + if reproduction.load_strict_json(attempt)["disposition"] == "failed" + ) + + +def test_exhausted_retry_does_not_skip_later_conditions(tmp_path: Path) -> None: + selection = _selection(episodes=2) + bundle = tmp_path / "exhausted-bundle" + + reproduction.run_full_study( + PROJECT_ROOT, + bundle, + selection=selection, + driver=ExhaustedRetryStudyDriver(), + ) + result = reproduction.verify_bundle(bundle, selection=selection) + + records = [ + reproduction.load_strict_json(path) for path in bundle.glob("runs/*/*/attempt.json.gz") + ] + first_condition = [ + record for record in records if record["condition_id"] == "steps-030--red-b-line" + ] + later_conditions = [ + record for record in records if record["condition_id"] != "steps-030--red-b-line" + ] + assert result["classification"] == "unavailable" + assert len(first_condition) == 4 + assert all(record["disposition"] == "failed" for record in first_condition) + assert len(later_conditions) == 16 + assert all(record["disposition"] == "valid" for record in later_conditions) + + +def test_offline_verifier_requires_complete_transitive_inventories(tmp_path: Path) -> None: + selection = _selection(episodes=1) + bundle = tmp_path / "bundle" + reproduction.run_full_study( + PROJECT_ROOT, + bundle, + selection=selection, + driver=StudyDriver(), + ) + condition = next((bundle / "runs").iterdir()) + inventory = reproduction.load_strict_json(condition / "inventory.json") + inventory["artifacts"] = [ + item for item in inventory["artifacts"] if not item["path"].endswith("/inventory.json") + ] + reproduction._write_json(condition / "inventory.json", inventory) + root_inventory = reproduction.load_strict_json(bundle / "inventory.json") + root_members = [bundle / item["path"] for item in root_inventory["artifacts"]] + reproduction._seal_inventory(bundle, root_members) + + with pytest.raises(ValueError, match="inventory membership is incomplete"): + reproduction.verify_bundle(bundle, selection=selection) diff --git a/tests/test_cyborg_scenario2.py b/tests/test_cyborg_scenario2.py index 7c75553..7d7e736 100644 --- a/tests/test_cyborg_scenario2.py +++ b/tests/test_cyborg_scenario2.py @@ -186,6 +186,18 @@ def test_canonical_scenario_reaches_driver_with_exact_scenario2_projection() -> "in": "None", "out": "all", } + assert driver.native["Subnets"]["Enterprise"]["Hosts"] == [ + "Enterprise0", + "Enterprise1", + "Enterprise2", + "Defender", + ] + assert driver.native["Subnets"]["Operational"]["Hosts"] == [ + "Op_Server0", + "Op_Host0", + "Op_Host1", + "Op_Host2", + ] assert set(driver.native["Agents"]) == {"Blue", "Green", "Red"} assert driver.native["Agents"]["Blue"]["actions"] == [ "Sleep", @@ -214,9 +226,9 @@ def test_canonical_scenario_reaches_driver_with_exact_scenario2_projection() -> "HTTPRFI", "HTTPSRFI", "SQLInjection", - "SSHBruteForce", "PrivilegeEscalate", "Impact", + "SSHBruteForce", ] assert driver.native["Agents"]["Red"]["INT"] == { "Hosts": {"User0": {"Interfaces": "All", "System info": "All"}} @@ -235,6 +247,23 @@ def test_canonical_scenario_reaches_driver_with_exact_scenario2_projection() -> item["hostname"] == "Defender" and item["name"] == "VeloServer" for item in driver.native["Agents"]["Blue"]["starting_sessions"] ) + assert [ + item["hostname"] for item in driver.native["Agents"]["Blue"]["starting_sessions"][:-1] + ] == [ + "User0", + "User1", + "User2", + "User3", + "User4", + "Enterprise0", + "Enterprise1", + "Enterprise2", + "Defender", + "Op_Server0", + "Op_Host0", + "Op_Host1", + "Op_Host2", + ] assert driver.native["Hosts"]["Enterprise2"]["info"]["Op_Server0"] == { "Interfaces": "IP Address" } diff --git a/tests/test_researcher_cli.py b/tests/test_researcher_cli.py index 7593afe..ccc682b 100644 --- a/tests/test_researcher_cli.py +++ b/tests/test_researcher_cli.py @@ -30,7 +30,11 @@ _NativeRewardComponent, _NativeTurnResult, ) -from raes_adapters.cyborg.researcher import RunControls, execute_episode +from raes_adapters.cyborg.researcher import ( + RunControls, + execute_episode, + execute_episode_series, +) _SDL = """ name: researcher-cage2 @@ -141,6 +145,7 @@ class FakeResearchDriver: handles: list[object] = field(default_factory=list) selections: list[ParticipantValidatedActionSelection] = field(default_factory=list) cleanup_calls: int = 0 + reset_calls: int = 0 def construct(self, descriptor: object, *, seed: int | None) -> object: del descriptor, seed @@ -160,6 +165,7 @@ def construct_execution( def reset(self, handle: object, *, seed: int | None) -> bool: del seed + self.reset_calls += 1 return handle in self.handles def step( @@ -507,6 +513,42 @@ def test_native_episode_binds_real_blue_selection_and_retains_raes_evidence() -> assert driver.selections[0].action_contract_address == "participant.action-contract.sleep" +def test_condition_series_reuses_native_session_and_binds_unique_run_evidence() -> None: + driver = FakeResearchDriver() + manifest, selection, configuration = _participant_artifacts() + controls = tuple( + RunControls( + run_id=f"cage2-slot-{index:05d}-attempt-01", + seed=153, + max_steps=1, + red_variant="sleep", + blue_manifest=manifest, + blue_selection=selection, + blue_configuration=configuration, + ) + for index in (1, 2) + ) + + episodes = execute_episode_series( + parse_sdl(textwrap.dedent(_SDL)), + controls, + driver=driver, + ) + + assert len(episodes) == 2 + assert driver.cleanup_calls == 2 + assert driver.reset_calls == 1 + assert len(driver.selections) == 2 + assert [episode.evidence_records[0].run_ref.ref_id for episode in episodes] == [ + item.run_id for item in controls + ] + assert len(episodes[0].evidence_records) == len(episodes[1].evidence_records) + assert [ + {record.run_ref.ref_id for record in episode.evidence_records} for episode in episodes + ] == [{item.run_id} for item in controls] + assert all(episode.cleanup_verified for episode in episodes) + + def test_full_scenario_episode_consumes_authored_evaluation_plan( monkeypatch: pytest.MonkeyPatch, ) -> None: From 4d5f3677d5e5f67316725062145eda355725a349 Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Wed, 12 Aug 2026 23:35:47 +0200 Subject: [PATCH 2/3] refactor(cyborg): clear reproduction quality findings --- src/raes_adapters/cli.py | 32 +- src/raes_adapters/cyborg/evaluator.py | 286 ++-- src/raes_adapters/cyborg/reproduction.py | 1772 ++++++++++++++-------- src/raes_adapters/cyborg/researcher.py | 27 +- 4 files changed, 1365 insertions(+), 752 deletions(-) diff --git a/src/raes_adapters/cli.py b/src/raes_adapters/cli.py index a79ffac..5bda375 100644 --- a/src/raes_adapters/cli.py +++ b/src/raes_adapters/cli.py @@ -160,6 +160,8 @@ def _single_participant_experiment_bindings_match( EXIT_RUNTIME = 5 EXIT_ARTIFACT = 6 EXIT_INTERNAL = 70 +_OUTPUT_UNAVAILABLE_CODE = "researcher.output.unavailable" +_OUTPUT_UNAVAILABLE_MESSAGE = "output root is unavailable" _INVENTORY_NAME = "inventory.json" _RUNTIME_FAILURE_CODE = "researcher.runtime.failure" _CONTROLS_INVALID_CODE = "researcher.validation.controls-invalid" @@ -1349,7 +1351,7 @@ def _run_conformance(adapter: _BackendAdapter, args: argparse.Namespace) -> int: output = _reserve_output(args.output) except _OutputFailure: raise _CommandFailure( - EXIT_OUTPUT, "researcher.output.unavailable", "output root is unavailable" + EXIT_OUTPUT, _OUTPUT_UNAVAILABLE_CODE, _OUTPUT_UNAVAILABLE_MESSAGE ) from None try: conformance_args: dict[str, object] = { @@ -1431,7 +1433,7 @@ def _native_environment( output = _reserve_output(args.output) except _OutputFailure: raise _CommandFailure( - EXIT_OUTPUT, "researcher.output.unavailable", "output root is unavailable" + EXIT_OUTPUT, _OUTPUT_UNAVAILABLE_CODE, _OUTPUT_UNAVAILABLE_MESSAGE ) from None return admitted, output @@ -1691,14 +1693,18 @@ def _dispatch(args: argparse.Namespace) -> int: """Dispatch one parsed command through its closed execution path.""" if args.command == "reproduce": - return _reproduce(args) - if args.command == "inspect": + result = _reproduce(args) + elif args.command == "inspect": print(json.dumps(_adapter(args).inspection_payload(), sort_keys=True)) - return 0 - adapter = _adapter(args) - if args.command == "validate": - return _validated_admission(adapter, args) - return _run_command(adapter, args) + result = 0 + else: + adapter = _adapter(args) + result = ( + _validated_admission(adapter, args) + if args.command == "validate" + else _run_command(adapter, args) + ) + return result def _reproduce(args: argparse.Namespace) -> int: @@ -1728,8 +1734,8 @@ def _reproduce(args: argparse.Namespace) -> int: if relative_output is None or not relative_output.parts: raise _CommandFailure( EXIT_OUTPUT, - "researcher.output.unavailable", - "output root is unavailable", + _OUTPUT_UNAVAILABLE_CODE, + _OUTPUT_UNAVAILABLE_MESSAGE, ) result: dict[str, object] try: @@ -1753,8 +1759,8 @@ def _reproduce(args: argparse.Namespace) -> int: except FileExistsError: raise _CommandFailure( EXIT_OUTPUT, - "researcher.output.unavailable", - "output root is unavailable", + _OUTPUT_UNAVAILABLE_CODE, + _OUTPUT_UNAVAILABLE_MESSAGE, ) from None except ValueError: raise _CommandFailure( diff --git a/src/raes_adapters/cyborg/evaluator.py b/src/raes_adapters/cyborg/evaluator.py index 0d88074..410a191 100644 --- a/src/raes_adapters/cyborg/evaluator.py +++ b/src/raes_adapters/cyborg/evaluator.py @@ -90,6 +90,162 @@ def _identity(*parts: object) -> str: return hashlib.sha256(encoded.encode("utf-8")).hexdigest() +def _blue_component_values_and_refs( + facts: tuple[_NativeEvaluationTurn, ...], + records: tuple[ExperimentEvidenceRecordModel, ...], +) -> tuple[ + dict[str, float], + dict[str, list[ExperimentEvidenceRecordReferenceModel]], +]: + """Accumulate allowlisted Blue components and their committed evidence refs.""" + + values: dict[str, float] = dict.fromkeys(_SUPPORTED_COMPONENTS, 0.0) + refs: dict[str, list[ExperimentEvidenceRecordReferenceModel]] = { + name: [] for name in _SUPPORTED_COMPONENTS + } + record_ids = {record.evidence_record_id for record in records} + for turn in facts: + for component in turn.components: + if component.participant_address != _BLUE: + continue + if component.component not in _SUPPORTED_COMPONENTS: + continue + values[component.component] += component.value + record_id = _identity( + turn.run_id, + turn.action_instance_id, + turn.logical_step, + component.participant_address, + component.target_address, + component.component, + ) + evidence_id = "evidence-record.cyborg-cage2." + record_id + if evidence_id in record_ids: + refs[component.component].append( + ExperimentEvidenceRecordReferenceModel( + ref_kind="evidence-record", + ref_id=evidence_id, + ref_version=_RECORD_VERSION, + ) + ) + return values, refs + + +def _blue_component_measure( + facts: tuple[_NativeEvaluationTurn, ...], + component_name: str, + value: float, + refs: list[ExperimentEvidenceRecordReferenceModel], + now: str, +) -> ExperimentDerivedMeasureModel: + """Build one allowlisted cumulative Blue component measure.""" + + identity = _identity(facts[-1].run_id, facts[-1].episode_id, len(facts)) + return ExperimentDerivedMeasureModel( + schema_version="experiment-derived-measure/v1", + derived_measure_id=f"measure.cyborg-cage2.cumulative-blue-{component_name}.{identity}", + measure_version="1.0.0", + measure_kind="metric", + metric_ref=ExperimentReferenceModel( + ref_kind="metric-definition", + ref_id=f"cage2-cumulative-blue-{component_name}-reward", + ref_version="1.0.0", + ), + method=ExperimentDerivedMeasureMethodModel( + method_id=f"cyborg-cage2-sum-blue-{component_name}-rewards", + method_version="1.0.0", + name=f"Cumulative committed Blue {component_name} reward", + description="Sum the named committed per-step Blue component in logical step order.", + ), + source_evidence_refs=refs, + generated_at=now, + value_status="reported", + value=value, + uncertainty="No uncertainty interval is inferred from one source run.", + limitations=[ + "This component is not a conformance result, objective outcome, " + "or replication-equivalence claim." + ], + provenance_refs=[ + ExperimentReferenceModel(ref_kind="run", ref_id=facts[-1].run_id, ref_version="1.0.0") + ], + ) + + +def _blue_component_measures( + facts: tuple[_NativeEvaluationTurn, ...], + records: tuple[ExperimentEvidenceRecordModel, ...], + now: str, +) -> list[ExperimentDerivedMeasureModel]: + """Build measures only for Blue components with retained source evidence.""" + + values, component_refs = _blue_component_values_and_refs(facts, records) + return [ + _blue_component_measure(facts, name, values[name], component_refs[name], now) + for name in sorted(_SUPPORTED_COMPONENTS) + if component_refs[name] + ] + + +def _blue_score_refs( + records: tuple[ExperimentEvidenceRecordModel, ...], +) -> list[ExperimentEvidenceRecordReferenceModel]: + """Return evidence references for committed per-step Blue rewards.""" + + return [ + ExperimentEvidenceRecordReferenceModel( + ref_kind="evidence-record", + ref_id=record.evidence_record_id, + ref_version=_RECORD_VERSION, + ) + for record in records + if f"participant={_BLUE};" in (record.raw_content.payload_summary or "") + and "meaning=per-step-reward;" in (record.raw_content.payload_summary or "") + ] + + +def _blue_score_measure( + facts: tuple[_NativeEvaluationTurn, ...], + records: tuple[ExperimentEvidenceRecordModel, ...], + now: str, +) -> ExperimentDerivedMeasureModel: + """Build the cumulative committed Blue score measure.""" + + identity = _identity(facts[-1].run_id, facts[-1].episode_id, len(facts)) + return ExperimentDerivedMeasureModel( + schema_version="experiment-derived-measure/v1", + derived_measure_id=f"measure.cyborg-cage2.cumulative-blue-score.{identity}", + measure_version="1.0.0", + measure_kind="score", + metric_ref=ExperimentReferenceModel( + ref_kind="metric-definition", + ref_id="cage2-cumulative-blue-reward", + ref_version="1.0.0", + ), + method=ExperimentDerivedMeasureMethodModel( + method_id="cyborg-cage2-sum-committed-blue-rewards", + method_version="1.0.0", + name="Cumulative committed Blue reward", + description="Sum exact per-step Blue rewards in committed logical-step order.", + ), + source_evidence_refs=_blue_score_refs(records), + generated_at=now, + value_status="reported", + value=sum(dict(turn.rewards)[_BLUE] for turn in facts), + uncertainty="No uncertainty interval is inferred from one source run.", + limitations=[ + "This score is not a conformance result, objective outcome, " + "scenario-correctness claim, or replication-equivalence claim." + ], + provenance_refs=[ + ExperimentReferenceModel(ref_kind="run", ref_id=facts[-1].run_id, ref_version="1.0.0"), + ExperimentReferenceModel( + ref_kind="other", ref_id=facts[-1].episode_id, ref_version="1.0.0" + ), + ], + ) + + class _PredicateBinding(NamedTuple): """Supported proposition fields resolved from one compiled operation.""" @@ -775,132 +931,12 @@ def _measures( records: tuple[ExperimentEvidenceRecordModel, ...], now: str, ) -> tuple[ExperimentDerivedMeasureModel, ...]: - measures: list[ExperimentDerivedMeasureModel] = [] + """Build the component and score measures for committed Blue reward facts.""" + if not facts or not records: - return tuple(measures) - blue_total = sum(dict(turn.rewards)[_BLUE] for turn in facts) - blue_components: dict[str, float] = dict.fromkeys(_SUPPORTED_COMPONENTS, 0.0) - component_refs: dict[str, list[ExperimentEvidenceRecordReferenceModel]] = { - name: [] for name in _SUPPORTED_COMPONENTS - } - record_by_id = {record.evidence_record_id: record for record in records} - for turn in facts: - for component in turn.components: - if ( - component.participant_address == _BLUE - and component.component in _SUPPORTED_COMPONENTS - ): - blue_components[component.component] += component.value - record_id = _identity( - turn.run_id, - turn.action_instance_id, - turn.logical_step, - component.participant_address, - component.target_address, - component.component, - ) - evidence_id = "evidence-record.cyborg-cage2." + record_id - if evidence_id in record_by_id: - component_refs[component.component].append( - ExperimentEvidenceRecordReferenceModel( - ref_kind="evidence-record", - ref_id=evidence_id, - ref_version=_RECORD_VERSION, - ) - ) - for component_name in sorted(_SUPPORTED_COMPONENTS): - refs_for_component = component_refs[component_name] - if refs_for_component: - measures.append( - ExperimentDerivedMeasureModel( - schema_version="experiment-derived-measure/v1", - derived_measure_id=( - f"measure.cyborg-cage2.cumulative-blue-{component_name}." - + _identity(facts[-1].run_id, facts[-1].episode_id, len(facts)) - ), - measure_version="1.0.0", - measure_kind="metric", - metric_ref=ExperimentReferenceModel( - ref_kind="metric-definition", - ref_id=f"cage2-cumulative-blue-{component_name}-reward", - ref_version="1.0.0", - ), - method=ExperimentDerivedMeasureMethodModel( - method_id=f"cyborg-cage2-sum-blue-{component_name}-rewards", - method_version="1.0.0", - name=f"Cumulative committed Blue {component_name} reward", - description=( - "Sum the named committed per-step Blue component in logical " - "step order." - ), - ), - source_evidence_refs=refs_for_component, - generated_at=now, - value_status="reported", - value=blue_components[component_name], - uncertainty="No uncertainty interval is inferred from one source run.", - limitations=[ - "This component is not a conformance result, objective outcome, " - "or replication-equivalence claim." - ], - provenance_refs=[ - ExperimentReferenceModel( - ref_kind="run", - ref_id=facts[-1].run_id, - ref_version="1.0.0", - ) - ], - ) - ) - refs = [ - ExperimentEvidenceRecordReferenceModel( - ref_kind="evidence-record", - ref_id=record.evidence_record_id, - ref_version=_RECORD_VERSION, - ) - for record in records - if f"participant={_BLUE};" in (record.raw_content.payload_summary or "") - and "meaning=per-step-reward;" in (record.raw_content.payload_summary or "") - ] - measures.append( - ExperimentDerivedMeasureModel( - schema_version="experiment-derived-measure/v1", - derived_measure_id=( - "measure.cyborg-cage2.cumulative-blue-score." - + _identity(facts[-1].run_id, facts[-1].episode_id, len(facts)) - ), - measure_version="1.0.0", - measure_kind="score", - metric_ref=ExperimentReferenceModel( - ref_kind="metric-definition", - ref_id="cage2-cumulative-blue-reward", - ref_version="1.0.0", - ), - method=ExperimentDerivedMeasureMethodModel( - method_id="cyborg-cage2-sum-committed-blue-rewards", - method_version="1.0.0", - name="Cumulative committed Blue reward", - description="Sum exact per-step Blue rewards in committed logical-step order.", - ), - source_evidence_refs=refs, - generated_at=now, - value_status="reported", - value=blue_total, - uncertainty="No uncertainty interval is inferred from one source run.", - limitations=[ - "This score is not a conformance result, objective outcome, " - "scenario-correctness claim, or replication-equivalence claim." - ], - provenance_refs=[ - ExperimentReferenceModel( - ref_kind="run", ref_id=facts[-1].run_id, ref_version="1.0.0" - ), - ExperimentReferenceModel( - ref_kind="other", ref_id=facts[-1].episode_id, ref_version="1.0.0" - ), - ], - ) - ) + return () + measures = _blue_component_measures(facts, records, now) + measures.append(_blue_score_measure(facts, records, now)) return tuple(measures) def status(self) -> dict[str, object]: diff --git a/src/raes_adapters/cyborg/reproduction.py b/src/raes_adapters/cyborg/reproduction.py index e38c218..1020838 100644 --- a/src/raes_adapters/cyborg/reproduction.py +++ b/src/raes_adapters/cyborg/reproduction.py @@ -18,11 +18,10 @@ from collections import Counter from collections.abc import Callable, Mapping, Sequence from contextlib import redirect_stderr, redirect_stdout, suppress -from dataclasses import dataclass, replace from importlib import metadata from pathlib import Path from statistics import NormalDist -from typing import cast +from typing import NamedTuple, cast from raes_contracts.contracts import ( # type: ignore[import-untyped] ExperimentArtifactRefModel, @@ -52,18 +51,26 @@ ) _INVALID_JSON = "invalid JSON artifact" +_INVALID_ATTEMPT_SLOT_JOIN = "attempt slot join is invalid" +_INVALID_FORBIDDEN_MATERIAL = "forbidden native material" +_INVALID_INVENTORY = "inventory is invalid" +_INVALID_PROTOCOL_SCHEDULE = "protocol schedule is invalid" +_INVALID_SCORE_MEASURE = "score measure is invalid" _DISPOSITIONS = ("excluded", "failed", "invalid", "valid") _RED_VARIANTS = ("b-line", "meander", "sleep") _TRIAL_LENGTHS = (30, 50, 100) +_STATE_OBSERVATION_TIER = "state/observation" +_OUTCOME_EVALUATION_TIER = "outcome/evaluation" _TIERS = ( "authored-source", "contract", "execution-control", - "state/observation", - "outcome/evaluation", + _STATE_OBSERVATION_TIER, + _OUTCOME_EVALUATION_TIER, "disclosure", ) _SCORE_METRIC_ID = "cage2-cumulative-blue-reward" +_SOURCE_PROFILE = "cage2-cyborg-2.1-source-26ce1c1" _SOURCE_COMMIT = "26ce1c1253fa9e2e73f25e6a7f2da32860c11257" _PAPER_SHA256 = "45a6e564da6dff67453ec75585e847228a7f407641f250f4abe28d1f29406f29" _PRIVATE_KEY_MARKERS = ( @@ -88,10 +95,22 @@ ) _MAX_PUBLIC_FILE_BYTES = 500 * 1024 _MAX_DECOMPRESSED_JSON_BYTES = 64 * 1024 * 1024 - - -@dataclass(frozen=True) -class ReproductionSelection: +_AGGREGATES_FILE = "aggregates.json" +_ATTEMPT_FILE = "attempt.json.gz" +_CLEANUP_FILE = "cleanup.json" +_ENVIRONMENT_FILE = "environment.json" +_EVIDENCE_INDEX_FILE = "evidence-index.json" +_GZIP_JSON_SUFFIX = ".json.gz" +_INVENTORY_FILE = "inventory.json" +_PROTOCOL_FILE = "protocol.json" +_REFERENCE_FILE = "reference.json" +_RUN_FILE = "run.json.gz" +_SOURCE_LEDGER_FILE = "source-ledger.json" +_TIERS_FILE = "tiers.json" +_VALIDATION_REPORT_FILE = "validation-report.md" + + +class ReproductionSelection(NamedTuple): """One immutable backend-local study selection.""" frozen_revision: str @@ -106,13 +125,26 @@ def with_cardinality_for_test(self, episodes: int) -> ReproductionSelection: if type(episodes) is not int or episodes < 1: raise ValueError("test cardinality is invalid") - return replace( - self, + return ReproductionSelection( frozen_revision=f"{self.frozen_revision}-test-{episodes}", episodes_per_condition=episodes, + seed=self.seed, + trial_lengths=self.trial_lengths, + red_variants=self.red_variants, + retry_limit=self.retry_limit, ) +class _AttemptOutcome(NamedTuple): + """Disposition-dependent fields for one terminal operational attempt.""" + + disposition: str + cleanup_verified: bool + score_measure: object = None + evidence_refs: Sequence[str] = () + diagnostic_code: str | None = None + + FROZEN_SELECTION = ReproductionSelection( frozen_revision="cage2-26ce1c1-ccs-sleeper-v1", episodes_per_condition=1000, @@ -234,14 +266,20 @@ def sha256_payload(payload: object) -> str: def _sha256_file(path: Path) -> str: + """Hash one file as raw bytes.""" + return hashlib.sha256(path.read_bytes()).hexdigest() def _reject_constant(_value: str) -> object: + """Reject non-finite JSON constants.""" + raise ValueError(_INVALID_JSON) def _closed_pairs(pairs: list[tuple[str, object]]) -> dict[str, object]: + """Build one JSON object while rejecting duplicate keys.""" + result: dict[str, object] = {} for key, value in pairs: if key in result: @@ -255,7 +293,7 @@ def _load_strict_json_value(path: Path) -> object: try: raw = path.read_bytes() - if path.name.endswith(".json.gz"): + if path.name.endswith(_GZIP_JSON_SUFFIX): raw = gzip.decompress(raw) if len(raw) > _MAX_DECOMPRESSED_JSON_BYTES: raise ValueError(_INVALID_JSON) @@ -264,7 +302,7 @@ def _load_strict_json_value(path: Path) -> object: object_pairs_hook=_closed_pairs, parse_constant=_reject_constant, ) - except (OSError, UnicodeError, ValueError, gzip.BadGzipFile) as error: + except (OSError, ValueError) as error: raise ValueError(_INVALID_JSON) from error return value @@ -279,10 +317,14 @@ def load_strict_json(path: Path) -> dict[str, object]: def _condition_id(trial_length: int, red_variant: str) -> str: + """Return the stable identifier for one matrix condition.""" + return f"steps-{trial_length:03d}--red-{red_variant}" def _schedule(selection: ReproductionSelection) -> list[dict[str, object]]: + """Expand one selection into its ordered logical slots.""" + schedule: list[dict[str, object]] = [] ordinal = 0 for trial_length in selection.trial_lengths: @@ -308,6 +350,8 @@ def _schedule(selection: ReproductionSelection) -> list[dict[str, object]]: def _schedule_partitions(selection: ReproductionSelection) -> list[dict[str, object]]: + """Describe the condition-level partitions of one ordered schedule.""" + schedule = _schedule(selection) partitions: list[dict[str, object]] = [] offset = 0 @@ -330,6 +374,8 @@ def _schedule_partitions(selection: ReproductionSelection) -> list[dict[str, obj def _artifact_refs(repo_root: Path) -> list[dict[str, object]]: + """Resolve and verify every frozen adapter input.""" + refs: list[dict[str, object]] = [] for artifact_id, relative in sorted(_INPUT_PATHS.items()): path = repo_root / relative @@ -351,6 +397,8 @@ def _artifact_refs(repo_root: Path) -> list[dict[str, object]]: def _contract_payloads(repo_root: Path) -> dict[str, object]: + """Load and content-bind the published RAES contract inputs.""" + contracts: dict[str, object] = { "experiment_spec": load_strict_json(repo_root / _INPUT_PATHS["experiment-authoring-input"]), "experiment_task": load_strict_json(repo_root / _INPUT_PATHS["experiment-task"]), @@ -368,6 +416,8 @@ def _contract_payloads(repo_root: Path) -> dict[str, object]: def _expected_artifact_refs() -> list[dict[str, object]]: + """Return the predeclared adapter-input references.""" + return [ { "artifact_id": artifact_id, @@ -380,11 +430,13 @@ def _expected_artifact_refs() -> list[dict[str, object]]: def _source_ledger(artifacts: Sequence[Mapping[str, object]]) -> dict[str, object]: + """Project source facts and known losses for this study.""" + return { "source": { "repository": "https://github.com/cage-challenge/cage-challenge-2", "commit": _SOURCE_COMMIT, - "profile_id": "cage2-cyborg-2.1-source-26ce1c1", + "profile_id": _SOURCE_PROFILE, }, "protocol_source_facts": { "qualified_evaluator": { @@ -424,6 +476,8 @@ def _source_ledger(artifacts: Sequence[Mapping[str, object]]) -> dict[str, objec def _published_results() -> dict[str, object]: + """Return the frozen published comparison values.""" + conditions = [] for trial_length in _TRIAL_LENGTHS: for red_variant in _RED_VARIANTS: @@ -453,9 +507,11 @@ def _declaration( artifacts: list[dict[str, object]], contracts: Mapping[str, object], ) -> dict[str, object]: + """Build the immutable study selection and method declaration.""" + return { "frozen_revision": selection.frozen_revision, - "source_profile": "cage2-cyborg-2.1-source-26ce1c1", + "source_profile": _SOURCE_PROFILE, "source_commit": _SOURCE_COMMIT, "baseline": { "observed_implementation": "cyborg-blue-sleep-policy", @@ -541,11 +597,15 @@ def build_declaration( def _require_exact_keys(value: Mapping[str, object], keys: set[str], label: str) -> None: + """Reject unknown or missing operational-index fields.""" + if set(value) != keys: raise ValueError(f"{label} has unknown or missing fields") def _validate_schedule(value: object, selection: ReproductionSelection) -> None: + """Validate an ordered slot schedule against its selection.""" + if not isinstance(value, list) or len(value) != ( len(selection.trial_lengths) * len(selection.red_variants) @@ -561,35 +621,38 @@ def _validate_schedule(value: object, selection: ReproductionSelection) -> None: raise ValueError("attempt schedule is invalid") -def validate_protocol( - payload: Mapping[str, object], - *, - selection: ReproductionSelection | None = None, -) -> None: - """Validate the closed frozen protocol and its content digest.""" +def _validated_protocol_declaration(payload: Mapping[str, object]) -> dict[str, object]: + """Return the digest-verified declaration from a protocol index.""" - _require_exact_keys( - payload, - {"declaration_sha256", "declaration"}, - "protocol", - ) + _require_exact_keys(payload, {"declaration_sha256", "declaration"}, "protocol") declaration = payload["declaration"] if not isinstance(declaration, dict): raise ValueError("protocol declaration is invalid") if payload["declaration_sha256"] != sha256_payload(declaration): raise ValueError("protocol declaration digest is invalid") + return declaration + + +def _validated_protocol_contracts( + declaration: Mapping[str, object], +) -> tuple[list[dict[str, object]], dict[str, object]]: + """Validate and return the frozen artifacts and published contracts.""" + artifacts = declaration.get("artifacts") if not isinstance(artifacts, list) or artifacts != _expected_artifact_refs(): raise ValueError("protocol artifacts are invalid") - if selection is None: - selection = _selection_from_declaration(declaration) contracts = declaration.get("published_contracts") if not isinstance(contracts, dict): raise ValueError("published contracts are invalid") - if { - name: sha256_payload(contract) for name, contract in contracts.items() - } != _FROZEN_CONTRACT_DIGESTS: + digests = {name: sha256_payload(contract) for name, contract in contracts.items()} + if digests != _FROZEN_CONTRACT_DIGESTS: raise ValueError("published contract identity is invalid") + return cast(list[dict[str, object]], artifacts), contracts + + +def _validate_contract_relationships(contracts: Mapping[str, object]) -> None: + """Validate the RAES-owned contracts and their required identity joins.""" + try: spec = ExperimentSpecModel.model_validate(contracts["experiment_spec"]) task = ExperimentTaskModel.model_validate(contracts["experiment_task"]) @@ -603,33 +666,55 @@ def validate_protocol( contracts["participant_configuration"] ) validate_participant_configuration_selection(participant_selection, configuration) - if ( - spec.task_ref.ref_id != task.task_id - or participant_selection.implementation_identity != manifest.identity - ): + valid_join = ( + spec.task_ref.ref_id == task.task_id + and participant_selection.implementation_identity == manifest.identity + ) + if not valid_join: raise ValueError except Exception as error: raise ValueError("published contracts are invalid") from error - expected = _declaration( - selection, - cast(list[dict[str, object]], artifacts), - contracts, - ) - if declaration.get("seed_policy") != expected["seed_policy"]: - raise ValueError("seed policy is invalid") - if declaration.get("attempt_policy") != expected["attempt_policy"]: - raise ValueError("attempt policy is invalid") - if declaration.get("aggregation") != expected["aggregation"]: - raise ValueError("aggregation method is invalid") - if declaration.get("comparison") != expected["comparison"]: - raise ValueError("comparison method is invalid") + + +def _validate_declaration_methods( + declaration: Mapping[str, object], expected: Mapping[str, object] +) -> None: + """Validate the predeclared seed, attempt, aggregation, and comparison methods.""" + + labels = { + "seed_policy": "seed policy is invalid", + "attempt_policy": "attempt policy is invalid", + "aggregation": "aggregation method is invalid", + "comparison": "comparison method is invalid", + } + for key, message in labels.items(): + if declaration.get(key) != expected[key]: + raise ValueError(message) for key, value in expected.items(): if key not in {"artifacts", "published_contracts"} and declaration.get(key) != value: raise ValueError("protocol declaration is invalid") + + +def validate_protocol( + payload: Mapping[str, object], + *, + selection: ReproductionSelection | None = None, +) -> None: + """Validate the closed frozen protocol and its content digest.""" + + declaration = _validated_protocol_declaration(payload) + artifacts, contracts = _validated_protocol_contracts(declaration) + if selection is None: + selection = _selection_from_declaration(declaration) + _validate_contract_relationships(contracts) + expected = _declaration(selection, artifacts, contracts) + _validate_declaration_methods(declaration, expected) _validate_schedule(_schedule(selection), selection) def _validate_source_ledger(payload: Mapping[str, object], protocol: Mapping[str, object]) -> None: + """Validate source facts against the frozen declaration.""" + declaration = protocol["declaration"] if not isinstance(declaration, dict): raise ValueError("source ledger artifact join is invalid") @@ -641,8 +726,10 @@ def _validate_source_ledger(payload: Mapping[str, object], protocol: Mapping[str def _score_measure(run_id: str, score: float) -> dict[str, object]: + """Build a RAES-owned derived score measure for a test projection.""" + if not math.isfinite(score): - raise ValueError("score measure is invalid") + raise ValueError(_INVALID_SCORE_MEASURE) payload = ExperimentDerivedMeasureModel( schema_version="experiment-derived-measure/v1", derived_measure_id=f"derived-measure.{run_id}.cumulative-blue-reward", @@ -673,7 +760,7 @@ def _score_measure(run_id: str, score: float) -> dict[str, object]: provenance_refs=[ { "ref_kind": "profile", - "ref_id": "cage2-cyborg-2.1-source-26ce1c1", + "ref_id": _SOURCE_PROFILE, "ref_version": _SOURCE_COMMIT, } ], @@ -723,10 +810,12 @@ def retry_record_for_test(original: Mapping[str, object], *, score: float) -> di def _validate_score_measure(value: object) -> float: + """Validate and return one finite RAES-owned score value.""" + try: measure = ExperimentDerivedMeasureModel.model_validate(value) except Exception as error: - raise ValueError("score measure is invalid") from error + raise ValueError(_INVALID_SCORE_MEASURE) from error score = measure.value if ( measure.measure_kind != "score" @@ -736,32 +825,13 @@ def _validate_score_measure(value: object) -> float: or not isinstance(score, (int, float)) or not math.isfinite(float(score)) ): - raise ValueError("score measure is invalid") + raise ValueError(_INVALID_SCORE_MEASURE) return float(score) -def _validate_attempt( - value: Mapping[str, object], protocol: Mapping[str, object], slot: Mapping[str, object] -) -> None: - expected_keys = { - "declaration_sha256", - "slot_id", - "run_id", - "predecessor_run_id", - "attempt_sequence", - "condition_id", - "trial_length", - "red_variant", - "episode_ordinal", - "disposition", - "diagnostic_code", - "cleanup_verified", - "score_measure", - "evidence_refs", - } - _require_exact_keys(value, expected_keys, "attempt record") - if value["declaration_sha256"] != protocol["declaration_sha256"]: - raise ValueError("attempt protocol join is invalid") +def _validate_attempt_slot_fields(value: Mapping[str, object], slot: Mapping[str, object]) -> None: + """Validate the immutable logical-slot fields on one attempt.""" + for key in ( "slot_id", "condition_id", @@ -770,22 +840,40 @@ def _validate_attempt( "episode_ordinal", ): if value[key] != slot[key]: - raise ValueError("attempt slot join is invalid") - sequence = value["attempt_sequence"] + raise ValueError(_INVALID_ATTEMPT_SLOT_JOIN) + + +def _attempt_retry_limit(protocol: Mapping[str, object]) -> int: + """Return the validated retry limit joined through the protocol.""" + declaration = protocol["declaration"] if not isinstance(declaration, dict): raise ValueError("attempt protocol join is invalid") retry_limit = cast(dict[str, object], declaration["attempt_policy"])["retry_limit"] - if ( - type(sequence) is not int - or type(retry_limit) is not int - or not 1 <= sequence <= retry_limit + 1 - ): + if type(retry_limit) is not int: + raise ValueError("attempt sequence is invalid") + return retry_limit + + +def _validate_attempt_identity( + value: Mapping[str, object], slot: Mapping[str, object], retry_limit: int +) -> None: + """Validate one attempt's sequence and predecessor-linked identity.""" + + sequence = value["attempt_sequence"] + if type(sequence) is not int or not 1 <= sequence <= retry_limit + 1: raise ValueError("attempt sequence is invalid") expected_run_id = f"{slot['slot_id']}-attempt-{sequence:02d}" - predecessor = None if sequence == 1 else f"{slot['slot_id']}-attempt-{sequence - 1:02d}" + predecessor = None + if sequence > 1: + predecessor = f"{slot['slot_id']}-attempt-{sequence - 1:02d}" if value["run_id"] != expected_run_id or value["predecessor_run_id"] != predecessor: raise ValueError("attempt identities are invalid") + + +def _validate_attempt_outcome(value: Mapping[str, object]) -> None: + """Validate disposition-dependent cleanup and score fields.""" + disposition = value["disposition"] if disposition not in _DISPOSITIONS: raise ValueError("attempt disposition is invalid") @@ -797,7 +885,38 @@ def _validate_attempt( raise ValueError("ineligible attempt has a score measure") +def _validate_attempt( + value: Mapping[str, object], protocol: Mapping[str, object], slot: Mapping[str, object] +) -> None: + """Validate one operational attempt against its logical slot.""" + + expected_keys = { + "declaration_sha256", + "slot_id", + "run_id", + "predecessor_run_id", + "attempt_sequence", + "condition_id", + "trial_length", + "red_variant", + "episode_ordinal", + "disposition", + "diagnostic_code", + "cleanup_verified", + "score_measure", + "evidence_refs", + } + _require_exact_keys(value, expected_keys, "attempt record") + if value["declaration_sha256"] != protocol["declaration_sha256"]: + raise ValueError("attempt protocol join is invalid") + _validate_attempt_slot_fields(value, slot) + _validate_attempt_identity(value, slot, _attempt_retry_limit(protocol)) + _validate_attempt_outcome(value) + + def _selection_from_declaration(declaration: Mapping[str, object]) -> ReproductionSelection: + """Recover the immutable selection from a validated declaration.""" + try: selection = ReproductionSelection( frozen_revision=cast(str, declaration["frozen_revision"]), @@ -812,18 +931,20 @@ def _selection_from_declaration(declaration: Mapping[str, object]) -> Reproducti if declaration["schedule_partitions"] != _schedule_partitions(selection): raise ValueError except (KeyError, TypeError, ValueError): - raise ValueError("protocol schedule is invalid") from None + raise ValueError(_INVALID_PROTOCOL_SCHEDULE) from None return selection def _slot_index(protocol: Mapping[str, object]) -> dict[str, dict[str, object]]: + """Index every declared logical slot by identifier.""" + declaration = protocol["declaration"] if not isinstance(declaration, dict): - raise ValueError("protocol schedule is invalid") + raise ValueError(_INVALID_PROTOCOL_SCHEDULE) selection = _selection_from_declaration(declaration) schedule = _schedule(selection) if declaration["schedule_partitions"] != _schedule_partitions(selection): - raise ValueError("protocol schedule is invalid") + raise ValueError(_INVALID_PROTOCOL_SCHEDULE) return {cast(str, item["slot_id"]): item for item in schedule if isinstance(item, dict)} @@ -845,7 +966,7 @@ def select_eligible_attempts( record = dict(raw) slot_id = record.get("slot_id") if not isinstance(slot_id, str) or slot_id not in slots: - raise ValueError("attempt slot join is invalid") + raise ValueError(_INVALID_ATTEMPT_SLOT_JOIN) _validate_attempt(record, protocol, slots[slot_id]) run_id = cast(str, record["run_id"]) if run_id in run_ids: @@ -866,6 +987,8 @@ def select_eligible_attempts( def _interval(values: Sequence[float], critical_value: float) -> dict[str, object]: + """Compute the predeclared normal-approximation interval.""" + count = len(values) if count == 0: return {"eligible_n": 0, "mean": None, "sample_variance": None, "interval_95": None} @@ -888,6 +1011,8 @@ def _interval(values: Sequence[float], critical_value: float) -> dict[str, objec def _finite_float(value: object) -> float: + """Return one finite non-boolean numeric value.""" + if ( isinstance(value, bool) or not isinstance(value, (int, float)) @@ -897,17 +1022,33 @@ def _finite_float(value: object) -> float: return float(value) -def _comparison( - declaration: Mapping[str, object], conditions: Sequence[Mapping[str, object]] -) -> dict[str, object]: - published = cast( - dict[str, object], cast(dict[str, object], declaration["comparison"])["published"] - ) - published_by_id = { - cast(str, item["condition_id"]): item - for item in cast(list[dict[str, object]], published["conditions"]) - } - minimum = cast(dict[str, object], declaration["aggregation"])["minimum_eligible_per_condition"] +def _reproduction_classification(*, exact: bool, bounded: bool, complete: bool) -> str: + """Return the predeclared result class without nested conditional expressions.""" + + if exact: + return "exact" + if bounded: + return "bounded" + return "failed" if complete else "unavailable" + + +def _outcome_tier_result(outcome_class: object) -> str: + """Map the aggregate result class to the CybORG outcome tier.""" + + if outcome_class in {"exact", "bounded"}: + return "pass" + if outcome_class == "failed": + return "fail" + return "weakened" + + +def _condition_comparison_flags( + conditions: Sequence[Mapping[str, object]], + published_by_id: Mapping[str, Mapping[str, object]], + minimum: object, +) -> tuple[bool, bool, bool]: + """Return completeness plus exact and bounded condition-level flags.""" + complete = all(item["eligible_n"] == minimum for item in conditions) exact = complete bounded = complete @@ -919,13 +1060,30 @@ def _comparison( continue decimals = cast(int, oracle["display_decimals"]) exact = exact and round(float(mean), decimals) == _finite_float(oracle["mean"]) - bounded = bounded and abs(float(mean) - _finite_float(oracle["mean"])) <= _finite_float( - oracle["bounded_margin"] - ) - total_mean = math.fsum(cast(float, item["mean"]) for item in conditions) if complete else None + difference = abs(float(mean) - _finite_float(oracle["mean"])) + bounded = bounded and difference <= _finite_float(oracle["bounded_margin"]) + return complete, exact, bounded + + +def _primary_comparison( + declaration: Mapping[str, object], + conditions: Sequence[Mapping[str, object]], + published: Mapping[str, object], + *, + complete: bool, + bounded: bool, +) -> tuple[float | None, float | None, list[float] | None, bool]: + """Compute the primary total and its interval-level bounded check.""" + + total_mean = None + if complete: + total_mean = math.fsum(cast(float, item["mean"]) for item in conditions) total_variance = None total_interval = None - if complete and all(item["sample_variance"] is not None for item in conditions): + variances_available = complete and all( + item["sample_variance"] is not None for item in conditions + ) + if variances_available: total_variance = math.fsum( cast(float, item["sample_variance"]) / cast(int, item["eligible_n"]) for item in conditions @@ -934,19 +1092,44 @@ def _comparison( cast(dict[str, object], declaration["aggregation"])["critical_value"] ) half_width = critical * math.sqrt(total_variance) - total_interval = [ - cast(float, total_mean) - half_width, - cast(float, total_mean) + half_width, - ] + mean = cast(float, total_mean) + total_interval = [mean - half_width, mean + half_width] published_interval = cast(list[float], published["primary_total_interval_95"]) - bounded = bounded and ( + intervals_cover = ( total_interval[0] <= _finite_float(published["primary_total"]) <= total_interval[1] - and published_interval[0] <= cast(float, total_mean) <= published_interval[1] + and published_interval[0] <= mean <= published_interval[1] ) + bounded = bounded and intervals_cover else: bounded = False - classification = ( - "exact" if exact else "bounded" if bounded else "failed" if complete else "unavailable" + return total_mean, total_variance, total_interval, bounded + + +def _comparison( + declaration: Mapping[str, object], conditions: Sequence[Mapping[str, object]] +) -> dict[str, object]: + """Compare observed conditions with the frozen published values.""" + + published = cast( + dict[str, object], cast(dict[str, object], declaration["comparison"])["published"] + ) + published_by_id = { + cast(str, item["condition_id"]): item + for item in cast(list[dict[str, object]], published["conditions"]) + } + minimum = cast(dict[str, object], declaration["aggregation"])["minimum_eligible_per_condition"] + complete, exact, bounded = _condition_comparison_flags(conditions, published_by_id, minimum) + total_mean, total_variance, total_interval, bounded = _primary_comparison( + declaration, + conditions, + published, + complete=complete, + bounded=bounded, + ) + classification = _reproduction_classification( + exact=exact, + bounded=bounded, + complete=complete, ) return { "classification": classification, @@ -1011,32 +1194,31 @@ def build_tiers( comparison = cast(dict[str, object], aggregates["comparison"]) outcome_class = comparison["classification"] cyborg_results = { - "authored-source": ("pass", ["protocol.json", "source-ledger.json"]), - "contract": ("pass", ["protocol.json", "runs/"]), + "authored-source": ("pass", [_PROTOCOL_FILE, _SOURCE_LEDGER_FILE]), + "contract": ("pass", [_PROTOCOL_FILE, "runs/"]), "execution-control": ( "weakened", ["source-ledger.json#loss-evaluation-seed-unbound", "protocol.json#seed_policy"], ), - "state/observation": ( + _STATE_OBSERVATION_TIER: ( "weakened", ["source-ledger.json#loss-native-observation-boundary"], ), - "outcome/evaluation": ( - "pass" - if outcome_class in {"exact", "bounded"} - else "fail" - if outcome_class == "failed" - else "weakened", + _OUTCOME_EVALUATION_TIER: ( + _outcome_tier_result(outcome_class), ["aggregates.json#comparison"], ), - "disclosure": ("pass", ["source-ledger.json#loss_refs", "validation-report.md"]), + "disclosure": ( + "pass", + [f"{_SOURCE_LEDGER_FILE}#loss_refs", _VALIDATION_REPORT_FILE], + ), } reference_keys = { "authored-source": "authored_source", "contract": "contract", "execution-control": "execution_control", - "state/observation": "state_observation", - "outcome/evaluation": "outcome_evaluation", + _STATE_OBSERVATION_TIER: "state_observation", + _OUTCOME_EVALUATION_TIER: "outcome_evaluation", "disclosure": "disclosure", } tier_rows = [] @@ -1090,16 +1272,47 @@ def build_tiers( def _scan_value(value: object) -> None: + """Recursively reject forbidden native or host material.""" + if isinstance(value, dict): for key, item in value.items(): if key in _FORBIDDEN_FIELDS: - raise ValueError("forbidden native material") + raise ValueError(_INVALID_FORBIDDEN_MATERIAL) _scan_value(item) elif isinstance(value, list): for item in value: _scan_value(item) elif isinstance(value, str) and any(token in value for token in _FORBIDDEN_TEXT): - raise ValueError("forbidden native material") + raise ValueError(_INVALID_FORBIDDEN_MATERIAL) + + +def _strict_public_json(path: Path, raw: bytes) -> object: + """Decode one bounded public JSON or gzip-compressed JSON artifact.""" + + if path.name.endswith(_GZIP_JSON_SUFFIX): + raw = gzip.decompress(raw) + if len(raw) > _MAX_DECOMPRESSED_JSON_BYTES: + raise ValueError(_INVALID_JSON) + return json.loads( + raw, + object_pairs_hook=_closed_pairs, + parse_constant=_reject_constant, + ) + + +def _scan_public_file(path: Path) -> None: + """Scan one regular public file for forbidden material and JSON fields.""" + + raw = path.read_bytes() + if any(token.encode() in raw for token in _FORBIDDEN_TEXT): + raise ValueError(_INVALID_FORBIDDEN_MATERIAL) + if path.suffix != ".json" and not path.name.endswith(_GZIP_JSON_SUFFIX): + return + try: + value = _strict_public_json(path, raw) + except ValueError as error: + raise ValueError(_INVALID_JSON) from error + _scan_value(value) def scan_public_tree(root: Path) -> None: @@ -1110,25 +1323,8 @@ def scan_public_tree(root: Path) -> None: for path in root.rglob("*"): if path.is_symlink() or not (path.is_file() or path.is_dir()): raise ValueError("public tree is invalid") - if not path.is_file(): - continue - raw = path.read_bytes() - if any(token.encode() in raw for token in _FORBIDDEN_TEXT): - raise ValueError("forbidden native material") - if path.suffix == ".json" or path.name.endswith(".json.gz"): - try: - if path.name.endswith(".json.gz"): - raw = gzip.decompress(raw) - if len(raw) > _MAX_DECOMPRESSED_JSON_BYTES: - raise ValueError(_INVALID_JSON) - value = json.loads( - raw, - object_pairs_hook=_closed_pairs, - parse_constant=_reject_constant, - ) - except (UnicodeError, ValueError, gzip.BadGzipFile) as error: - raise ValueError(_INVALID_JSON) from error - _scan_value(value) + if path.is_file(): + _scan_public_file(path) def _reserve_directory(path: Path) -> Path: @@ -1140,6 +1336,8 @@ def _reserve_directory(path: Path) -> Path: def _write_json(path: Path, payload: object) -> None: + """Atomically write one canonical JSON object.""" + atomic_write_json_artifact(path, payload) @@ -1184,12 +1382,16 @@ def _write_text(path: Path, value: str) -> None: def _media_type(path: Path) -> str: + """Return the allowlisted media type for one exported file.""" + if path.suffix == ".gz": return "application/gzip" return "application/json" if path.suffix == ".json" else "text/markdown" def _inventory_entry(root: Path, path: Path) -> dict[str, object]: + """Build one bounded relative inventory entry.""" + if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root.resolve()): raise ValueError("inventory member is invalid") content = path.read_bytes() @@ -1204,55 +1406,80 @@ def _inventory_entry(root: Path, path: Path) -> dict[str, object]: def _seal_inventory(root: Path, members: Sequence[Path]) -> dict[str, object]: + """Write an inventory after all of its members are final.""" + inventory: dict[str, object] = { "artifacts": [ _inventory_entry(root, path) for path in sorted(members, key=lambda item: item.relative_to(root).as_posix()) ] } - _write_json(root / "inventory.json", inventory) + _write_json(root / _INVENTORY_FILE, inventory) return inventory +def _inventory_item_mapping(item: object) -> tuple[dict[str, object], str]: + """Return one shape-validated inventory entry and relative path.""" + + expected_keys = {"path", "media_type", "sha256", "size_bytes"} + if not isinstance(item, dict) or set(item) != expected_keys: + raise ValueError(_INVALID_INVENTORY) + relative = item["path"] + if not isinstance(relative, str): + raise ValueError(_INVALID_INVENTORY) + return item, relative + + +def _inventory_candidate(root: Path, relative: str, seen: set[str]) -> Path: + """Resolve one safe, unique, regular inventory member path.""" + + source = root / relative + candidate = source.resolve() + invalid_path = ( + relative in seen + or Path(relative).is_absolute() + or ".." in Path(relative).parts + or not candidate.is_relative_to(root.resolve()) + or source.is_symlink() + or not candidate.is_file() + ) + if invalid_path: + raise ValueError(_INVALID_INVENTORY) + return candidate + + +def _validated_inventory_member(root: Path, item: object, seen: set[str]) -> str: + """Validate one inventory member and return its relative path.""" + + mapping, relative = _inventory_item_mapping(item) + candidate = _inventory_candidate(root, relative, seen) + if mapping != _inventory_entry(root, candidate): + raise ValueError("inventory digest is invalid") + return relative + + +def _verify_direct_inventory_members(root: Path, seen: set[str]) -> None: + """Require the inventory to cover every direct regular file.""" + + direct_files = { + path.name for path in root.iterdir() if path.is_file() and path.name != _INVENTORY_FILE + } + inventoried = {Path(relative).name for relative in seen if len(Path(relative).parts) == 1} + if direct_files != inventoried: + raise ValueError("inventory membership is incomplete") + + def _verify_inventory(root: Path, *, expected_members: set[str] | None = None) -> dict[str, object]: - inventory = load_strict_json(root / "inventory.json") + """Verify one sealed inventory and its exact membership.""" + + inventory = load_strict_json(root / _INVENTORY_FILE) if set(inventory) != {"artifacts"} or not isinstance(inventory["artifacts"], list): - raise ValueError("inventory is invalid") + raise ValueError(_INVALID_INVENTORY) seen: set[str] = set() for item in inventory["artifacts"]: - if not isinstance(item, dict) or set(item) != { - "path", - "media_type", - "sha256", - "size_bytes", - }: - raise ValueError("inventory is invalid") - relative = item["path"] - if not isinstance(relative, str): - raise ValueError("inventory is invalid") - source = root / relative - candidate = source.resolve() - if ( - relative in seen - or Path(relative).is_absolute() - or ".." in Path(relative).parts - or not candidate.is_relative_to(root.resolve()) - or source.is_symlink() - or not candidate.is_file() - ): - raise ValueError("inventory is invalid") + relative = _validated_inventory_member(root, item, seen) seen.add(relative) - expected = _inventory_entry(root, candidate) - if item != expected: - raise ValueError("inventory digest is invalid") - direct_files = { - path.name for path in root.iterdir() if path.is_file() and path.name != "inventory.json" - } - inventoried_direct_files = { - Path(relative).name for relative in seen if len(Path(relative).parts) == 1 - } - if direct_files != inventoried_direct_files: - raise ValueError("inventory membership is incomplete") + _verify_direct_inventory_members(root, seen) if expected_members is not None and seen != expected_members: raise ValueError("inventory membership is incomplete") return inventory @@ -1261,6 +1488,8 @@ def _verify_inventory(root: Path, *, expected_members: set[str] | None = None) - def _write_schedule_partitions( root: Path, protocol: Mapping[str, object], selection: ReproductionSelection ) -> list[Path]: + """Write the bounded per-condition schedule files.""" + plans = root / "plans" os.mkdir(plans, 0o700) by_condition: dict[str, list[dict[str, object]]] = {} @@ -1288,8 +1517,8 @@ def write_declaration( root = _reserve_directory(output) protocol, source_ledger = build_declaration(repo_root, selection=selection) - protocol_path = root / "protocol.json" - ledger_path = root / "source-ledger.json" + protocol_path = root / _PROTOCOL_FILE + ledger_path = root / _SOURCE_LEDGER_FILE _write_json(protocol_path, protocol) _write_json(ledger_path, source_ledger) plans = _write_schedule_partitions(root, protocol, selection) @@ -1306,6 +1535,8 @@ def _load_study_inputs( ParticipantImplementationSelectionModel, ParticipantConfigurationResultModel, ]: + """Load the authored scenario and published run contracts.""" + from raes import parse_sdl # type: ignore[import-untyped] scenario = parse_sdl( @@ -1360,7 +1591,7 @@ def capture_reference_path(repo_root: Path) -> dict[str, object]: "evaluation": len(result.execution_plan.evaluation.operations), }, "evidence_refs": [ - "reference.json", + _REFERENCE_FILE, "protocol.json#explicit_non_claims", ], "limitations": [ @@ -1392,23 +1623,48 @@ def _validate_reference(reference: Mapping[str, object]) -> None: }, "reference evidence", ) + _validate_reference_identity(reference) + _validate_reference_diagnostics(reference) + _validate_reference_counts(reference) + _validate_reference_limitations(reference) + + +def _validate_reference_identity(reference: Mapping[str, object]) -> None: + """Validate the fixed reference-processor identity and tier results.""" + if ( reference["processor"] != "raes-processor-reference" or reference["scenario_name"] != "cage2-research" or reference["target_manifest"] != "cyborg-cage2" - or reference["execution_control"] != "weakened" - or reference["state_observation"] != "weakened" - or reference["outcome_evaluation"] != "weakened" - or reference["evidence_refs"] != ["reference.json", "protocol.json#explicit_non_claims"] + or reference["evidence_refs"] != [_REFERENCE_FILE, f"{_PROTOCOL_FILE}#explicit_non_claims"] ): raise ValueError("reference evidence identity is invalid") + _validate_reference_tiers(reference) + + +def _validate_reference_tiers(reference: Mapping[str, object]) -> None: + """Validate the fixed reference-path tier strengths.""" + + weakened = ("execution_control", "state_observation", "outcome_evaluation") + if any(reference[key] != "weakened" for key in weakened): + raise ValueError("reference evidence identity is invalid") if any(reference[key] not in {"pass", "fail"} for key in ("authored_source", "contract")): raise ValueError("reference evidence result is invalid") if reference["disclosure"] != "pass": raise ValueError("reference evidence result is invalid") + + +def _validate_reference_diagnostics(reference: Mapping[str, object]) -> None: + """Validate the bounded planning diagnostic code list.""" + codes = reference["planning_error_codes"] if not isinstance(codes, list) or any(not isinstance(code, str) or not code for code in codes): raise ValueError("reference diagnostics are invalid") + + +def _validate_reference_counts(reference: Mapping[str, object]) -> None: + """Validate the three non-negative planned-operation counts.""" + counts = reference["planned_operation_counts"] if ( not isinstance(counts, dict) @@ -1416,6 +1672,11 @@ def _validate_reference(reference: Mapping[str, object]) -> None: or any(type(value) is not int or value < 0 for value in counts.values()) ): raise ValueError("reference operation counts are invalid") + + +def _validate_reference_limitations(reference: Mapping[str, object]) -> None: + """Validate the fixed reference-path limitations statement.""" + limitations = reference["limitations"] if not isinstance(limitations, list) or limitations != [ "The reference processor realizes authored contracts and a target plan only.", @@ -1425,6 +1686,8 @@ def _validate_reference(reference: Mapping[str, object]) -> None: def _evidence_step(record: ExperimentEvidenceRecordModel) -> int: + """Return the logical step bound to one evidence record.""" + for ref in record.source_refs: if ref.ref_id.startswith("logical-step:"): return int(ref.ref_id.removeprefix("logical-step:")) @@ -1490,6 +1753,8 @@ def _write_episode_evidence( task: ExperimentTaskModel, scenario_digest: str, ) -> dict[str, object]: + """Persist one valid episode's exact RAES evidence closure.""" + evidence_records = episode.evidence_records measures = episode.derived_measures referenced = {ref.ref_id for measure in measures for ref in measure.source_evidence_refs} @@ -1508,7 +1773,7 @@ def _write_episode_evidence( "diagnostics": [item.model_dump(mode="json") for item in episode.diagnostics], }, ) - evidence_index = run_root / "evidence-index.json" + evidence_index = run_root / _EVIDENCE_INDEX_FILE _write_json( evidence_index, { @@ -1536,15 +1801,24 @@ def _write_episode_evidence( ], sensitivity="redacted", ) + retained_episode = EpisodeEvidence( + completed_steps=episode.completed_steps, + evidence_records=tuple(retained), + derived_measures=episode.derived_measures, + proposition_truth_results=episode.proposition_truth_results, + objective_results=episode.objective_results, + diagnostics=episode.diagnostics, + cleanup_verified=episode.cleanup_verified, + ) run = archival_run( controls=controls, scenario_digest=scenario_digest, task=task, - episode=replace(episode, evidence_records=tuple(retained)), + episode=retained_episode, evidence_artifact=artifact, ) validate_experiment_run_against_task(task, run) - _write_gzip_json(run_root / "run.json.gz", run.model_dump(mode="json")) + _write_gzip_json(run_root / _RUN_FILE, run.model_dump(mode="json")) score = next( ( item @@ -1561,9 +1835,9 @@ def _write_episode_evidence( return { "score_measure": score.model_dump(mode="json"), "evidence_refs": [ - "evidence-index.json", + _EVIDENCE_INDEX_FILE, "episode-summary.json.gz", - "run.json.gz", + _RUN_FILE, ], } @@ -1571,14 +1845,11 @@ def _write_episode_evidence( def _attempt_record( protocol: Mapping[str, object], slot: Mapping[str, object], - *, attempt_sequence: int, - disposition: str, - cleanup_verified: bool, - score_measure: object = None, - evidence_refs: Sequence[str] = (), - diagnostic_code: str | None = None, + outcome: _AttemptOutcome, ) -> dict[str, object]: + """Build one immutable terminal operational attempt record.""" + slot_id = cast(str, slot["slot_id"]) run_id = f"{slot_id}-attempt-{attempt_sequence:02d}" record = { @@ -1593,11 +1864,11 @@ def _attempt_record( "trial_length": slot["trial_length"], "red_variant": slot["red_variant"], "episode_ordinal": slot["episode_ordinal"], - "disposition": disposition, - "diagnostic_code": diagnostic_code, - "cleanup_verified": cleanup_verified, - "score_measure": score_measure, - "evidence_refs": list(evidence_refs), + "disposition": outcome.disposition, + "diagnostic_code": outcome.diagnostic_code, + "cleanup_verified": outcome.cleanup_verified, + "score_measure": outcome.score_measure, + "evidence_refs": list(outcome.evidence_refs), } _validate_attempt(record, protocol, slot) return record @@ -1619,6 +1890,8 @@ def _condition_controls( selection: ParticipantImplementationSelectionModel, configuration: ParticipantConfigurationResultModel, ) -> tuple[RunControls, ...]: + """Bind one condition's logical slots to RAES run controls.""" + declaration = cast(dict[str, object], protocol["declaration"]) seed = cast(int, cast(dict[str, object], declaration["seed_policy"])["seed"]) return tuple( @@ -1636,7 +1909,11 @@ def _condition_controls( def _write_environment(protocol: Mapping[str, object]) -> dict[str, object]: + """Project bounded apparatus provenance without host-sensitive values.""" + def version(name: str) -> str: + """Return one installed distribution version or an absence marker.""" + try: return metadata.version(name) except metadata.PackageNotFoundError: @@ -1718,6 +1995,8 @@ def _validate_environment( def _validation_report_text(tiers: Mapping[str, object]) -> str: + """Render the stable human-readable validation summary.""" + return ( "# CAGE-2 protocol reproduction validation\n\n" f"Frozen claim: `{tiers['strongest_supported_claim']}`.\n\n" @@ -1734,206 +2013,563 @@ def _validation_report_text(tiers: Mapping[str, object]) -> str: def _write_validation_report(root: Path, tiers: Mapping[str, object]) -> Path: - path = root / "validation-report.md" + """Write the stable validation summary.""" + + path = root / _VALIDATION_REPORT_FILE _write_text(path, _validation_report_text(tiers)) return path def _condition_inventory(condition_root: Path) -> Path: - members = [condition_root / "cleanup.json"] - members.extend(sorted(condition_root.glob("*/inventory.json"))) + """Seal one condition from its cleanup and attempt inventories.""" + + members = [condition_root / _CLEANUP_FILE] + members.extend(sorted(condition_root.glob(f"*/{_INVENTORY_FILE}"))) _seal_inventory(condition_root, members) - return condition_root / "inventory.json" + return condition_root / _INVENTORY_FILE -def run_full_study( - repo_root: Path, - output: Path, - *, - selection: ReproductionSelection = FROZEN_SELECTION, - driver: object | None = None, - progress: Callable[[str, int, int], None] | None = None, -) -> Path: - """Execute, retain, derive, and seal the complete frozen native study.""" +class _StudyContext(NamedTuple): + """Immutable dependencies shared across every condition attempt.""" - source_root = repo_root.resolve() - root = _reserve_directory(output) - protocol, source_ledger = build_declaration(source_root, selection=selection) - protocol_path = root / "protocol.json" - ledger_path = root / "source-ledger.json" - _write_json(protocol_path, protocol) - _write_json(ledger_path, source_ledger) - plan_paths = _write_schedule_partitions(root, protocol, selection) - scenario, task, manifest, participant_selection, configuration = _load_study_inputs(source_root) - reference = capture_reference_path(source_root) - reference_path = root / "reference.json" - _write_json(reference_path, reference) - environment_path = root / "environment.json" - _write_json(environment_path, _write_environment(protocol)) - runtime_driver = cast( + root: Path + protocol: Mapping[str, object] + scenario: object + task: ExperimentTaskModel + manifest: ParticipantImplementationManifestModel + participant_selection: ParticipantImplementationSelectionModel + configuration: ParticipantConfigurationResultModel + driver: CyborgDriver + checkpoint_stream: Callable[[], object] + restore_stream: Callable[[object], None] + scenario_digest: str + progress: Callable[[str, int, int], None] | None + + +class _ConditionAttemptResult(NamedTuple): + """Execution state needed to seal one complete condition attempt.""" + + sequence: int + succeeded: bool + attempt_slots: list[dict[str, object]] + staged: dict[str, dict[str, object]] + run_roots: dict[str, Path] + + +def _recoverable_study_driver( + driver: object | None, selection: ReproductionSelection +) -> tuple[CyborgDriver, Callable[[], object], Callable[[object], None]]: + """Initialize and return a driver with recoverable ordered-stream controls.""" + + runtime_driver = cast( CyborgDriver, driver if driver is not None else SourceInstalledCyborgDriver(expected_version="2.1"), ) - begin_stream_value = getattr(runtime_driver, "begin_ordered_stream", None) - checkpoint_stream_value = getattr(runtime_driver, "ordered_stream_checkpoint", None) - restore_stream_value = getattr(runtime_driver, "restore_ordered_stream", None) - if not all( - callable(item) - for item in (begin_stream_value, checkpoint_stream_value, restore_stream_value) - ): + begin_value = getattr(runtime_driver, "begin_ordered_stream", None) + checkpoint_value = getattr(runtime_driver, "ordered_stream_checkpoint", None) + restore_value = getattr(runtime_driver, "restore_ordered_stream", None) + if not all(callable(item) for item in (begin_value, checkpoint_value, restore_value)): raise ValueError("study driver has no recoverable ordered stream") - begin_stream = cast(Callable[[int], None], begin_stream_value) - checkpoint_stream = cast(Callable[[], object], checkpoint_stream_value) - restore_stream = cast(Callable[[object], None], restore_stream_value) + begin_stream = cast(Callable[[int], None], begin_value) + checkpoint_stream = cast(Callable[[], object], checkpoint_value) + restore_stream = cast(Callable[[object], None], restore_value) begin_stream(selection.seed) - runs_root = root / "runs" - os.mkdir(runs_root, 0o700) - all_attempts: list[dict[str, object]] = [] - condition_inventories: list[Path] = [] + return runtime_driver, checkpoint_stream, restore_stream + + +def _condition_attempt_roots( + condition_root: Path, attempt_slots: Sequence[Mapping[str, object]] +) -> dict[str, Path]: + """Reserve one private directory for every run in a condition attempt.""" + + roots: dict[str, Path] = {} + for attempt_slot in attempt_slots: + run_id = cast(str, attempt_slot["run_id"]) + run_root = condition_root / run_id + os.mkdir(run_root, 0o700) + roots[run_id] = run_root + return roots + + +def _execute_condition_attempt( + context: _StudyContext, + condition_id: str, + slots: Sequence[Mapping[str, object]], + sequence: int, +) -> _ConditionAttemptResult: + """Execute one full condition attempt and retain its per-run evidence.""" + + attempt_slots = [_slot_for_attempt(slot, sequence) for slot in slots] + condition_root = context.root / "runs" / condition_id + run_roots = _condition_attempt_roots(condition_root, attempt_slots) + staged: dict[str, dict[str, object]] = {} + + def consume(controls: RunControls, episode: EpisodeEvidence) -> None: + """Persist one episode and report bounded condition progress.""" + + run_root = run_roots[controls.run_id] + staged[controls.run_id] = _write_episode_evidence( + run_root, + (run_root / _EVIDENCE_INDEX_FILE).relative_to(context.root).as_posix(), + controls, + episode, + context.task, + context.scenario_digest, + ) + if context.progress is not None: + context.progress(condition_id, len(staged), len(slots)) + + controls = _condition_controls( + attempt_slots, + context.protocol, + context.manifest, + context.participant_selection, + context.configuration, + ) + succeeded = False + try: + with ( + Path(os.devnull).open("w", encoding="utf-8") as sink, + redirect_stdout(sink), + redirect_stderr(sink), + ): + execute_episode_series( + context.scenario, + controls, + driver=context.driver, + episode_consumer=consume, + retain_evidence=False, + ) + succeeded = len(staged) == len(slots) + except Exception: + succeeded = False + return _ConditionAttemptResult(sequence, succeeded, attempt_slots, staged, run_roots) + + +def _discard_failed_run(run_root: Path) -> None: + """Delete incomplete portable artifacts after validating their boundary.""" + + for artifact in run_root.iterdir(): + if artifact.is_symlink() or not artifact.is_file(): + raise ValueError("failed attempt artifact boundary is invalid") + artifact.unlink() + + +def _condition_attempt_records( + protocol: Mapping[str, object], + slots: Sequence[Mapping[str, object]], + result: _ConditionAttemptResult, +) -> list[dict[str, object]]: + """Seal and return every operational record from one condition attempt.""" + + records: list[dict[str, object]] = [] + for slot, attempt_slot in zip(slots, result.attempt_slots, strict=True): + run_id = cast(str, attempt_slot["run_id"]) + run_root = result.run_roots[run_id] + if result.succeeded: + stage = result.staged[run_id] + outcome = _AttemptOutcome( + disposition="valid", + cleanup_verified=True, + score_measure=stage["score_measure"], + evidence_refs=cast(list[str], stage["evidence_refs"]), + ) + else: + _discard_failed_run(run_root) + outcome = _AttemptOutcome( + disposition="failed", + cleanup_verified=False, + diagnostic_code="cyborg-reproduction.condition-session-failed", + ) + record = _attempt_record(protocol, slot, result.sequence, outcome) + _write_gzip_json(run_root / _ATTEMPT_FILE, record) + _seal_inventory(run_root, [item for item in run_root.rglob("*") if item.is_file()]) + records.append(record) + return records + + +def _run_study_condition( + context: _StudyContext, + condition_id: str, + slots: Sequence[Mapping[str, object]], + retry_limit: int, +) -> tuple[list[dict[str, object]], Path]: + """Execute the retry-bounded lifecycle for one independent condition.""" + + condition_root = context.root / "runs" / condition_id + os.mkdir(condition_root, 0o700) + checkpoint = context.checkpoint_stream() + attempts: list[dict[str, object]] = [] + cleanup_attempts: list[dict[str, object]] = [] + completed = False + for sequence in range(1, retry_limit + 2): + if sequence > 1: + context.restore_stream(checkpoint) + result = _execute_condition_attempt(context, condition_id, slots, sequence) + cleanup_attempts.append( + { + "attempt_sequence": sequence, + "run_count": len(result.attempt_slots), + "verified": result.succeeded, + } + ) + attempts.extend(_condition_attempt_records(context.protocol, slots, result)) + if result.succeeded: + completed = True + break + _write_json( + condition_root / _CLEANUP_FILE, + { + "condition_id": condition_id, + "completed": completed, + "attempts": cleanup_attempts, + }, + ) + return attempts, _condition_inventory(condition_root) + + +def _execute_study_schedule( + context: _StudyContext, selection: ReproductionSelection +) -> tuple[list[dict[str, object]], list[Path]]: + """Execute all frozen conditions in their declared stream order.""" + + os.mkdir(context.root / "runs", 0o700) schedule = _schedule(selection) - scenario_digest = task.scenario_ref.ref_digest - if not isinstance(scenario_digest, str): - raise ValueError("task scenario digest is unavailable") + all_attempts: list[dict[str, object]] = [] + inventories: list[Path] = [] for partition in _schedule_partitions(selection): condition_id = cast(str, partition["condition_id"]) - condition_root = runs_root / condition_id - os.mkdir(condition_root, 0o700) slots = [item for item in schedule if item["condition_id"] == condition_id] - condition_checkpoint = checkpoint_stream() - cleanup_attempts: list[dict[str, object]] = [] - condition_completed = False - for attempt_sequence in range(1, selection.retry_limit + 2): - if attempt_sequence > 1: - restore_stream(condition_checkpoint) - attempt_slots = [_slot_for_attempt(slot, attempt_sequence) for slot in slots] - staged: dict[str, dict[str, object]] = {} - run_roots: dict[str, Path] = {} - for attempt_slot in attempt_slots: - run_id = cast(str, attempt_slot["run_id"]) - run_root = condition_root / run_id - os.mkdir(run_root, 0o700) - run_roots[run_id] = run_root - - def consume( - controls: RunControls, - episode: EpisodeEvidence, - *, - _staged: dict[str, dict[str, object]] = staged, - _run_roots: dict[str, Path] = run_roots, - _condition_id: str = condition_id, - _slot_count: int = len(slots), - ) -> None: - _staged[controls.run_id] = _write_episode_evidence( - _run_roots[controls.run_id], - (_run_roots[controls.run_id] / "evidence-index.json") - .relative_to(root) - .as_posix(), - controls, - episode, - task, - scenario_digest, - ) - if progress is not None: - progress(_condition_id, len(_staged), _slot_count) - - controls = _condition_controls( - attempt_slots, - protocol, - manifest, - participant_selection, - configuration, - ) - condition_success = False - try: - with ( - Path(os.devnull).open("w", encoding="utf-8") as sink, - redirect_stdout(sink), - redirect_stderr(sink), - ): - execute_episode_series( - scenario, - controls, - driver=runtime_driver, - episode_consumer=consume, - retain_evidence=False, - ) - condition_success = len(staged) == len(slots) - except Exception: - condition_success = False - cleanup_attempts.append( - { - "attempt_sequence": attempt_sequence, - "run_count": len(attempt_slots), - "verified": condition_success, - } - ) - for slot, attempt_slot in zip(slots, attempt_slots, strict=True): - run_id = cast(str, attempt_slot["run_id"]) - run_root = run_roots[run_id] - if condition_success: - stage = staged[run_id] - record = _attempt_record( - protocol, - slot, - attempt_sequence=attempt_sequence, - disposition="valid", - cleanup_verified=True, - score_measure=stage["score_measure"], - evidence_refs=cast(list[str], stage["evidence_refs"]), - ) - else: - for artifact in run_root.iterdir(): - if artifact.is_symlink() or not artifact.is_file(): - raise ValueError("failed attempt artifact boundary is invalid") - artifact.unlink() - record = _attempt_record( - protocol, - slot, - attempt_sequence=attempt_sequence, - disposition="failed", - cleanup_verified=False, - diagnostic_code="cyborg-reproduction.condition-session-failed", - ) - attempt_path = run_root / "attempt.json.gz" - _write_gzip_json(attempt_path, record) - members = [item for item in run_root.rglob("*") if item.is_file()] - _seal_inventory(run_root, members) - all_attempts.append(record) - if condition_success: - condition_completed = True - break - _write_json( - condition_root / "cleanup.json", - { - "condition_id": condition_id, - "completed": condition_completed, - "attempts": cleanup_attempts, - }, + attempts, inventory = _run_study_condition( + context, condition_id, slots, selection.retry_limit ) - condition_inventories.append(_condition_inventory(condition_root)) - aggregates = compute_aggregates(protocol, all_attempts) - aggregates_path = root / "aggregates.json" + all_attempts.extend(attempts) + inventories.append(inventory) + return all_attempts, inventories + + +def _finalize_study_bundle( + root: Path, + protocol: Mapping[str, object], + reference: Mapping[str, object], + attempts: Sequence[Mapping[str, object]], + members: Sequence[Path], +) -> None: + """Write the deterministic projections and seal the completed study.""" + + aggregates = compute_aggregates(protocol, attempts) + aggregates_path = root / _AGGREGATES_FILE _write_json(aggregates_path, aggregates) tiers = build_tiers(protocol, aggregates, reference) - tiers_path = root / "tiers.json" + tiers_path = root / _TIERS_FILE _write_json(tiers_path, tiers) report_path = _write_validation_report(root, tiers) scan_public_tree(root) - root_members = [ + _seal_inventory(root, [*members, aggregates_path, tiers_path, report_path]) + + +def run_full_study( + repo_root: Path, + output: Path, + *, + selection: ReproductionSelection = FROZEN_SELECTION, + driver: object | None = None, + progress: Callable[[str, int, int], None] | None = None, +) -> Path: + """Execute, retain, derive, and seal the complete frozen native study.""" + + source_root = repo_root.resolve() + root = _reserve_directory(output) + protocol, source_ledger = build_declaration(source_root, selection=selection) + protocol_path = root / _PROTOCOL_FILE + ledger_path = root / _SOURCE_LEDGER_FILE + _write_json(protocol_path, protocol) + _write_json(ledger_path, source_ledger) + plan_paths = _write_schedule_partitions(root, protocol, selection) + scenario, task, manifest, participant_selection, configuration = _load_study_inputs(source_root) + reference = capture_reference_path(source_root) + reference_path = root / _REFERENCE_FILE + _write_json(reference_path, reference) + environment_path = root / _ENVIRONMENT_FILE + _write_json(environment_path, _write_environment(protocol)) + runtime_driver, checkpoint_stream, restore_stream = _recoverable_study_driver(driver, selection) + scenario_digest = task.scenario_ref.ref_digest + if not isinstance(scenario_digest, str): + raise ValueError("task scenario digest is unavailable") + context = _StudyContext( + root, + protocol, + scenario, + task, + manifest, + participant_selection, + configuration, + runtime_driver, + checkpoint_stream, + restore_stream, + scenario_digest, + progress, + ) + all_attempts, condition_inventories = _execute_study_schedule(context, selection) + root_members = ( protocol_path, ledger_path, reference_path, environment_path, - aggregates_path, - tiers_path, - report_path, *plan_paths, *condition_inventories, - ] - _seal_inventory(root, root_members) + ) + _finalize_study_bundle(root, protocol, reference, all_attempts, root_members) return root +def _validated_evidence_index(run_root: Path) -> dict[str, object]: + """Load and validate the top-level evidence index shape.""" + + index = load_strict_json(run_root / _EVIDENCE_INDEX_FILE) + _require_exact_keys(index, {"record_count", "chunks", "episode_summary"}, "evidence index") + chunks = index["chunks"] + if not isinstance(chunks, list) or not chunks: + raise ValueError("evidence index is invalid") + return index + + +def _evidence_chunk_mapping(chunk: object) -> dict[str, object]: + """Return one shape-validated evidence chunk entry.""" + + if not isinstance(chunk, dict): + raise ValueError("evidence index is invalid") + _require_exact_keys( + chunk, + {"path", "sha256", "record_count", "first_logical_step", "last_logical_step"}, + "evidence chunk", + ) + return chunk + + +def _chunk_records( + run_root: Path, chunk: Mapping[str, object] +) -> list[ExperimentEvidenceRecordModel]: + """Load and validate the contents of one evidence chunk.""" + + relative = chunk["path"] + if not isinstance(relative, str) or Path(relative).name != relative: + raise ValueError("evidence chunk path is invalid") + path = run_root / relative + if _sha256_file(path) != chunk["sha256"]: + raise ValueError("evidence chunk digest is invalid") + values = _load_strict_json_value(path) + if not isinstance(values, list) or len(values) != chunk["record_count"]: + raise ValueError("evidence chunk is invalid") + return [ExperimentEvidenceRecordModel.model_validate(item) for item in values] + + +def _validate_chunk_order( + records: Sequence[ExperimentEvidenceRecordModel], chunk: Mapping[str, object] +) -> None: + """Validate monotonic steps and the declared evidence-chunk bounds.""" + + steps = [_evidence_step(record) for record in records] + valid_order = ( + bool(steps) + and steps == sorted(steps) + and min(steps) == chunk["first_logical_step"] + and max(steps) == chunk["last_logical_step"] + ) + if not valid_order: + raise ValueError("evidence chunk ordering is invalid") + + +def _validated_chunk_records(run_root: Path, chunk: object) -> list[ExperimentEvidenceRecordModel]: + """Load and validate one ordered evidence chunk.""" + + mapping = _evidence_chunk_mapping(chunk) + records = _chunk_records(run_root, mapping) + _validate_chunk_order(records, mapping) + return records + + +def _validated_evidence_records( + run_root: Path, index: Mapping[str, object] +) -> list[ExperimentEvidenceRecordModel]: + """Return the complete unique record sequence declared by an index.""" + + records = [ + record + for chunk in cast(list[dict[str, object]], index["chunks"]) + for record in _validated_chunk_records(run_root, chunk) + ] + record_ids = [record.evidence_record_id for record in records] + if len(record_ids) != index["record_count"] or len(record_ids) != len(set(record_ids)): + raise ValueError("evidence record closure is invalid") + return records + + +def _validated_episode_summary( + run_root: Path, index: Mapping[str, object] +) -> tuple[Path, dict[str, object], list[ExperimentDerivedMeasureModel]]: + """Load the digest-bound episode summary and its derived measures.""" + + summary_ref = index["episode_summary"] + if not isinstance(summary_ref, dict): + raise ValueError("episode summary reference is invalid") + _require_exact_keys(summary_ref, {"path", "sha256"}, "episode summary reference") + relative = summary_ref["path"] + if not isinstance(relative, str) or Path(relative).name != relative: + raise ValueError("episode summary path is invalid") + summary_path = run_root / relative + if _sha256_file(summary_path) != summary_ref["sha256"]: + raise ValueError("episode summary digest is invalid") + summary = load_strict_json(summary_path) + _require_exact_keys( + summary, + {"derived_measures", "proposition_truth", "objective_results", "diagnostics"}, + "episode summary", + ) + values = summary["derived_measures"] + if not isinstance(values, list) or not values: + raise ValueError("derived measures are invalid") + measures = [ExperimentDerivedMeasureModel.model_validate(item) for item in values] + return summary_path, summary, measures + + +def _referenced_record_ids( + measures: Sequence[ExperimentDerivedMeasureModel], +) -> set[str]: + """Return the exact record identifiers referenced by derived measures.""" + + return {ref.ref_id for measure in measures for ref in measure.source_evidence_refs} + + +def _retained_evidence_records( + records: Sequence[ExperimentEvidenceRecordModel], referenced: set[str] +) -> list[ExperimentEvidenceRecordModel]: + """Select and validate the exact referenced evidence closure.""" + + retained = [record for record in records if record.evidence_record_id in referenced] + if {record.evidence_record_id for record in retained} != referenced: + raise ValueError("derived-measure evidence closure is invalid") + return retained + + +def _rewrite_compacted_index( + run_root: Path, + summary_path: Path, + retained: Sequence[ExperimentEvidenceRecordModel], +) -> bytes: + """Replace evidence chunks and return the rewritten index bytes.""" + + for path in run_root.glob("evidence-*.json.gz"): + path.unlink() + chunks = _write_evidence_chunks(run_root, retained) + index_path = run_root / _EVIDENCE_INDEX_FILE + _write_json( + index_path, + { + "record_count": len(retained), + "chunks": chunks, + "episode_summary": { + "path": summary_path.relative_to(run_root).as_posix(), + "sha256": _sha256_file(summary_path), + }, + }, + ) + return index_path.read_bytes() + + +def _rewrite_compacted_run( + run_root: Path, + task: ExperimentTaskModel, + referenced: set[str], + index_bytes: bytes, +) -> None: + """Rebind one run contract to its compacted evidence index and reseal it.""" + + run_payload = load_strict_json(run_root / _RUN_FILE) + artifacts = cast(list[dict[str, object]], run_payload["evidence_artifacts"]) + if len(artifacts) != 1: + raise ValueError("run evidence artifacts are invalid") + artifacts[0]["checksum"] = { + "algorithm": "sha256", + "value": hashlib.sha256(index_bytes).hexdigest(), + } + artifacts[0]["size_bytes"] = len(index_bytes) + traceability = cast(dict[str, object], run_payload["traceability"]) + traceability["evidence_record_refs"] = [ + item + for item in cast(list[dict[str, object]], traceability["evidence_record_refs"]) + if item["ref_id"] in referenced + ] + run = ExperimentRunModel.model_validate(run_payload) + validate_experiment_run_against_task(task, run) + _write_gzip_json(run_root / _RUN_FILE, run.model_dump(mode="json")) + members = [ + path for path in run_root.iterdir() if path.is_file() and path.name != _INVENTORY_FILE + ] + _seal_inventory(run_root, members) + + +def _compact_valid_run_evidence(run_root: Path, task: ExperimentTaskModel) -> None: + """Reduce one valid run to records referenced by retained measures.""" + + index = _validated_evidence_index(run_root) + summary_path, _, measures = _validated_episode_summary(run_root, index) + referenced = _referenced_record_ids(measures) + records = _validated_evidence_records(run_root, index) + retained = _retained_evidence_records(records, referenced) + index_bytes = _rewrite_compacted_index(run_root, summary_path, retained) + _rewrite_compacted_run(run_root, task, referenced, index_bytes) + + +def _validated_attempt_from_root( + run_root: Path, + protocol: Mapping[str, object], + slots: Mapping[str, Mapping[str, object]], +) -> dict[str, object]: + """Load and validate one attempt joined to a declared logical slot.""" + + attempt = load_strict_json(run_root / _ATTEMPT_FILE) + slot_id = attempt.get("slot_id") + if not isinstance(slot_id, str) or slot_id not in slots: + raise ValueError(_INVALID_ATTEMPT_SLOT_JOIN) + _validate_attempt(attempt, protocol, slots[slot_id]) + return attempt + + +def _condition_run_roots(condition_root: Path) -> list[Path]: + """Return the ordered direct run directories for one condition.""" + + return sorted(path for path in condition_root.iterdir() if path.is_dir()) + + +def _verify_condition_inventory(condition_root: Path, run_roots: Sequence[Path]) -> None: + """Verify one condition inventory against its direct run inventories.""" + + _verify_inventory( + condition_root, + expected_members={_CLEANUP_FILE} + | {f"{run_root.name}/{_INVENTORY_FILE}" for run_root in run_roots}, + ) + + +def _refresh_bundle_projections( + bundle: Path, + protocol: Mapping[str, object], + attempts: Sequence[Mapping[str, object]], + root_members: Sequence[Path], +) -> None: + """Recompute and reseal deterministic projections after compaction.""" + + aggregates = compute_aggregates(protocol, attempts) + _write_json(bundle / _AGGREGATES_FILE, aggregates) + reference = load_strict_json(bundle / _REFERENCE_FILE) + _validate_reference(reference) + tiers = build_tiers(protocol, aggregates, reference) + _write_json(bundle / _TIERS_FILE, tiers) + _write_validation_report(bundle, tiers) + _seal_inventory(bundle, root_members) + + def compact_bundle_evidence( root: Path, *, @@ -1949,7 +2585,7 @@ def compact_bundle_evidence( """ bundle = root.resolve() - protocol = load_strict_json(bundle / "protocol.json") + protocol = load_strict_json(bundle / _PROTOCOL_FILE) selection = _selection_from_declaration(cast(dict[str, object], protocol["declaration"])) validate_protocol(protocol, selection=selection) _verify_inventory(bundle) @@ -1958,130 +2594,39 @@ def compact_bundle_evidence( ) task = ExperimentTaskModel.model_validate(contracts["experiment_task"]) slots = _slot_index(protocol) - root_inventory = load_strict_json(bundle / "inventory.json") + root_inventory = load_strict_json(bundle / _INVENTORY_FILE) inventory_artifacts = cast(list[dict[str, object]], root_inventory["artifacts"]) root_members = [bundle / cast(str, item["path"]) for item in inventory_artifacts] condition_roots = sorted(path for path in (bundle / "runs").iterdir() if path.is_dir()) run_roots = [ run_root for condition_root in condition_roots - for run_root in sorted(path for path in condition_root.iterdir() if path.is_dir()) + for run_root in _condition_run_roots(condition_root) ] total = len(run_roots) completed = 0 all_attempts: list[dict[str, object]] = [] for condition_root in condition_roots: - condition_run_roots = sorted(path for path in condition_root.iterdir() if path.is_dir()) - _verify_inventory( - condition_root, - expected_members={"cleanup.json"} - | {f"{run_root.name}/inventory.json" for run_root in condition_run_roots}, - ) + condition_run_roots = _condition_run_roots(condition_root) + _verify_condition_inventory(condition_root, condition_run_roots) for run_root in condition_run_roots: _verify_inventory(run_root) - attempt = load_strict_json(run_root / "attempt.json.gz") - slot_id = attempt.get("slot_id") - if not isinstance(slot_id, str) or slot_id not in slots: - raise ValueError("attempt slot join is invalid") - _validate_attempt(attempt, protocol, slots[slot_id]) + attempt = _validated_attempt_from_root(run_root, protocol, slots) all_attempts.append(attempt) if attempt["disposition"] == "valid": - index = load_strict_json(run_root / "evidence-index.json") - summary_ref = cast(dict[str, object], index["episode_summary"]) - summary_path = run_root / cast(str, summary_ref["path"]) - if _sha256_file(summary_path) != summary_ref["sha256"]: - raise ValueError("episode summary digest is invalid") - summary = load_strict_json(summary_path) - measures = [ - ExperimentDerivedMeasureModel.model_validate(item) - for item in cast(list[dict[str, object]], summary["derived_measures"]) - ] - referenced = { - ref.ref_id for measure in measures for ref in measure.source_evidence_refs - } - records: list[ExperimentEvidenceRecordModel] = [] - for chunk in cast(list[dict[str, object]], index["chunks"]): - chunk_path = run_root / cast(str, chunk["path"]) - if _sha256_file(chunk_path) != chunk["sha256"]: - raise ValueError("evidence chunk digest is invalid") - values = _load_strict_json_value(chunk_path) - if not isinstance(values, list) or len(values) != chunk["record_count"]: - raise ValueError("evidence chunk is invalid") - chunk_records = [ - ExperimentEvidenceRecordModel.model_validate(item) for item in values - ] - steps = [_evidence_step(record) for record in chunk_records] - if ( - not steps - or steps != sorted(steps) - or min(steps) != chunk["first_logical_step"] - or max(steps) != chunk["last_logical_step"] - ): - raise ValueError("evidence chunk ordering is invalid") - records.extend(chunk_records) - if len(records) != index["record_count"]: - raise ValueError("evidence record closure is invalid") - retained = [record for record in records if record.evidence_record_id in referenced] - if {record.evidence_record_id for record in retained} != referenced: - raise ValueError("derived-measure evidence closure is invalid") - for path in run_root.glob("evidence-*.json.gz"): - path.unlink() - chunks = _write_evidence_chunks(run_root, retained) - _write_json( - run_root / "evidence-index.json", - { - "record_count": len(retained), - "chunks": chunks, - "episode_summary": { - "path": summary_path.relative_to(run_root).as_posix(), - "sha256": _sha256_file(summary_path), - }, - }, - ) - index_bytes = (run_root / "evidence-index.json").read_bytes() - run_payload = load_strict_json(run_root / "run.json.gz") - artifacts = cast(list[dict[str, object]], run_payload["evidence_artifacts"]) - if len(artifacts) != 1: - raise ValueError("run evidence artifacts are invalid") - artifacts[0]["checksum"] = { - "algorithm": "sha256", - "value": hashlib.sha256(index_bytes).hexdigest(), - } - artifacts[0]["size_bytes"] = len(index_bytes) - traceability = cast(dict[str, object], run_payload["traceability"]) - traceability["evidence_record_refs"] = [ - item - for item in cast(list[dict[str, object]], traceability["evidence_record_refs"]) - if item["ref_id"] in referenced - ] - run = ExperimentRunModel.model_validate(run_payload) - validate_experiment_run_against_task(task, run) - _write_gzip_json(run_root / "run.json.gz", run.model_dump(mode="json")) - _seal_inventory( - run_root, - [ - path - for path in run_root.iterdir() - if path.is_file() and path.name != "inventory.json" - ], - ) + _compact_valid_run_evidence(run_root, task) completed += 1 if progress is not None: progress(completed, total) _condition_inventory(condition_root) - aggregates = compute_aggregates(protocol, all_attempts) - _write_json(bundle / "aggregates.json", aggregates) - reference = load_strict_json(bundle / "reference.json") - _validate_reference(reference) - tiers = build_tiers(protocol, aggregates, reference) - _write_json(bundle / "tiers.json", tiers) - _write_validation_report(bundle, tiers) - _seal_inventory(bundle, root_members) + _refresh_bundle_projections(bundle, protocol, all_attempts, root_members) verify_bundle(bundle, selection=selection) return bundle def _verify_schedule_partitions(root: Path, protocol: Mapping[str, object]) -> None: + """Verify every condition plan against the frozen declaration.""" + declaration = cast(dict[str, object], protocol["declaration"]) expected = _slot_index(protocol) observed: list[dict[str, object]] = [] @@ -2107,84 +2652,12 @@ def _verify_schedule_partitions(root: Path, protocol: Mapping[str, object]) -> N raise ValueError("schedule partitions do not join the protocol") -def _valid_run_evidence( - bundle: Path, - run_root: Path, - attempt: Mapping[str, object], - task: ExperimentTaskModel, -) -> None: - """Validate the sealed portable evidence closure for one eligible attempt.""" +def _reported_score_measure( + measures: Sequence[ExperimentDerivedMeasureModel], +) -> ExperimentDerivedMeasureModel | None: + """Return the retained reported score measure when present.""" - index_path = run_root / "evidence-index.json" - index = load_strict_json(index_path) - _require_exact_keys(index, {"record_count", "chunks", "episode_summary"}, "evidence index") - chunks = index["chunks"] - if not isinstance(chunks, list) or not chunks: - raise ValueError("evidence index is invalid") - record_ids: list[str] = [] - for chunk in chunks: - if not isinstance(chunk, dict): - raise ValueError("evidence index is invalid") - _require_exact_keys( - chunk, - { - "path", - "sha256", - "record_count", - "first_logical_step", - "last_logical_step", - }, - "evidence chunk", - ) - relative = chunk["path"] - if not isinstance(relative, str) or Path(relative).name != relative: - raise ValueError("evidence chunk path is invalid") - path = run_root / relative - if _sha256_file(path) != chunk["sha256"]: - raise ValueError("evidence chunk digest is invalid") - values = _load_strict_json_value(path) - if not isinstance(values, list) or len(values) != chunk["record_count"]: - raise ValueError("evidence chunk is invalid") - records = [ExperimentEvidenceRecordModel.model_validate(item) for item in values] - steps = [_evidence_step(record) for record in records] - if ( - not steps - or steps != sorted(steps) - or min(steps) != chunk["first_logical_step"] - or max(steps) != chunk["last_logical_step"] - ): - raise ValueError("evidence chunk ordering is invalid") - record_ids.extend(record.evidence_record_id for record in records) - if len(record_ids) != index["record_count"] or len(record_ids) != len(set(record_ids)): - raise ValueError("evidence record closure is invalid") - - summary_ref = index["episode_summary"] - if not isinstance(summary_ref, dict): - raise ValueError("episode summary reference is invalid") - _require_exact_keys(summary_ref, {"path", "sha256"}, "episode summary reference") - summary_relative = summary_ref["path"] - if not isinstance(summary_relative, str) or Path(summary_relative).name != summary_relative: - raise ValueError("episode summary path is invalid") - summary_path = run_root / summary_relative - if _sha256_file(summary_path) != summary_ref["sha256"]: - raise ValueError("episode summary digest is invalid") - summary = load_strict_json(summary_path) - _require_exact_keys( - summary, - {"derived_measures", "proposition_truth", "objective_results", "diagnostics"}, - "episode summary", - ) - measure_values = summary["derived_measures"] - if not isinstance(measure_values, list) or not measure_values: - raise ValueError("derived measures are invalid") - measures = [ExperimentDerivedMeasureModel.model_validate(item) for item in measure_values] - known_records = set(record_ids) - referenced_records = { - ref.ref_id for measure in measures for ref in measure.source_evidence_refs - } - if referenced_records != known_records: - raise ValueError("derived-measure evidence closure is invalid") - score = next( + return next( ( item for item in measures @@ -2194,40 +2667,150 @@ def _valid_run_evidence( ), None, ) - if score is None or canonical_json_bytes(score.model_dump(mode="json")) != canonical_json_bytes( - attempt["score_measure"] - ): + + +def _validate_attempt_episode_score( + attempt: Mapping[str, object], measures: Sequence[ExperimentDerivedMeasureModel] +) -> None: + """Require an attempt score to equal its retained episode score measure.""" + + score = _reported_score_measure(measures) + if score is None: + raise ValueError("attempt score does not join the episode evidence") + observed = canonical_json_bytes(score.model_dump(mode="json")) + if observed != canonical_json_bytes(attempt["score_measure"]): raise ValueError("attempt score does not join the episode evidence") - run_path = run_root / "run.json.gz" - run_payload = load_strict_json(run_path) - run = ExperimentRunModel.model_validate(run_payload) - validate_experiment_run_against_task(task, run) + +def _validate_run_traceability( + run_payload: Mapping[str, object], + known_records: set[str], + measures: Sequence[ExperimentDerivedMeasureModel], +) -> None: + """Validate the run's exact evidence-record and measure reference closure.""" + traceability = cast(dict[str, object], run_payload["traceability"]) - trace_record_refs = cast(list[dict[str, object]], traceability["evidence_record_refs"]) - trace_measure_refs = cast(list[dict[str, object]], traceability["derived_measure_refs"]) - if {cast(str, item["ref_id"]) for item in trace_record_refs} != known_records: + record_refs = cast(list[dict[str, object]], traceability["evidence_record_refs"]) + measure_refs = cast(list[dict[str, object]], traceability["derived_measure_refs"]) + if {cast(str, item["ref_id"]) for item in record_refs} != known_records: raise ValueError("run evidence traceability is incomplete") - if {cast(str, item["ref_id"]) for item in trace_measure_refs} != { - item.derived_measure_id for item in measures - }: + expected_measures = {item.derived_measure_id for item in measures} + if {cast(str, item["ref_id"]) for item in measure_refs} != expected_measures: raise ValueError("run measure traceability is incomplete") - artifact_relative = index_path.relative_to(bundle).as_posix() + + +def _validate_run_artifact( + bundle: Path, + run_payload: Mapping[str, object], + index_path: Path, +) -> None: + """Validate the run's sole portable evidence artifact reference.""" + + relative = index_path.relative_to(bundle).as_posix() artifacts = cast(list[dict[str, object]], run_payload["evidence_artifacts"]) - if len(artifacts) != 1 or artifacts[0].get("uri") != artifact_relative: + if len(artifacts) != 1 or artifacts[0].get("uri") != relative: raise ValueError("run evidence artifact path is invalid") checksum = artifacts[0].get("checksum") if not isinstance(checksum, dict) or checksum.get("value") != _sha256_file(index_path): raise ValueError("run evidence artifact digest is invalid") - result_summaries = cast(dict[str, dict[str, object]], run_payload["result_summaries"]) - result = result_summaries.get("cage2-cumulative-blue-reward-result") - if result is None or float(cast(float, result.get("value"))) != _validate_score_measure( - attempt["score_measure"] - ): + + +def _validate_run_score_summary( + run_payload: Mapping[str, object], attempt: Mapping[str, object] +) -> None: + """Validate the archival run's primary score summary.""" + + summaries = cast(dict[str, dict[str, object]], run_payload["result_summaries"]) + result = summaries.get("cage2-cumulative-blue-reward-result") + if result is None: raise ValueError("run score summary is invalid") + expected = _validate_score_measure(attempt["score_measure"]) + if float(cast(float, result.get("value"))) != expected: + raise ValueError("run score summary is invalid") + + +def _validate_run_evidence_contract( + bundle: Path, + run_root: Path, + attempt: Mapping[str, object], + task: ExperimentTaskModel, + known_records: set[str], + measures: Sequence[ExperimentDerivedMeasureModel], +) -> None: + """Validate the RAES run and every retained evidence join.""" + + run_payload = load_strict_json(run_root / _RUN_FILE) + run = ExperimentRunModel.model_validate(run_payload) + validate_experiment_run_against_task(task, run) + _validate_run_traceability(run_payload, known_records, measures) + _validate_run_artifact(bundle, run_payload, run_root / _EVIDENCE_INDEX_FILE) + _validate_run_score_summary(run_payload, attempt) + + +def _valid_run_evidence( + bundle: Path, + run_root: Path, + attempt: Mapping[str, object], + task: ExperimentTaskModel, +) -> None: + """Validate the sealed portable evidence closure for one eligible attempt.""" + + index = _validated_evidence_index(run_root) + records = _validated_evidence_records(run_root, index) + _, _, measures = _validated_episode_summary(run_root, index) + known_records = {record.evidence_record_id for record in records} + if _referenced_record_ids(measures) != known_records: + raise ValueError("derived-measure evidence closure is invalid") + _validate_attempt_episode_score(attempt, measures) + _validate_run_evidence_contract(bundle, run_root, attempt, task, known_records, measures) + + +def _validated_condition_roots(runs_root: Path, expected: set[str]) -> list[Path]: + """Return direct condition directories after exact membership validation.""" + + entries = list(runs_root.iterdir()) + roots = sorted(path for path in entries if path.is_dir()) + invalid_membership = ( + any(path.is_symlink() or not path.is_dir() for path in entries) + or {path.name for path in roots} != expected + ) + if invalid_membership: + raise ValueError("run condition membership is invalid") + return roots + + +def _validate_run_root_membership(run_root: Path) -> None: + """Reject links and nested directories inside one sealed run.""" + + if run_root.is_symlink() or any(path.is_dir() for path in run_root.iterdir()): + raise ValueError("run artifact membership is invalid") + + +def _load_condition_attempts( + bundle: Path, + condition_root: Path, + protocol: Mapping[str, object], + slots: Mapping[str, Mapping[str, object]], + task: ExperimentTaskModel, +) -> list[dict[str, object]]: + """Load every sealed attempt under one condition.""" + + attempts: list[dict[str, object]] = [] + run_roots = _condition_run_roots(condition_root) + _verify_condition_inventory(condition_root, run_roots) + for run_root in run_roots: + _validate_run_root_membership(run_root) + _verify_inventory(run_root) + attempt = _validated_attempt_from_root(run_root, protocol, slots) + if attempt["disposition"] == "valid": + _valid_run_evidence(bundle, run_root, attempt, task) + attempts.append(attempt) + return attempts def _load_attempts(root: Path, protocol: Mapping[str, object]) -> list[dict[str, object]]: + """Load and validate every sealed operational attempt.""" + attempts: list[dict[str, object]] = [] slots = _slot_index(protocol) declaration = cast(dict[str, object], protocol["declaration"]) @@ -2235,31 +2818,12 @@ def _load_attempts(root: Path, protocol: Mapping[str, object]) -> list[dict[str, task = ExperimentTaskModel.model_validate(contracts["experiment_task"]) runs_root = root / "runs" expected_conditions = {cast(str, slot["condition_id"]) for slot in slots.values()} - condition_roots = sorted(path for path in runs_root.iterdir() if path.is_dir()) - if ( - any(path.is_symlink() or not path.is_dir() for path in runs_root.iterdir()) - or {path.name for path in condition_roots} != expected_conditions - ): - raise ValueError("run condition membership is invalid") - for condition_root in condition_roots: - run_roots = sorted(path for path in condition_root.iterdir() if path.is_dir()) - _verify_inventory( - condition_root, - expected_members={"cleanup.json"} - | {f"{run_root.name}/inventory.json" for run_root in run_roots}, - ) - for run_root in run_roots: - if run_root.is_symlink() or any(path.is_dir() for path in run_root.iterdir()): - raise ValueError("run artifact membership is invalid") - _verify_inventory(run_root) - attempt = load_strict_json(run_root / "attempt.json.gz") - slot_id = attempt.get("slot_id") - if not isinstance(slot_id, str) or slot_id not in slots: - raise ValueError("attempt slot join is invalid") - _validate_attempt(attempt, protocol, slots[slot_id]) - if attempt["disposition"] == "valid": - _valid_run_evidence(root, run_root, attempt, task) - attempts.append(attempt) + condition_roots = _validated_condition_roots(runs_root, expected_conditions) + attempts.extend( + attempt + for condition_root in condition_roots + for attempt in _load_condition_attempts(root, condition_root, protocol, slots, task) + ) return attempts @@ -2277,20 +2841,20 @@ def verify_bundle( or {path.name for path in bundle.iterdir() if path.is_dir()} != expected_root_directories ): raise ValueError("bundle directory membership is invalid") - protocol = load_strict_json(bundle / "protocol.json") + protocol = load_strict_json(bundle / _PROTOCOL_FILE) validate_protocol(protocol, selection=selection) declaration = cast(dict[str, object], protocol["declaration"]) conditions = {cast(str, slot["condition_id"]) for slot in _slot_index(protocol).values()} _verify_inventory( bundle, expected_members={ - "protocol.json", - "source-ledger.json", - "reference.json", - "environment.json", - "aggregates.json", - "tiers.json", - "validation-report.md", + _PROTOCOL_FILE, + _SOURCE_LEDGER_FILE, + _REFERENCE_FILE, + _ENVIRONMENT_FILE, + _AGGREGATES_FILE, + _TIERS_FILE, + _VALIDATION_REPORT_FILE, *{ cast(str, partition["path"]) for partition in cast(list[dict[str, object]], declaration["schedule_partitions"]) @@ -2298,23 +2862,23 @@ def verify_bundle( *{f"runs/{condition}/inventory.json" for condition in conditions}, }, ) - ledger = load_strict_json(bundle / "source-ledger.json") + ledger = load_strict_json(bundle / _SOURCE_LEDGER_FILE) _validate_source_ledger(ledger, protocol) - _validate_environment(load_strict_json(bundle / "environment.json"), protocol) + _validate_environment(load_strict_json(bundle / _ENVIRONMENT_FILE), protocol) _verify_schedule_partitions(bundle, protocol) attempts = _load_attempts(bundle, protocol) aggregates = compute_aggregates(protocol, attempts) if canonical_json_bytes(aggregates) != canonical_json_bytes( - load_strict_json(bundle / "aggregates.json") + load_strict_json(bundle / _AGGREGATES_FILE) ): raise ValueError("aggregate recomputation does not match") - reference = load_strict_json(bundle / "reference.json") + reference = load_strict_json(bundle / _REFERENCE_FILE) _validate_reference(reference) tiers = build_tiers(protocol, aggregates, reference) - if canonical_json_bytes(tiers) != canonical_json_bytes(load_strict_json(bundle / "tiers.json")): + if canonical_json_bytes(tiers) != canonical_json_bytes(load_strict_json(bundle / _TIERS_FILE)): raise ValueError("tier recomputation does not match") try: - report = (bundle / "validation-report.md").read_text(encoding="utf-8") + report = (bundle / _VALIDATION_REPORT_FILE).read_text(encoding="utf-8") except (OSError, UnicodeError) as error: raise ValueError("validation report is invalid") from error if report != _validation_report_text(tiers): diff --git a/src/raes_adapters/cyborg/researcher.py b/src/raes_adapters/cyborg/researcher.py index 2a0c31e..78dd475 100644 --- a/src/raes_adapters/cyborg/researcher.py +++ b/src/raes_adapters/cyborg/researcher.py @@ -483,6 +483,22 @@ def _begin_independent_run( return _initialize_participants(target, baseline, controls, diagnostics) +def _validated_series_first(controls: Sequence[RunControls]) -> RunControls: + """Return the shared condition controls after consistency validation.""" + + if not controls: + raise ValueError("researcher condition series is empty") + first = controls[0] + if any( + item.seed != first.seed + or item.max_steps != first.max_steps + or item.red_variant != first.red_variant + for item in controls + ): + raise ValueError("researcher condition series controls are inconsistent") + return first + + def execute_episode_series( scenario: object, controls: Sequence[RunControls], @@ -498,16 +514,7 @@ def execute_episode_series( resets continue, rather than restart, that stream. """ - if not controls: - raise ValueError("researcher condition series is empty") - first = controls[0] - if any( - item.seed != first.seed - or item.max_steps != first.max_steps - or item.red_variant != first.red_variant - for item in controls - ): - raise ValueError("researcher condition series controls are inconsistent") + first = _validated_series_first(controls) target = create_cyborg_target( scenario=scenario, seed=None, From 03d1a7626832503eef0de3ee85a7fd9a4fab1be2 Mon Sep 17 00:00:00 2001 From: Brad Edwards Date: Thu, 13 Aug 2026 00:23:45 +0200 Subject: [PATCH 3/3] refactor(cyborg): centralize attempt validation message --- src/raes_adapters/cyborg/reproduction.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/raes_adapters/cyborg/reproduction.py b/src/raes_adapters/cyborg/reproduction.py index 1020838..773ad8c 100644 --- a/src/raes_adapters/cyborg/reproduction.py +++ b/src/raes_adapters/cyborg/reproduction.py @@ -52,6 +52,7 @@ _INVALID_JSON = "invalid JSON artifact" _INVALID_ATTEMPT_SLOT_JOIN = "attempt slot join is invalid" +_INVALID_ATTEMPT_SEQUENCE = "attempt sequence is invalid" _INVALID_FORBIDDEN_MATERIAL = "forbidden native material" _INVALID_INVENTORY = "inventory is invalid" _INVALID_PROTOCOL_SCHEDULE = "protocol schedule is invalid" @@ -851,7 +852,7 @@ def _attempt_retry_limit(protocol: Mapping[str, object]) -> int: raise ValueError("attempt protocol join is invalid") retry_limit = cast(dict[str, object], declaration["attempt_policy"])["retry_limit"] if type(retry_limit) is not int: - raise ValueError("attempt sequence is invalid") + raise ValueError(_INVALID_ATTEMPT_SEQUENCE) return retry_limit @@ -862,7 +863,7 @@ def _validate_attempt_identity( sequence = value["attempt_sequence"] if type(sequence) is not int or not 1 <= sequence <= retry_limit + 1: - raise ValueError("attempt sequence is invalid") + raise ValueError(_INVALID_ATTEMPT_SEQUENCE) expected_run_id = f"{slot['slot_id']}-attempt-{sequence:02d}" predecessor = None if sequence > 1: @@ -979,7 +980,7 @@ def select_eligible_attempts( for slot_id in slots: ordered = sorted(by_slot[slot_id], key=lambda item: cast(int, item["attempt_sequence"])) if [item["attempt_sequence"] for item in ordered] != list(range(1, len(ordered) + 1)): - raise ValueError("attempt sequence is invalid") + raise ValueError(_INVALID_ATTEMPT_SEQUENCE) valid = next((item for item in ordered if item["disposition"] == "valid"), None) if valid is not None: selected.append(valid)