diff --git a/.env.example b/.env.example index 11e2f1a..b909212 100644 --- a/.env.example +++ b/.env.example @@ -3,11 +3,12 @@ AZURE_API_ENDPOINT=https://your-resource.openai.azure.com/ AZURE_API_KEY= AZURE_API_VERSION=preview -# Claude subscription route used by OpenHands ACPAgent +# Claude subscription route used by OpenCode's pinned OAuth plugin CLAUDE_CODE_OAUTH_TOKEN= -# Optional Docker/runtime overrides -GENESISBENCH_DOCKER_IMAGE=genesisbench-simulation-heuristics-ant-v1-runner:latest -GENESISBENCH_DOCKER_PLATFORM= +# Optional Daytona sandbox route +DAYTONA_API_KEY= +DAYTONA_TARGET=us + +# Runtime overrides GENESISBENCH_ENV_FILE=.env -GENESISBENCH_OPENHANDS_PYTHON= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d250e5..1f2633c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,15 +41,20 @@ jobs: tasks/simulation_heuristics_ant_v1 --level publication-grade - - name: Run native BenchFlow oracle - run: >- - uv run bench eval run - --tasks-dir tasks/simulation_heuristics_ant_v1 - --agent oracle - --sandbox docker - --context-root . - --jobs-dir /tmp/genesisbench-oracle - --concurrency 1 + - name: Run native BenchFlow oracle smoke + shell: bash + run: | + full_config="$(mktemp)" + cp tasks/simulation_heuristics_ant_v1/verifier/config.toml "$full_config" + trap 'cp "$full_config" tasks/simulation_heuristics_ant_v1/verifier/config.toml; rm -f "$full_config"' EXIT + cp tasks/simulation_heuristics_ant_v1/verifier/config_smoke.toml tasks/simulation_heuristics_ant_v1/verifier/config.toml + uv run bench eval run \ + --tasks-dir tasks/simulation_heuristics_ant_v1 \ + --agent oracle \ + --sandbox docker \ + --context-root . \ + --jobs-dir /tmp/genesisbench-oracle \ + --concurrency 1 - name: Audit packaged leaderboard run: uv run python scripts/audit_simulation_heuristics_ant_v1_leaderboard.py diff --git a/.gitignore b/.gitignore index 351cf09..2a4e4f1 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ *.pyc *.egg-info/ +_vizdoom.ini build/ dist/ diff --git a/README.md b/README.md index 832b131..4aed901 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,10 @@ instruction-tuned language model. ## Reference Task: Simulation Heuristics Ant v1 -`tasks/simulation_heuristics_ant_v1/` is the first executable task and the canonical example for -future contributors. An agent receives a weak rhythmic CPG/PD controller for -Gymnasium `Ant-v5`, repeatedly edits and evaluates it, and submits -`final_policy/policy.py`. +`tasks/simulation_heuristics_ant_v1/` is the first executable task and the +canonical package example. The complete article-derived suite contains nine +tasks spanning MuJoCo locomotion, Atari RAM and vision control, VizDoom, +long-horizon recovery, and the aggregate Atari57 workflow. The package follows BenchFlow `0.6.5`'s native `task.md` format (`schema_version: "1.3"`, document version `"0.6"`). @@ -76,15 +76,16 @@ uv run python scripts/prepare_task.py \ --force ``` -The prepared OpenHands workspace deliberately excludes `verifier/`, `oracle/`, +The prepared OpenCode workspace deliberately excludes `verifier/`, `oracle/`, and `evidence/`. -## OpenHands Experiment +## OpenCode Article-Suite Experiment -Build the isolated runner: +OpenCode is the default and only leaderboard harness for the nine-task suite. +Install the Daytona dependency when using the hosted sandbox: ```bash -sh scripts/build_simulation_heuristics_ant_v1_runner_image.sh +uv sync --extra dev --extra sandbox-daytona ``` Configure credentials: @@ -93,24 +94,51 @@ Configure credentials: cp .env.example .env ``` -Run one agent: +Run one model across all nine tasks: ```bash -uv run python scripts/run_simulation_heuristics_ant_v1_experiment.py \ - --model gpt-5.6-sol \ - --minutes 30 +uv run python scripts/run_article_suite.py \ + --model gpt-5.6-sol ``` -See `experiments/simulation_heuristics_ant_v1/README.md` for model routes, fairness controls, artifact -layout, and leaderboard regeneration. +Run all four canonical models and rebuild the aggregate leaderboard: -## Current Leaderboard +```bash +uv run python scripts/run_article_suite.py \ + --all-models +uv run python scripts/build_article_suite_leaderboard.py +``` + +See `experiments/article_suite/README.md` for the exact model routes, task +manifest, isolation controls, and scoring contract. The task-by-task research +mapping is documented in `docs/learning-beyond-gradients-suite.md`. + +## Article-Suite Leaderboard + +The first OpenCode sweep across all nine article-derived tasks: + +| Rank | Model | Nine-task average | +| ---: | --- | ---: | +| 1 | GPT-5.5 | 43.19 | +| 2 | Claude Opus 4.8 | 39.82 | +| 3 | GPT-5.6 Sol | 39.38 | +| 4 | GPT-5.4 Mini | -29.72 | + +See [`leaderboard/ARTICLE_SUITE.md`](leaderboard/ARTICLE_SUITE.md) for every +per-task score and `leaderboard/article_suite.json` for the machine-readable +leaderboard. + +Scores are unbounded normalized values: `0` matches the public starter and +`100` matches the trusted article-level reference. Negative scores are genuine +regressions; scores above `100` exceed the reference. + +## Legacy Ant-Only Leaderboard ![GenesisBench Simulation Heuristics Ant v1 leaderboard](leaderboard/simulation_heuristics_ant_v1_leaderboard.png) -The first four-model OpenHands sweep used equal 30-minute budgets and each -model's highest supported reasoning setting. Machine-readable results and -packaged policies are in `leaderboard/`. +The table below is the historical Ant-only OpenHands sweep. It remains for +provenance; new GenesisBench leaderboard runs use OpenCode and the nine-task +article suite. | Rank | Model | Hidden-suite score | | ---: | --- | ---: | diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 1b08408..aea2097 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -24,3 +24,26 @@ The Apache License 2.0 text is included in `LICENSES/Apache-2.0.txt`. GenesisBench as a combined project is distributed under GPL-3.0. Apache-2.0 is compatible with GPLv3; upstream notices remain applicable to the derived files. + +## Learning Beyond Gradients Atari controllers + +The Pong RAM and Breakout RAM/RGB task anchors retain or adapt controller +structure and parameterization from the corresponding public article +artifacts: + +- `tasks/simulation_heuristics_pong_ram_v1/{starter_policy,oracle}/policy.py` +- `tasks/simulation_heuristics_pong_ram_v1/verifier/anchor_policies/*.py` +- `tasks/simulation_heuristics_breakout_ram_v1/{starter_policy,oracle}/policy.py` +- `tasks/simulation_heuristics_breakout_ram_v1/verifier/anchor_policies/*.py` +- `tasks/simulation_heuristics_breakout_rgb_v1/{starter_policy,oracle}/policy.py` +- `tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchor_policies/*.py` + +Upstream: + +- Project: `Trinkle23897/learning-beyond-gradients` +- Sources: `atari/pong/heuristic_pong.py` and + `atari/breakout/heuristic_breakout.py` +- Reviewed revision: `3555c2956c257d49a5015b782cbe485b14fd659e` +- Copyright notice retained in derived files: + `Copyright 2021 Garena Online Private Limited` +- License: Apache License 2.0 diff --git a/docs/learning-beyond-gradients-suite.md b/docs/learning-beyond-gradients-suite.md new file mode 100644 index 0000000..c0e818f --- /dev/null +++ b/docs/learning-beyond-gradients-suite.md @@ -0,0 +1,77 @@ +# Learning Beyond Gradients task suite + +GenesisBench represents each experiment-family row in the article as one +native task package. Together with the original Ant task, the suite contains +nine tasks. + +| Task | Article contract | Trusted reproduction | +| --- | --- | --- | +| `simulation_heuristics_ant_v1` | CPG/PD through adaptive residual MPC | Gymnasium mean 5895.932216 with exact source/action parity | +| `simulation_heuristics_pong_ram_v1` | RAM-only Pong, score 21 | 21 on hidden seeds and reset variants | +| `simulation_heuristics_breakout_ram_v1` | `387 → 507 → 839 → 864` | 864 on nominal and shifted starts | +| `simulation_heuristics_breakout_rgb_v1` | `310 → 428 → 864` pixel transfer | 864 on nominal and shifted starts | +| `simulation_heuristics_halfcheetah_v1` | staged-tree MPC, seeds 100–104 | mean 11836.693449819431 | +| `simulation_heuristics_vizdoom_d1_v1` | screen CV + `HEALTH` | mean 0.9440999741666019 | +| `simulation_heuristics_vizdoom_d3_v1` | screen CV + five public variables | returns `[545,475,480,440,690,500,600,595,530,715]` | +| `simulation_heuristics_atari57_v1` | 57 games × 2 modes × 3 searches | complete 342-slot and 6.84B-frame protocol | +| `simulation_heuristics_montezuma_v1` | 400-point macro route plus recovery | 400 in 1,769 steps with recovery variants | + +## Common benchmark boundary + +Each task follows the original Ant package: + +```text +fixed public starter +→ OpenCode research in an isolated sandbox +→ public development evaluator +→ standardized final artifact +→ hidden verifier +→ starter/reference normalized score +``` + +The task workspace never contains `verifier/`, `oracle/`, or `evidence/`. + +## Atari57 exception + +Atari57 is one aggregate task rather than 114 standalone tasks. Its artifact +resolves one policy for every `(game, observation mode, repeat index)` tuple: + +```text +57 × 2 × 3 = 342 policy/search slots +``` + +An official result is eligible only when the interaction ledger records all +342 searches at 20,000,000 frames each. Incomplete submissions return a clean +zero and are not presented as article reproductions. + +## Model and harness contract + +All new leaderboard runs use the BenchFlow `opencode` ACP harness. The canonical +matrix is: + +| Model | Provider | Effort | +| --- | --- | --- | +| GPT-5.6 Sol | Azure | `max` | +| GPT-5.5 | Azure | `xhigh` | +| Claude Opus 4.8 | Claude OAuth through pinned OpenCode plugin | `max` | +| GPT-5.4 Mini | Azure | `xhigh` | + +OpenCode talks directly to the provider because BenchFlow 0.6.5's +chat-completions gateway cannot faithfully transform Azure GPT-5.6 Sol tool +calls. BenchFlow continues to own task staging, Daytona/Docker isolation, ACP +trajectory capture, timing, and verifier execution. + +## Aggregate score + +The article-suite leaderboard reports every normalized task score and their +unweighted arithmetic mean: + +```text +average = sum(nine normalized task scores) / 9 +``` + +The runner and resumable leaderboard builder live in: + +- `scripts/run_article_suite.py` +- `scripts/build_article_suite_leaderboard.py` +- `experiments/article_suite/` diff --git a/docs/simulation-heuristics-ant-v1-task-design.md b/docs/simulation-heuristics-ant-v1-task-design.md index 13f8014..9a4cb80 100644 --- a/docs/simulation-heuristics-ant-v1-task-design.md +++ b/docs/simulation-heuristics-ant-v1-task-design.md @@ -30,9 +30,34 @@ That experiment evolved an interpretable controller through: - warm-started planning and adaptive gait timing. The public artifacts report a five-episode mean near `6005` in EnvPool's -`Ant-v5`. GenesisBench does not ask agents to reproduce that exact public -solution on the exact public setup because copying a completed controller would -not measure autonomous research. +`Ant-v5`: mean `6005.521`, minimum `5776.805`, and maximum `6146.208`. + +GenesisBench keeps the public weak CPG/PD starter so agents still face an +authentic policy-improvement problem. The trusted oracle and normalization +anchor now reproduce the article's final controller instead of using an +unrelated stronger rhythmic policy. + +That reference combines: + +- speed-adaptive phase increment and stance duty; +- higher-harmonic CPG joint targets; +- posture and heading feedback; +- a 10-step copied-MuJoCo planning horizon; +- 96 residual-action candidates per external step; +- temporal residual smoothing and warm-start plan decay; +- forward, control, posture, yaw, height, health, and terminal-velocity + objective terms. + +The source used EnvPool `1.1.1`; GenesisBench uses Gymnasium. The supplied +`ant_envpool.xml` and Gymnasium's Ant XML parse to the same MuJoCo model, but +the two runtimes use different reset random-number streams. Machine-readable +source hashes and both result families live in +`tasks/simulation_heuristics_ant_v1/evidence/source_provenance.json`. + +On the development host, both the imported source policy and the GenesisBench +adaptation produced the same Gymnasium seed-`0..4` returns: mean +`5895.932216`, minimum `5791.444245`, and maximum `6131.400491`. A separate +50-step action-parity probe had maximum absolute difference `0.0`. ## Benchmark translation @@ -81,6 +106,17 @@ It must expose `Policy` or `make_policy` and produce finite eight-dimensional actions from observations. The final evaluator imports this artifact in a clean process after the agent exits. +Policies may optionally expose: + +```python +configure_simulator(model_xml_path=..., frame_skip=...) +``` + +The evaluator supplies a copied model matching the current episode. It never +passes the live `Env`, mutable scored `MjData`, reward, `info`, or hidden suite +configuration. This preserves the article's model-based planning capability +without letting a policy step or mutate the scored simulator. + ## Hidden evaluation The reproducibility suite evaluates: @@ -116,6 +152,11 @@ policies on the same platform as the submission. This keeps `0` and `100` stable across small MuJoCo platform differences. A hosted private suite may instead inject fixed scores through its private anchors file. +The reference and oracle policy files are byte-identical. The verifier caches +evaluations by `policy.py` SHA-256 fingerprint so an oracle run evaluates the +expensive MPC controller once rather than once as the submission and again as +the reference anchor. + Interpretation: - `0`: matches the starter; @@ -141,11 +182,23 @@ authoritative interaction meter before making sample-efficiency claims across agents. Internal MPC transitions must also remain separate from external environment -steps in any future accounting. +steps in any future accounting. The final reference performs roughly +`96 x 10` copied-model environment steps for every scored external action, so +full hidden evaluation is intentionally a long-running publication check. + +Ordinary unit tests use an injected one-step hidden config. The full +five-seed, 1,000-step reproduction is opt-in through: + +```bash +GENESISBENCH_RUN_SLOW_ANT_MPC=1 \ + uv run pytest -q tests/test_simulation_heuristics_ant_v1.py +``` ## Benchmark integrity - The agent container receives the public task but not `verifier/`. +- The agent receives only a copied model path for optional MPC, never the live + scored environment. - Credentials are supplied through a temporary mode-`0600` file and removed after agent startup. - Final scores come from the independently imported final policy. diff --git a/experiments/article_suite/README.md b/experiments/article_suite/README.md new file mode 100644 index 0000000..8a908d2 --- /dev/null +++ b/experiments/article_suite/README.md @@ -0,0 +1,88 @@ +# Learning Beyond Gradients article suite + +This experiment evaluates the four canonical GenesisBench model routes across +the nine task packages derived from the article: + +1. `simulation_heuristics_ant_v1` +2. `simulation_heuristics_pong_ram_v1` +3. `simulation_heuristics_breakout_ram_v1` +4. `simulation_heuristics_breakout_rgb_v1` +5. `simulation_heuristics_halfcheetah_v1` +6. `simulation_heuristics_vizdoom_d1_v1` +7. `simulation_heuristics_vizdoom_d3_v1` +8. `simulation_heuristics_atari57_v1` +9. `simulation_heuristics_montezuma_v1` + +Every run uses BenchFlow's registered `opencode` ACP harness. OpenHands is not +part of this suite. + +## Credentials + +Credential values are read from the process environment or an env file. They +are never copied into a task workspace or committed artifact. + +The Azure routes require: + +```text +AZURE_API_ENDPOINT +AZURE_API_KEY +``` + +The Claude Opus route uses the pinned OpenCode Claude-auth plugin and requires: + +```text +CLAUDE_CODE_OAUTH_TOKEN +``` + +## Run + +Docker must be running because the authoritative suite uses isolated BenchFlow +task environments: + +```bash +uv run python scripts/run_article_suite.py \ + --env-file /path/to/credentials.env \ + --model gpt-5.6-sol +``` + +Atari57 requests more CPU, memory, and storage than the current Daytona account +allows. The all-nine commands therefore use the default local Docker sandbox. +Use Daytona only for selected non-Atari tasks. + +Run all four models sequentially: + +```bash +uv run python scripts/run_article_suite.py \ + --env-file /path/to/credentials.env \ + --all-models +``` + +Long runs are resumable at task granularity. For example: + +```bash +uv run python scripts/run_article_suite.py \ + --env-file /path/to/credentials.env \ + --model gpt-5.6-sol \ + --task simulation_heuristics_halfcheetah_v1 \ + --sandbox daytona +``` + +The leaderboard builder selects the latest successful result for every +model/task pair across all run batches. + +GPT-5.6 Sol is routed directly through OpenCode's Azure Responses-API provider +with reasoning effort `max`. BenchFlow still owns sandboxing, task staging, +ACP trajectory capture, and verifier execution. Trusted LiteLLM usage tracking +is disabled for this suite because BenchFlow 0.6.5's chat-completions gateway +cannot faithfully transform GPT-5.6 Sol tool calls; the run metadata records +this limitation explicitly. + +## Build the aggregate leaderboard + +```bash +uv run python scripts/build_article_suite_leaderboard.py +``` + +The aggregate score is the arithmetic mean of the nine normalized task scores. +Each task maps its starter policy to `0` and its trusted article-level reference +to `100`. diff --git a/experiments/article_suite/models.toml b/experiments/article_suite/models.toml new file mode 100644 index 0000000..a09287e --- /dev/null +++ b/experiments/article_suite/models.toml @@ -0,0 +1,31 @@ +version = "1.0" + +[[models]] +id = "gpt-5.6-sol" +display_name = "GPT-5.6 Sol" +model = "azure/gpt-5.6-sol" +provider = "azure" +provider_reasoning_effort = "max" + +[[models]] +id = "gpt-5.5" +display_name = "GPT-5.5" +model = "azure/gpt-5.5" +provider = "azure" +provider_reasoning_effort = "xhigh" + +[[models]] +id = "claude-opus-4.8" +display_name = "Claude Opus 4.8" +model = "anthropic/claude-opus-4-8" +provider = "claude_oauth" +provider_reasoning_effort = "max" +agent_idle_timeout_sec = 0 +daytona_pty_readline_timeout_sec = 3600 + +[[models]] +id = "gpt-5.4-mini" +display_name = "GPT-5.4 Mini" +model = "azure/gpt-5.4-mini" +provider = "azure" +provider_reasoning_effort = "xhigh" diff --git a/experiments/simulation_heuristics_ant_v1/README.md b/experiments/simulation_heuristics_ant_v1/README.md index e4379d4..8a42b45 100644 --- a/experiments/simulation_heuristics_ant_v1/README.md +++ b/experiments/simulation_heuristics_ant_v1/README.md @@ -1,4 +1,9 @@ -# Simulation Heuristics Ant v1 OpenHands sweep +# Legacy Simulation Heuristics Ant v1 OpenHands sweep + +This directory documents the historical Ant-only sweep. It is retained so the +published Ant leaderboard can be reproduced and audited. New experiments must +use the OpenCode article-suite runner documented in +`experiments/article_suite/README.md`. This experiment runs the same GenesisBench `simulation_heuristics_ant_v1` policy-improvement task with four model diff --git a/leaderboard/ARTICLE_SUITE.md b/leaderboard/ARTICLE_SUITE.md new file mode 100644 index 0000000..08187da --- /dev/null +++ b/leaderboard/ARTICLE_SUITE.md @@ -0,0 +1,10 @@ +# GenesisBench Learning Beyond Gradients Article Suite + +The aggregate score is the arithmetic mean of nine normalized task scores. Starter policies map to 0 and trusted article-level references map to 100. + +| Rank | Model | Harness | Effort | Average | Ant | Pong | Breakout RAM | Breakout RGB | HalfCheetah | Doom D1 | Doom D3 | Atari57 | Montezuma | +| ---: | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | GPT-5.5 | opencode | xhigh | 43.19 | -16.89 | 45.83 | 95.60 | 70.76 | 2.74 | 85.20 | 5.48 | 0.00 | 100.00 | +| 2 | Claude Opus 4.8 | opencode | max | 39.82 | 14.05 | 100.00 | 0.00 | 0.00 | 26.99 | 95.06 | 122.26 | 0.00 | 0.00 | +| 3 | GPT-5.6 Sol | opencode | max | 39.38 | 39.20 | 52.92 | 100.00 | 70.76 | 15.87 | 49.41 | 26.24 | 0.00 | 0.00 | +| 4 | GPT-5.4 Mini | opencode | xhigh | -29.72 | -31.36 | -75.00 | 2.52 | 15.52 | -8.17 | -161.12 | -9.86 | 0.00 | 0.00 | diff --git a/leaderboard/README.md b/leaderboard/README.md index 35facd6..00da15b 100644 --- a/leaderboard/README.md +++ b/leaderboard/README.md @@ -1,4 +1,17 @@ -# GenesisBench Simulation Heuristics Ant v1 Leaderboard +# GenesisBench Leaderboards + +## Learning Beyond Gradients article suite + +| Rank | Model | Nine-task average | +| ---: | --- | ---: | +| 1 | GPT-5.5 | 43.19 | +| 2 | Claude Opus 4.8 | 39.82 | +| 3 | GPT-5.6 Sol | 39.38 | +| 4 | GPT-5.4 Mini | -29.72 | + +See [`ARTICLE_SUITE.md`](ARTICLE_SUITE.md) for all nine task columns. + +## Legacy Simulation Heuristics Ant v1 ![GenesisBench Simulation Heuristics Ant v1 leaderboard](simulation_heuristics_ant_v1_leaderboard.png) diff --git a/leaderboard/article_suite.json b/leaderboard/article_suite.json new file mode 100644 index 0000000..0148ab5 --- /dev/null +++ b/leaderboard/article_suite.json @@ -0,0 +1,234 @@ +{ + "aggregation": "arithmetic_mean_of_normalized_task_scores", + "benchmark": "learning_beyond_gradients_article_suite", + "generated_at": "2026-07-13T15:34:22.260689+00:00", + "rows": [ + { + "average_normalized_score": 43.19059338577778, + "harness": "opencode", + "model": "GPT-5.5", + "model_id": "gpt-5.5", + "provider_reasoning_effort": "xhigh", + "rank": 1, + "source_runs": { + "simulation_heuristics_ant_v1": "leaderboard/runs/article_suite/20260713T095807Z/gpt-5.5", + "simulation_heuristics_atari57_v1": "leaderboard/runs/article_suite/20260713T111604Z/gpt-5.5", + "simulation_heuristics_breakout_ram_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "simulation_heuristics_halfcheetah_v1": "leaderboard/runs/article_suite/20260713T103810Z/gpt-5.5", + "simulation_heuristics_montezuma_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "simulation_heuristics_pong_ram_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/runs/article_suite/20260713T102218Z/gpt-5.5", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5" + }, + "submission_details": { + "simulation_heuristics_ant_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/score.json", + "simulation_heuristics_atari57_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/score.json", + "simulation_heuristics_breakout_ram_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/score.json", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/score.json", + "simulation_heuristics_halfcheetah_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/score.json", + "simulation_heuristics_montezuma_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/score.json", + "simulation_heuristics_pong_ram_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/score.json", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/score.json", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/score.json" + }, + "task_scores": { + "simulation_heuristics_ant_v1": -16.891796576, + "simulation_heuristics_atari57_v1": 0.0, + "simulation_heuristics_breakout_ram_v1": 95.597484277, + "simulation_heuristics_breakout_rgb_v1": 70.758122744, + "simulation_heuristics_halfcheetah_v1": 2.73755349, + "simulation_heuristics_montezuma_v1": 100.0, + "simulation_heuristics_pong_ram_v1": 45.833333333, + "simulation_heuristics_vizdoom_d1_v1": 85.199359782, + "simulation_heuristics_vizdoom_d3_v1": 5.481283422 + } + }, + { + "average_normalized_score": 39.81691467833333, + "harness": "opencode", + "model": "Claude Opus 4.8", + "model_id": "claude-opus-4.8", + "provider_reasoning_effort": "max", + "rank": 2, + "source_runs": { + "simulation_heuristics_ant_v1": "leaderboard/runs/article_suite/20260713T142919Z/claude-opus-4.8", + "simulation_heuristics_atari57_v1": "leaderboard/runs/article_suite/20260713T123222Z/claude-opus-4.8", + "simulation_heuristics_breakout_ram_v1": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "simulation_heuristics_halfcheetah_v1": "leaderboard/runs/article_suite/20260713T111714Z/claude-opus-4.8", + "simulation_heuristics_montezuma_v1": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8", + "simulation_heuristics_pong_ram_v1": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8" + }, + "submission_details": { + "simulation_heuristics_ant_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/score.json", + "simulation_heuristics_atari57_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/score.json", + "simulation_heuristics_breakout_ram_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/score.json", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/score.json", + "simulation_heuristics_halfcheetah_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/score.json", + "simulation_heuristics_montezuma_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/score.json", + "simulation_heuristics_pong_ram_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/score.json", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/score.json", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/score.json" + }, + "task_scores": { + "simulation_heuristics_ant_v1": 14.050353128, + "simulation_heuristics_atari57_v1": 0.0, + "simulation_heuristics_breakout_ram_v1": 0.0, + "simulation_heuristics_breakout_rgb_v1": 0.0, + "simulation_heuristics_halfcheetah_v1": 26.986707946, + "simulation_heuristics_montezuma_v1": 0.0, + "simulation_heuristics_pong_ram_v1": 100.0, + "simulation_heuristics_vizdoom_d1_v1": 95.055812742, + "simulation_heuristics_vizdoom_d3_v1": 122.259358289 + } + }, + { + "average_normalized_score": 39.37755364144445, + "harness": "opencode", + "model": "GPT-5.6 Sol", + "model_id": "gpt-5.6-sol", + "provider_reasoning_effort": "max", + "rank": 3, + "source_runs": { + "simulation_heuristics_ant_v1": "leaderboard/runs/article_suite/20260713T093815Z/gpt-5.6-sol", + "simulation_heuristics_atari57_v1": "leaderboard/runs/article_suite/20260713T111156Z/gpt-5.6-sol", + "simulation_heuristics_breakout_ram_v1": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "simulation_heuristics_halfcheetah_v1": "leaderboard/runs/article_suite/20260713T084340Z/gpt-5.6-sol", + "simulation_heuristics_montezuma_v1": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "simulation_heuristics_pong_ram_v1": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/runs/article_suite/20260713T101636Z/gpt-5.6-sol", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/runs/article_suite/20260713T083940Z/gpt-5.6-sol" + }, + "submission_details": { + "simulation_heuristics_ant_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/score.json", + "simulation_heuristics_atari57_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/score.json", + "simulation_heuristics_breakout_ram_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/score.json", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/score.json", + "simulation_heuristics_halfcheetah_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/score.json", + "simulation_heuristics_montezuma_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/score.json", + "simulation_heuristics_pong_ram_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/score.json", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/score.json", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/score.json" + }, + "task_scores": { + "simulation_heuristics_ant_v1": 39.203600385, + "simulation_heuristics_atari57_v1": 0.0, + "simulation_heuristics_breakout_ram_v1": 100.0, + "simulation_heuristics_breakout_rgb_v1": 70.758122744, + "simulation_heuristics_halfcheetah_v1": 15.873072595, + "simulation_heuristics_montezuma_v1": 0.0, + "simulation_heuristics_pong_ram_v1": 52.916666667, + "simulation_heuristics_vizdoom_d1_v1": 49.409889366, + "simulation_heuristics_vizdoom_d3_v1": 26.236631016 + } + }, + { + "average_normalized_score": -29.71822245, + "harness": "opencode", + "model": "GPT-5.4 Mini", + "model_id": "gpt-5.4-mini", + "provider_reasoning_effort": "xhigh", + "rank": 4, + "source_runs": { + "simulation_heuristics_ant_v1": "leaderboard/runs/article_suite/20260713T104315Z/gpt-5.4-mini", + "simulation_heuristics_atari57_v1": "leaderboard/runs/article_suite/20260713T111944Z/gpt-5.4-mini", + "simulation_heuristics_breakout_ram_v1": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "simulation_heuristics_halfcheetah_v1": "leaderboard/runs/article_suite/20260713T093314Z/gpt-5.4-mini", + "simulation_heuristics_montezuma_v1": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "simulation_heuristics_pong_ram_v1": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/runs/article_suite/20260713T102707Z/gpt-5.4-mini", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/runs/article_suite/20260713T104034Z/gpt-5.4-mini" + }, + "submission_details": { + "simulation_heuristics_ant_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/score.json", + "simulation_heuristics_atari57_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/score.json", + "simulation_heuristics_breakout_ram_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/score.json", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/score.json", + "simulation_heuristics_halfcheetah_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/score.json", + "simulation_heuristics_montezuma_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/score.json", + "simulation_heuristics_pong_ram_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/score.json", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/score.json", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/score.json" + }, + "task_scores": { + "simulation_heuristics_ant_v1": -31.35954984, + "simulation_heuristics_atari57_v1": 0.0, + "simulation_heuristics_breakout_ram_v1": 2.51572327, + "simulation_heuristics_breakout_rgb_v1": 15.523465704, + "simulation_heuristics_halfcheetah_v1": -8.167582379, + "simulation_heuristics_montezuma_v1": 0.0, + "simulation_heuristics_pong_ram_v1": -75.0, + "simulation_heuristics_vizdoom_d1_v1": -161.116433137, + "simulation_heuristics_vizdoom_d3_v1": -9.859625668 + } + } + ], + "source_runs": { + "claude-opus-4.8": { + "simulation_heuristics_ant_v1": "leaderboard/runs/article_suite/20260713T142919Z/claude-opus-4.8", + "simulation_heuristics_atari57_v1": "leaderboard/runs/article_suite/20260713T123222Z/claude-opus-4.8", + "simulation_heuristics_breakout_ram_v1": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "simulation_heuristics_halfcheetah_v1": "leaderboard/runs/article_suite/20260713T111714Z/claude-opus-4.8", + "simulation_heuristics_montezuma_v1": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8", + "simulation_heuristics_pong_ram_v1": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8" + }, + "gpt-5.4-mini": { + "simulation_heuristics_ant_v1": "leaderboard/runs/article_suite/20260713T104315Z/gpt-5.4-mini", + "simulation_heuristics_atari57_v1": "leaderboard/runs/article_suite/20260713T111944Z/gpt-5.4-mini", + "simulation_heuristics_breakout_ram_v1": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "simulation_heuristics_halfcheetah_v1": "leaderboard/runs/article_suite/20260713T093314Z/gpt-5.4-mini", + "simulation_heuristics_montezuma_v1": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "simulation_heuristics_pong_ram_v1": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/runs/article_suite/20260713T102707Z/gpt-5.4-mini", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/runs/article_suite/20260713T104034Z/gpt-5.4-mini" + }, + "gpt-5.5": { + "simulation_heuristics_ant_v1": "leaderboard/runs/article_suite/20260713T095807Z/gpt-5.5", + "simulation_heuristics_atari57_v1": "leaderboard/runs/article_suite/20260713T111604Z/gpt-5.5", + "simulation_heuristics_breakout_ram_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "simulation_heuristics_halfcheetah_v1": "leaderboard/runs/article_suite/20260713T103810Z/gpt-5.5", + "simulation_heuristics_montezuma_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "simulation_heuristics_pong_ram_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/runs/article_suite/20260713T102218Z/gpt-5.5", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5" + }, + "gpt-5.6-sol": { + "simulation_heuristics_ant_v1": "leaderboard/runs/article_suite/20260713T093815Z/gpt-5.6-sol", + "simulation_heuristics_atari57_v1": "leaderboard/runs/article_suite/20260713T111156Z/gpt-5.6-sol", + "simulation_heuristics_breakout_ram_v1": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "simulation_heuristics_breakout_rgb_v1": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "simulation_heuristics_halfcheetah_v1": "leaderboard/runs/article_suite/20260713T084340Z/gpt-5.6-sol", + "simulation_heuristics_montezuma_v1": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "simulation_heuristics_pong_ram_v1": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "simulation_heuristics_vizdoom_d1_v1": "leaderboard/runs/article_suite/20260713T101636Z/gpt-5.6-sol", + "simulation_heuristics_vizdoom_d3_v1": "leaderboard/runs/article_suite/20260713T083940Z/gpt-5.6-sol" + } + }, + "task_count": 9, + "task_digest_compatibility": { + "simulation_heuristics_ant_v1": { + "sha256:bbb533da0cb86459f4d49dee667e6c73ac54c0188bc40e54e911d50ef3c3bc38": "Score-equivalent to the current task. The only later change adds a fail-closed internal timeout for candidates whose verifier would otherwise exceed BenchFlow's deadline." + } + }, + "tasks": [ + "simulation_heuristics_ant_v1", + "simulation_heuristics_pong_ram_v1", + "simulation_heuristics_breakout_ram_v1", + "simulation_heuristics_breakout_rgb_v1", + "simulation_heuristics_halfcheetah_v1", + "simulation_heuristics_vizdoom_d1_v1", + "simulation_heuristics_vizdoom_d3_v1", + "simulation_heuristics_atari57_v1", + "simulation_heuristics_montezuma_v1" + ] +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/metadata.json new file mode 100644 index 0000000..a4719be --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/metadata.json @@ -0,0 +1,29 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:9da0e00147cf66804e6c2fc17869606bea8f260850c5447989a8880eef940d45", + "digest_compatibility_note": null, + "finished_at": 1783956862.2606888, + "harness": "opencode", + "model": { + "agent_idle_timeout_sec": 0, + "daytona_pty_readline_timeout_sec": 3600, + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 14.050353128, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T142919Z/claude-opus-4.8", + "task": "simulation_heuristics_ant_v1", + "task_digest": "sha256:9da0e00147cf66804e6c2fc17869606bea8f260850c5447989a8880eef940d45", + "token_usage": { + "final_input_tokens": 0.0, + "final_output_tokens": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0 + }, + "tool_calls": 48 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/score.json new file mode 100644 index 0000000..ac313ff --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/score.json @@ -0,0 +1,141 @@ +{ + "hidden_nominal": { + "episodes": [ + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.025934887526091188, + "policy_error": null, + "return": 2947.879693227018, + "seed": 101, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 102.77571899817438, + "x_velocity": 1.8673987220154231 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.02599245577584952, + "policy_error": null, + "return": 3000.163186740716, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 105.33522389819196, + "x_velocity": 2.1574081280110136 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.02603654988342896, + "policy_error": null, + "return": 2772.4088822782287, + "seed": 307, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 93.7636515447122, + "x_velocity": 2.4267700657179603 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.02577067888341844, + "policy_error": null, + "return": 2990.8335078285263, + "seed": 401, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 104.90485217547173, + "x_velocity": 2.078351224632513 + } + ], + "fall_rate": 0.0, + "invalid_episode_rate": 0.0, + "max_return": 3000.163186740716, + "max_steps": 1000, + "mean_action_latency_ms": 0.025933643017197028, + "mean_return": 2927.821317518622, + "mean_x_position": 101.69486165413757, + "min_return": 2772.4088822782287, + "policy_path": "submitted_artifact", + "score": 2927.821317518622 + }, + "hidden_nominal_weight": 0.7, + "hidden_robustness": { + "episodes": [ + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.025812971696723253, + "policy_error": null, + "return": 3166.4558227561265, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 113.74752702527248, + "x_velocity": 3.0189141499053562 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.02593717718264088, + "policy_error": null, + "return": 3177.1212044211834, + "seed": 601, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 114.37538391255823, + "x_velocity": 2.4397722759977114 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.026038738433271646, + "policy_error": null, + "return": 2839.9126867592117, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 97.73666401480078, + "x_velocity": 0.6070665236407535 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.026594983239192516, + "policy_error": null, + "return": 2628.443172724322, + "seed": 601, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 87.07088591700881, + "x_velocity": 1.804739145616736 + } + ], + "fall_rate": 0.0, + "invalid_episode_rate": 0.0, + "max_return": 3177.1212044211834, + "max_steps": 1000, + "mean_action_latency_ms": 0.026095967637957074, + "mean_return": 2952.983221665211, + "mean_x_position": 103.23261521741007, + "min_return": 2628.443172724322, + "policy_path": "submitted_artifact", + "score": 2952.983221665211 + }, + "hidden_robustness_weight": 0.3, + "normalized_score": 14.050353128, + "reference_score": 5942.333244755838, + "score": 2935.3698887625987, + "starter_score": 2443.815838992282, + "unique_policy_evaluations": 3 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/metadata.json new file mode 100644 index 0000000..4b1b2ef --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/metadata.json @@ -0,0 +1,28 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:a3ec10ab61a65c03ca17e6460147a3178e222bd1c48964727ea9daae54717ebe", + "digest_compatibility_note": null, + "finished_at": 1783951813.6763892, + "harness": "opencode", + "model": { + "agent_idle_timeout_sec": 0, + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T123222Z/claude-opus-4.8", + "task": "simulation_heuristics_atari57_v1", + "task_digest": "sha256:a3ec10ab61a65c03ca17e6460147a3178e222bd1c48964727ea9daae54717ebe", + "token_usage": { + "final_input_tokens": 2.0, + "final_output_tokens": 789.0, + "input_tokens": 2.0, + "output_tokens": 789.0, + "total_tokens": 74672.0 + }, + "tool_calls": 65 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/score.json new file mode 100644 index 0000000..7788108 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/score.json @@ -0,0 +1,15 @@ +{ + "disqualification_reason": "Official suite requires 342 completed search-ledger records; found 0.", + "evaluation": null, + "interaction_budget": { + "completed_trajectories": 0, + "counted_env_steps": 0, + "planned_trajectories": 342, + "target_env_steps": 6840000000 + }, + "normalized_score": 0.0, + "protocol_complete": false, + "reference_score": 0.8283015254994576, + "score": 0.0, + "starter_score": 0.0 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/metadata.json new file mode 100644 index 0000000..517c316 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:77215337f89f950f83b8f7436f799959c2ddccd4102552c6302b13d30cc57199", + "digest_compatibility_note": null, + "finished_at": 1783940397.5345018, + "harness": "opencode", + "model": { + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "task": "simulation_heuristics_breakout_ram_v1", + "task_digest": "sha256:77215337f89f950f83b8f7436f799959c2ddccd4102552c6302b13d30cc57199", + "token_usage": { + "final_input_tokens": 0.0, + "final_output_tokens": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0 + }, + "tool_calls": 53 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/score.json new file mode 100644 index 0000000..66044e3 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/score.json @@ -0,0 +1,80 @@ +{ + "normalized_score": 0.0, + "reference_score": 864.0, + "score": 387.0, + "starter_score": 387.0, + "suites": { + "hidden_nominal": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 4, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.004508449714329025, + "observation_mode": "ram", + "policy_error": null, + "return": 387.0, + "seed": 101, + "terminated": false, + "truncated": true, + "variant": "nominal", + "warmup_steps": 0 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 387.0, + "max_steps": 108000, + "mean_action_latency_ms": 0.004508449714329025, + "mean_return": 387.0, + "min_return": 387.0, + "observation_mode": "ram", + "policy_path": "submitted_artifact", + "score": 387.0 + }, + "hidden_shifted_start": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 4, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.004490307122641416, + "observation_mode": "ram", + "policy_error": null, + "return": 387.0, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "left_6", + "warmup_steps": 6 + }, + { + "final_lives": 4, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.00453408551294598, + "observation_mode": "ram", + "policy_error": null, + "return": 387.0, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "right_6", + "warmup_steps": 6 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 387.0, + "max_steps": 108000, + "mean_action_latency_ms": 0.004512196317793698, + "mean_return": 387.0, + "min_return": 387.0, + "observation_mode": "ram", + "policy_path": "submitted_artifact", + "score": 387.0 + } + }, + "target_gap": -477.0, + "target_score": 864.0 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/metadata.json new file mode 100644 index 0000000..a796e83 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:7e126380af5345a4bfa0c9d29e45e68b8f1aa20d5d655d12187744100b24441a", + "digest_compatibility_note": null, + "finished_at": 1783940397.5345018, + "harness": "opencode", + "model": { + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "task": "simulation_heuristics_breakout_rgb_v1", + "task_digest": "sha256:7e126380af5345a4bfa0c9d29e45e68b8f1aa20d5d655d12187744100b24441a", + "token_usage": { + "final_input_tokens": 0.0, + "final_output_tokens": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0 + }, + "tool_calls": 46 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/score.json new file mode 100644 index 0000000..6f0c5d2 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/score.json @@ -0,0 +1,66 @@ +{ + "normalized_score": 0.0, + "reference_score": 864.0, + "score": 310.0, + "starter_score": 310.0, + "suites": { + "hidden_nominal": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 0, + "invalid_action": false, + "length": 11358, + "mean_action_latency_ms": 0.4728028780663805, + "observation_mode": "rgb", + "policy_error": null, + "return": 310.0, + "seed": 101, + "terminated": true, + "truncated": false, + "variant": "nominal", + "warmup_steps": 0 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 310.0, + "max_steps": 30000, + "mean_action_latency_ms": 0.4728028780663805, + "mean_return": 310.0, + "min_return": 310.0, + "observation_mode": "rgb", + "policy_path": "submitted_artifact", + "score": 310.0 + }, + "hidden_shifted_start": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 0, + "invalid_action": false, + "length": 11352, + "mean_action_latency_ms": 0.48375160852496374, + "observation_mode": "rgb", + "policy_error": null, + "return": 310.0, + "seed": 211, + "terminated": true, + "truncated": false, + "variant": "right_6", + "warmup_steps": 6 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 310.0, + "max_steps": 30000, + "mean_action_latency_ms": 0.48375160852496374, + "mean_return": 310.0, + "min_return": 310.0, + "observation_mode": "rgb", + "policy_path": "submitted_artifact", + "score": 310.0 + } + }, + "target_gap": -554.0, + "target_score": 864.0 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/metadata.json new file mode 100644 index 0000000..0a6c45a --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/metadata.json @@ -0,0 +1,28 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:80c439f53e4ab964f9d7443cd7fb8f25cf6645a0bc288b0496b871c1800ebe78", + "digest_compatibility_note": null, + "finished_at": 1783947693.1667001, + "harness": "opencode", + "model": { + "agent_idle_timeout_sec": 0, + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 26.986707946, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T111714Z/claude-opus-4.8", + "task": "simulation_heuristics_halfcheetah_v1", + "task_digest": "sha256:80c439f53e4ab964f9d7443cd7fb8f25cf6645a0bc288b0496b871c1800ebe78", + "token_usage": { + "final_input_tokens": 0.0, + "final_output_tokens": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0 + }, + "tool_calls": 45 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/score.json new file mode 100644 index 0000000..62f5166 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/score.json @@ -0,0 +1,98 @@ +{ + "hidden_nominal": { + "episodes": [ + { + "control_cost": 317.1566189676523, + "forward_reward": 6636.299989167559, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 122.95847277203575, + "policy_error": null, + "return": 6319.143370199905, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 331.8043416293857, + "x_velocity": 8.216671429596408 + }, + { + "control_cost": 314.24980780109763, + "forward_reward": 6272.707764268625, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 123.8214553501457, + "policy_error": null, + "return": 5958.45795646753, + "seed": 307, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 313.6619722793685, + "x_velocity": 2.1795015376377602 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 6319.143370199905, + "max_steps": 1000, + "mean_action_latency_ms": 123.38996406109072, + "mean_control_cost": 315.703213384375, + "mean_forward_reward": 6454.503876718092, + "mean_return": 6138.800663333717, + "mean_x_position": 322.73315695437714, + "min_return": 5958.45795646753, + "policy_path": "submitted_artifact", + "score": 6138.800663333717 + }, + "hidden_nominal_weight": 0.7, + "hidden_robustness": { + "episodes": [ + { + "control_cost": 312.9004553593695, + "forward_reward": 7637.3131141575, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 119.81285913009197, + "policy_error": null, + "return": 7324.412658798133, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 381.8479773304825, + "x_velocity": 8.203061195650385 + }, + { + "control_cost": 321.7466217055917, + "forward_reward": 4492.963484628226, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 125.38341190014035, + "policy_error": null, + "return": 4171.216862922633, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 224.63049585401836, + "x_velocity": 2.2801165132939816 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 7324.412658798133, + "max_steps": 1000, + "mean_action_latency_ms": 122.59813551511616, + "mean_control_cost": 317.3235385324806, + "mean_forward_reward": 6065.138299392864, + "mean_return": 5747.814760860383, + "mean_x_position": 303.23923659225045, + "min_return": 4171.216862922633, + "policy_path": "submitted_artifact", + "score": 5747.814760860383 + }, + "hidden_robustness_weight": 0.3, + "normalized_score": 26.986707946, + "reference_score": 11487.256451471967, + "score": 6021.504892591717, + "starter_score": 4001.2886158917113 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/metadata.json new file mode 100644 index 0000000..e58d81a --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/metadata.json @@ -0,0 +1,28 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:33650c821735e31eea00f2ba46ae09b5d19134820c2ec005e8e96d92cb0b66d0", + "digest_compatibility_note": null, + "finished_at": 1783945917.227471, + "harness": "opencode", + "model": { + "agent_idle_timeout_sec": 0, + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8", + "task": "simulation_heuristics_montezuma_v1", + "task_digest": "sha256:33650c821735e31eea00f2ba46ae09b5d19134820c2ec005e8e96d92cb0b66d0", + "token_usage": { + "final_input_tokens": 0.0, + "final_output_tokens": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0 + }, + "tool_calls": 109 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/score.json new file mode 100644 index 0000000..57093ba --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/score.json @@ -0,0 +1,216 @@ +{ + "normalized_score": 0.0, + "reference_score": 400.0, + "score": 0.0, + "starter_score": 0.0, + "suites": { + "hidden_seeds": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.09138794615864754, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10007, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + }, + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.15000148533055416, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10103, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.12069471574460085, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.1 + }, + "noop_variants": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.120189058112786, + "policy_error": null, + "policy_steps": 2597, + "pre_policy_noops": 3, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "startup_noop_3" + }, + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.10412243624359288, + "policy_error": null, + "policy_steps": 2587, + "pre_policy_noops": 13, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "startup_noop_13" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.11215574717818944, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.1 + }, + "recovery": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 512, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.09218373670173012, + "policy_error": null, + "policy_steps": 2084, + "pre_policy_noops": 4, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_512_noop_4" + }, + { + "bootstrap_steps": 512, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.1754581480501936, + "policy_error": null, + "policy_steps": 2080, + "pre_policy_noops": 8, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_512_noop_8" + }, + { + "bootstrap_steps": 768, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.0963938997580054, + "policy_error": null, + "policy_steps": 1830, + "pre_policy_noops": 2, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_768_noop_2" + }, + { + "bootstrap_steps": 900, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.09158327189438484, + "policy_error": null, + "policy_steps": 1700, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_900" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.1139047641010785, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.55 + }, + "reproduction": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.09677237544495326, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.09677237544495326, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.25 + } + }, + "target_score": 400.0 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/metadata.json new file mode 100644 index 0000000..c925b50 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:f0b6f90f1d8072e75f0e10f90f4b51e51e31b3590bb2c2736778ba155b23389f", + "digest_compatibility_note": null, + "finished_at": 1783940397.5345018, + "harness": "opencode", + "model": { + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 100.0, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T090958Z/claude-opus-4.8", + "task": "simulation_heuristics_pong_ram_v1", + "task_digest": "sha256:f0b6f90f1d8072e75f0e10f90f4b51e51e31b3590bb2c2736778ba155b23389f", + "token_usage": { + "final_input_tokens": 2.0, + "final_output_tokens": 654.0, + "input_tokens": 2.0, + "output_tokens": 654.0, + "total_tokens": 67139.0 + }, + "tool_calls": 41 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/score.json new file mode 100644 index 0000000..26916ec --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/score.json @@ -0,0 +1,133 @@ +{ + "normalized_score": 100.0, + "reference_score": 21.0, + "score": 21.0, + "starter_score": -2.9999999999999996, + "suite_weights": { + "hidden_nominal": 0.7, + "hidden_reset_robustness": 0.3 + }, + "suites": { + "hidden_nominal": { + "episodes": [ + { + "invalid_action": false, + "length": 22904, + "mean_action_latency_ms": 0.0027309710952318966, + "points_against": 0, + "points_for": 21, + "policy_error": null, + "score": 21.0, + "seed": 101, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "article_nominal", + "won": true + }, + { + "invalid_action": false, + "length": 22904, + "mean_action_latency_ms": 0.0028047818248786377, + "points_against": 0, + "points_for": 21, + "policy_error": null, + "score": 21.0, + "seed": 211, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "article_nominal", + "won": true + }, + { + "invalid_action": false, + "length": 22904, + "mean_action_latency_ms": 0.002708563199211491, + "points_against": 0, + "points_for": 21, + "policy_error": null, + "score": 21.0, + "seed": 307, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "article_nominal", + "won": true + } + ], + "invalid_episode_rate": 0.0, + "max_score": 21.0, + "max_steps": 27000, + "mean_action_latency_ms": 0.002748105373107342, + "mean_score": 21.0, + "min_score": 21.0, + "policy_path": "submitted_artifact", + "score": 21.0, + "target_score": 21.0, + "target_score_rate": 1.0, + "win_rate": 1.0 + }, + "hidden_reset_robustness": { + "episodes": [ + { + "invalid_action": false, + "length": 22904, + "mean_action_latency_ms": 0.0026605053441459132, + "points_against": 0, + "points_for": 21, + "policy_error": null, + "score": 21.0, + "seed": 503, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "randomized_reset", + "won": true + }, + { + "invalid_action": false, + "length": 22900, + "mean_action_latency_ms": 0.0028647481988481036, + "points_against": 0, + "points_for": 21, + "policy_error": null, + "score": 21.0, + "seed": 601, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "randomized_reset", + "won": true + }, + { + "invalid_action": false, + "length": 22901, + "mean_action_latency_ms": 0.003009833623868847, + "points_against": 0, + "points_for": 21, + "policy_error": null, + "score": 21.0, + "seed": 701, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "randomized_reset", + "won": true + } + ], + "invalid_episode_rate": 0.0, + "max_score": 21.0, + "max_steps": 27000, + "mean_action_latency_ms": 0.0028450290556209547, + "mean_score": 21.0, + "min_score": 21.0, + "policy_path": "submitted_artifact", + "score": 21.0, + "target_score": 21.0, + "target_score_rate": 1.0, + "win_rate": 1.0 + } + }, + "target_score": 21.0 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/metadata.json new file mode 100644 index 0000000..601ec25 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/metadata.json @@ -0,0 +1,28 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:b3431b238bec4e66af6189d30bcd15d5cd227144dfe8bf0dd1011aa5416c1436", + "digest_compatibility_note": null, + "finished_at": 1783945917.227471, + "harness": "opencode", + "model": { + "agent_idle_timeout_sec": 0, + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 95.055812742, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8", + "task": "simulation_heuristics_vizdoom_d1_v1", + "task_digest": "sha256:b3431b238bec4e66af6189d30bcd15d5cd227144dfe8bf0dd1011aa5416c1436", + "token_usage": { + "final_input_tokens": 2.0, + "final_output_tokens": 886.0, + "input_tokens": 2.0, + "output_tokens": 886.0, + "total_tokens": 76716.0 + }, + "tool_calls": 74 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/score.json new file mode 100644 index 0000000..f0c8e14 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/score.json @@ -0,0 +1,211 @@ +{ + "anchor_calibration": { + "envpool_version": "1.1.1", + "mode": "numeric", + "platform": "linux-x86_64" + }, + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": true + }, + "normalized_score": 95.055812742, + "reference_score": 0.8178199748694897, + "resource_cleanup": { + "removed_bytes": 22517424, + "removed_resources": 27, + "runs": [ + { + "removed_bytes": 22517424, + "removed_resources": 27, + "terminated_processes": 0 + }, + { + "removed_bytes": 0, + "removed_resources": 0, + "terminated_processes": 0 + } + ], + "terminated_processes": 0 + }, + "score": 0.7992199431359768, + "starter_score": 0.44161998964846133, + "suites": { + "hidden_nominal": { + "batch_seed": 101, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "HEALTH": 60.0 + }, + "invalid_action": false, + "lane": 0, + "length": 2100, + "mean_action_latency_ms": 0.0805211794518289, + "policy_error": null, + "return": 0.7839999310672283, + "seed": 101, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 92.0 + }, + "invalid_action": false, + "lane": 1, + "length": 2100, + "mean_action_latency_ms": 0.07343609506885211, + "policy_error": null, + "return": 0.8799999393522739, + "seed": 102, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 92.0 + }, + "invalid_action": false, + "lane": 2, + "length": 2100, + "mean_action_latency_ms": 0.06823671626902762, + "policy_error": null, + "return": 0.9279999360442162, + "seed": 103, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 85.0 + }, + "invalid_action": false, + "lane": 3, + "length": 2100, + "mean_action_latency_ms": 0.06539602542207354, + "policy_error": null, + "return": 0.7649999484419823, + "seed": 104, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 92.0 + }, + "invalid_action": false, + "lane": 4, + "length": 2100, + "mean_action_latency_ms": 0.06430936090293385, + "policy_error": null, + "return": 0.8799999393522739, + "seed": 105, + "terminated": false, + "truncated": true + } + ], + "frame_skip": 1, + "invalid_episode_rate": 0.0, + "max_return": 0.9279999360442162, + "max_steps": 2100, + "mean_action_latency_ms": 0.07037987542294319, + "mean_return": 0.8473999388515949, + "min_return": 0.7649999484419823, + "policy_path": "submitted_artifact", + "render_height": 180, + "render_width": 240, + "scenario": "d1", + "score": 0.8473999388515949 + }, + "hidden_secondary": { + "batch_seed": 607, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "HEALTH": 100.0 + }, + "invalid_action": false, + "lane": 0, + "length": 2100, + "mean_action_latency_ms": 0.09118304100065004, + "policy_error": null, + "return": 0.9359999448060989, + "seed": 607, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 100.0 + }, + "invalid_action": false, + "lane": 1, + "length": 2100, + "mean_action_latency_ms": 0.07564157158845947, + "policy_error": null, + "return": 0.8399999439716339, + "seed": 608, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 6.0 + }, + "invalid_action": false, + "lane": 2, + "length": 768, + "mean_action_latency_ms": 0.05176020446621502, + "policy_error": null, + "return": -0.062000010162591934, + "seed": 609, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 92.0 + }, + "invalid_action": false, + "lane": 3, + "length": 2100, + "mean_action_latency_ms": 0.06992494243951071, + "policy_error": null, + "return": 0.8159999437630177, + "seed": 610, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 100.0 + }, + "invalid_action": false, + "lane": 4, + "length": 2100, + "mean_action_latency_ms": 0.07116406978595824, + "policy_error": null, + "return": 0.9039999432861805, + "seed": 611, + "terminated": false, + "truncated": true + } + ], + "frame_skip": 1, + "invalid_episode_rate": 0.0, + "max_return": 0.9359999448060989, + "max_steps": 2100, + "mean_action_latency_ms": 0.0719347658561587, + "mean_return": 0.6867999531328678, + "min_return": -0.062000010162591934, + "policy_path": "submitted_artifact", + "render_height": 180, + "render_width": 240, + "scenario": "d1", + "score": 0.6867999531328678 + } + } +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/metadata.json new file mode 100644 index 0000000..7c52bcb --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/metadata.json @@ -0,0 +1,28 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:fcc6ed05673b7981aa17d60ca4e0d355fbcb57a6cbca293c0fa201ab97cdd081", + "digest_compatibility_note": null, + "finished_at": 1783945917.227471, + "harness": "opencode", + "model": { + "agent_idle_timeout_sec": 0, + "display_name": "Claude Opus 4.8", + "id": "claude-opus-4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max" + }, + "normalized_score": 122.259358289, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T110612Z/claude-opus-4.8", + "task": "simulation_heuristics_vizdoom_d3_v1", + "task_digest": "sha256:fcc6ed05673b7981aa17d60ca4e0d355fbcb57a6cbca293c0fa201ab97cdd081", + "token_usage": { + "final_input_tokens": 0.0, + "final_output_tokens": 0.0, + "input_tokens": 0.0, + "output_tokens": 0.0, + "total_tokens": 0.0 + }, + "tool_calls": 89 +} diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/score.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/score.json new file mode 100644 index 0000000..19961f6 --- /dev/null +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/score.json @@ -0,0 +1,229 @@ +{ + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": true + }, + "normalized_score": 122.259358289, + "reference_score": 331.1, + "score": 397.7, + "starter_score": 31.9, + "suites": { + "hidden_nominal": { + "batch_seed": 101, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 255.0, + "HEALTH": 91.0, + "HITCOUNT": 26.0, + "KILLCOUNT": 23.0 + }, + "invalid_action": false, + "lane": 0, + "length": 1050, + "mean_action_latency_ms": 2.7141923617039407, + "policy_error": null, + "return": 485.0, + "seed": 101, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "AMMO2": 74.0, + "DAMAGECOUNT": 240.0, + "HEALTH": 91.0, + "HITCOUNT": 22.0, + "KILLCOUNT": 22.0 + }, + "invalid_action": false, + "lane": 1, + "length": 1050, + "mean_action_latency_ms": 2.699280280087675, + "policy_error": null, + "return": 460.0, + "seed": 102, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 110.0, + "HEALTH": 9.0, + "HITCOUNT": 11.0, + "KILLCOUNT": 11.0 + }, + "invalid_action": false, + "lane": 2, + "length": 570, + "mean_action_latency_ms": 2.7295927664167, + "policy_error": null, + "return": 220.0, + "seed": 103, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 2.0, + "DAMAGECOUNT": 390.0, + "HEALTH": 100.0, + "HITCOUNT": 40.0, + "KILLCOUNT": 34.0 + }, + "invalid_action": false, + "lane": 3, + "length": 1050, + "mean_action_latency_ms": 2.7340504562570933, + "policy_error": null, + "return": 730.0, + "seed": 104, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 165.0, + "HEALTH": 10.0, + "HITCOUNT": 19.0, + "KILLCOUNT": 17.0 + }, + "invalid_action": false, + "lane": 4, + "length": 780, + "mean_action_latency_ms": 2.7058095945857272, + "policy_error": null, + "return": 335.0, + "seed": 105, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 2, + "invalid_episode_rate": 0.0, + "max_return": 730.0, + "max_steps": 1050, + "mean_action_latency_ms": 2.7165850918102272, + "mean_return": 446.0, + "min_return": 220.0, + "policy_path": "submitted_artifact", + "render_height": 480, + "render_width": 640, + "scenario": "d3", + "score": 446.0 + }, + "hidden_secondary": { + "batch_seed": 607, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 130.0, + "HEALTH": 9.0, + "HITCOUNT": 10.0, + "KILLCOUNT": 10.0 + }, + "invalid_action": false, + "lane": 0, + "length": 650, + "mean_action_latency_ms": 2.729112282395363, + "policy_error": null, + "return": 230.0, + "seed": 607, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 25.0, + "HEALTH": 22.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 + }, + "invalid_action": false, + "lane": 1, + "length": 432, + "mean_action_latency_ms": 2.695080444768623, + "policy_error": null, + "return": 45.0, + "seed": 608, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 15.0, + "DAMAGECOUNT": 145.0, + "HEALTH": 100.0, + "HITCOUNT": 14.0, + "KILLCOUNT": 14.0 + }, + "invalid_action": false, + "lane": 2, + "length": 1050, + "mean_action_latency_ms": 2.6943841115349816, + "policy_error": null, + "return": 285.0, + "seed": 609, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 130.0, + "HEALTH": 24.0, + "HITCOUNT": 11.0, + "KILLCOUNT": 11.0 + }, + "invalid_action": false, + "lane": 3, + "length": 663, + "mean_action_latency_ms": 2.6851553528894128, + "policy_error": null, + "return": 240.0, + "seed": 610, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 71.0, + "DAMAGECOUNT": 335.0, + "HEALTH": 100.0, + "HITCOUNT": 35.0, + "KILLCOUNT": 29.0 + }, + "invalid_action": false, + "lane": 4, + "length": 1050, + "mean_action_latency_ms": 2.6883774224136556, + "policy_error": null, + "return": 625.0, + "seed": 611, + "terminated": false, + "truncated": true + } + ], + "frame_skip": 2, + "invalid_episode_rate": 0.0, + "max_return": 625.0, + "max_steps": 1050, + "mean_action_latency_ms": 2.6984219228004074, + "mean_return": 285.0, + "min_return": 45.0, + "policy_path": "submitted_artifact", + "render_height": 480, + "render_width": 640, + "scenario": "d3", + "score": 285.0 + } + } +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/metadata.json new file mode 100644 index 0000000..4b26367 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:9da0e00147cf66804e6c2fc17869606bea8f260850c5447989a8880eef940d45", + "digest_compatibility_note": "Score-equivalent to the current task. The only later change adds a fail-closed internal timeout for candidates whose verifier would otherwise exceed BenchFlow's deadline.", + "finished_at": 1783941566.485236, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": -31.35954984, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T104315Z/gpt-5.4-mini", + "task": "simulation_heuristics_ant_v1", + "task_digest": "sha256:bbb533da0cb86459f4d49dee667e6c73ac54c0188bc40e54e911d50ef3c3bc38", + "token_usage": { + "final_input_tokens": 937.0, + "final_output_tokens": 225.0, + "input_tokens": 937.0, + "output_tokens": 225.0, + "total_tokens": 16010.0 + }, + "tool_calls": 16 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/score.json new file mode 100644 index 0000000..d52943f --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/score.json @@ -0,0 +1,141 @@ +{ + "hidden_nominal": { + "episodes": [ + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.0308447620482184, + "policy_error": null, + "return": 1280.8625113025073, + "seed": 101, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 19.567406497399496, + "x_velocity": -0.9134368973606399 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.0313320025743451, + "policy_error": null, + "return": 1283.799292320564, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 19.068321119541704, + "x_velocity": 0.46909212578562176 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.031062497466336936, + "policy_error": null, + "return": 1572.9322897606214, + "seed": 307, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 33.97698637026477, + "x_velocity": 0.5331860974504821 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.031104908091947436, + "policy_error": null, + "return": 1434.9175295491777, + "seed": 401, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 26.72069848487529, + "x_velocity": -0.37097842149982796 + } + ], + "fall_rate": 0.0, + "invalid_episode_rate": 0.0, + "max_return": 1572.9322897606214, + "max_steps": 1000, + "mean_action_latency_ms": 0.031086042545211967, + "mean_return": 1393.1279057332176, + "mean_x_position": 24.833353118020312, + "min_return": 1280.8625113025073, + "policy_path": "submitted_artifact", + "score": 1393.1279057332176 + }, + "hidden_nominal_weight": 0.7, + "hidden_robustness": { + "episodes": [ + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.030873946117935702, + "policy_error": null, + "return": 1236.6108861288565, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 17.199024467312558, + "x_velocity": -1.096540091826057 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.030729250283911824, + "policy_error": null, + "return": 1224.1827372398914, + "seed": 601, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 16.74834914717757, + "x_velocity": -0.43087566591353266 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.031124928907956928, + "policy_error": null, + "return": 1374.2109418292014, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 24.592110185851382, + "x_velocity": -0.4027636104083854 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.03148250802769326, + "policy_error": null, + "return": 1118.4220406880218, + "seed": 601, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 11.881812530236811, + "x_velocity": -0.21271365790770602 + } + ], + "fall_rate": 0.0, + "invalid_episode_rate": 0.0, + "max_return": 1374.2109418292014, + "max_steps": 1000, + "mean_action_latency_ms": 0.031052658334374428, + "mean_return": 1238.3566514714926, + "mean_x_position": 17.605324082644582, + "min_return": 1118.4220406880218, + "policy_path": "submitted_artifact", + "score": 1238.3566514714926 + }, + "hidden_robustness_weight": 0.3, + "normalized_score": -31.35954984, + "reference_score": 5942.333244755838, + "score": 1346.6965294546999, + "starter_score": 2443.815838992282, + "unique_policy_evaluations": 3 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/metadata.json new file mode 100644 index 0000000..6faccf9 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:a3ec10ab61a65c03ca17e6460147a3178e222bd1c48964727ea9daae54717ebe", + "digest_compatibility_note": null, + "finished_at": 1783941641.828137, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T111944Z/gpt-5.4-mini", + "task": "simulation_heuristics_atari57_v1", + "task_digest": "sha256:a3ec10ab61a65c03ca17e6460147a3178e222bd1c48964727ea9daae54717ebe", + "token_usage": { + "final_input_tokens": 461.0, + "final_output_tokens": 240.0, + "input_tokens": 461.0, + "output_tokens": 240.0, + "total_tokens": 10941.0 + }, + "tool_calls": 16 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/score.json new file mode 100644 index 0000000..7788108 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/score.json @@ -0,0 +1,15 @@ +{ + "disqualification_reason": "Official suite requires 342 completed search-ledger records; found 0.", + "evaluation": null, + "interaction_budget": { + "completed_trajectories": 0, + "counted_env_steps": 0, + "planned_trajectories": 342, + "target_env_steps": 6840000000 + }, + "normalized_score": 0.0, + "protocol_complete": false, + "reference_score": 0.8283015254994576, + "score": 0.0, + "starter_score": 0.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/metadata.json new file mode 100644 index 0000000..9d0c949 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:77215337f89f950f83b8f7436f799959c2ddccd4102552c6302b13d30cc57199", + "digest_compatibility_note": null, + "finished_at": 1783935048.629668, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 2.51572327, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "task": "simulation_heuristics_breakout_ram_v1", + "task_digest": "sha256:77215337f89f950f83b8f7436f799959c2ddccd4102552c6302b13d30cc57199", + "token_usage": { + "final_input_tokens": 932.0, + "final_output_tokens": 181.0, + "input_tokens": 932.0, + "output_tokens": 181.0, + "total_tokens": 15449.0 + }, + "tool_calls": 17 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/score.json new file mode 100644 index 0000000..c41aa06 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/score.json @@ -0,0 +1,80 @@ +{ + "normalized_score": 2.51572327, + "reference_score": 864.0, + "score": 399.0, + "starter_score": 387.0, + "suites": { + "hidden_nominal": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 3, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.0051444856271027, + "observation_mode": "ram", + "policy_error": null, + "return": 399.0, + "seed": 101, + "terminated": false, + "truncated": true, + "variant": "nominal", + "warmup_steps": 0 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 399.0, + "max_steps": 108000, + "mean_action_latency_ms": 0.0051444856271027, + "mean_return": 399.0, + "min_return": 399.0, + "observation_mode": "ram", + "policy_path": "submitted_artifact", + "score": 399.0 + }, + "hidden_shifted_start": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 3, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.005121370694487942, + "observation_mode": "ram", + "policy_error": null, + "return": 399.0, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "left_6", + "warmup_steps": 6 + }, + { + "final_lives": 3, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.004679645820201754, + "observation_mode": "ram", + "policy_error": null, + "return": 399.0, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "right_6", + "warmup_steps": 6 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 399.0, + "max_steps": 108000, + "mean_action_latency_ms": 0.004900508257344848, + "mean_return": 399.0, + "min_return": 399.0, + "observation_mode": "ram", + "policy_path": "submitted_artifact", + "score": 399.0 + } + }, + "target_gap": -465.0, + "target_score": 864.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/metadata.json new file mode 100644 index 0000000..dcb0bef --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:7e126380af5345a4bfa0c9d29e45e68b8f1aa20d5d655d12187744100b24441a", + "digest_compatibility_note": null, + "finished_at": 1783935048.629668, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 15.523465704, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "task": "simulation_heuristics_breakout_rgb_v1", + "task_digest": "sha256:7e126380af5345a4bfa0c9d29e45e68b8f1aa20d5d655d12187744100b24441a", + "token_usage": { + "final_input_tokens": 859.0, + "final_output_tokens": 202.0, + "input_tokens": 859.0, + "output_tokens": 202.0, + "total_tokens": 23077.0 + }, + "tool_calls": 21 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/score.json new file mode 100644 index 0000000..5d3ff02 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/score.json @@ -0,0 +1,66 @@ +{ + "normalized_score": 15.523465704, + "reference_score": 864.0, + "score": 396.0, + "starter_score": 310.0, + "suites": { + "hidden_nominal": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 0, + "invalid_action": false, + "length": 11673, + "mean_action_latency_ms": 0.4428296772221238, + "observation_mode": "rgb", + "policy_error": null, + "return": 396.0, + "seed": 101, + "terminated": true, + "truncated": false, + "variant": "nominal", + "warmup_steps": 0 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 396.0, + "max_steps": 30000, + "mean_action_latency_ms": 0.4428296772221238, + "mean_return": 396.0, + "min_return": 396.0, + "observation_mode": "rgb", + "policy_path": "submitted_artifact", + "score": 396.0 + }, + "hidden_shifted_start": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 0, + "invalid_action": false, + "length": 11667, + "mean_action_latency_ms": 0.4506273232825088, + "observation_mode": "rgb", + "policy_error": null, + "return": 396.0, + "seed": 211, + "terminated": true, + "truncated": false, + "variant": "right_6", + "warmup_steps": 6 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 396.0, + "max_steps": 30000, + "mean_action_latency_ms": 0.4506273232825088, + "mean_return": 396.0, + "min_return": 396.0, + "observation_mode": "rgb", + "policy_path": "submitted_artifact", + "score": 396.0 + } + }, + "target_gap": -468.0, + "target_score": 864.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/metadata.json new file mode 100644 index 0000000..50495ed --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:80c439f53e4ab964f9d7443cd7fb8f25cf6645a0bc288b0496b871c1800ebe78", + "digest_compatibility_note": null, + "finished_at": 1783937429.008938, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": -8.167582379, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T093314Z/gpt-5.4-mini", + "task": "simulation_heuristics_halfcheetah_v1", + "task_digest": "sha256:80c439f53e4ab964f9d7443cd7fb8f25cf6645a0bc288b0496b871c1800ebe78", + "token_usage": { + "final_input_tokens": 1370.0, + "final_output_tokens": 203.0, + "input_tokens": 1370.0, + "output_tokens": 203.0, + "total_tokens": 16933.0 + }, + "tool_calls": 12 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/score.json new file mode 100644 index 0000000..4b8932e --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/score.json @@ -0,0 +1,98 @@ +{ + "hidden_nominal": { + "episodes": [ + { + "control_cost": 344.8222642093897, + "forward_reward": 3834.116914743516, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.06746797123923898, + "policy_error": null, + "return": 3489.2946505341256, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 191.6951879081835, + "x_velocity": 5.427317695066039 + }, + { + "control_cost": 346.87259566783905, + "forward_reward": 3854.082422006625, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.06828817958012223, + "policy_error": null, + "return": 3507.209826338786, + "seed": 307, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 192.73070516626822, + "x_velocity": 6.038759707036823 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 3507.209826338786, + "max_steps": 1000, + "mean_action_latency_ms": 0.0678780754096806, + "mean_control_cost": 345.84742993861437, + "mean_forward_reward": 3844.099668375071, + "mean_return": 3498.2522384364556, + "mean_x_position": 192.21294653722586, + "min_return": 3489.2946505341256, + "policy_path": "submitted_artifact", + "score": 3498.2522384364556 + }, + "hidden_nominal_weight": 0.7, + "hidden_robustness": { + "episodes": [ + { + "control_cost": 345.874118052423, + "forward_reward": 3808.7975607576186, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.06775542348623276, + "policy_error": null, + "return": 3462.9234427051965, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 190.42219966048805, + "x_velocity": 1.4021610504933653 + }, + { + "control_cost": 367.54270024597645, + "forward_reward": 3178.5489849598293, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.06758225383237004, + "policy_error": null, + "return": 2811.006284713854, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 158.90977087059895, + "x_velocity": 3.0848840894137 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 3462.9234427051965, + "max_steps": 1000, + "mean_action_latency_ms": 0.0676688386593014, + "mean_control_cost": 356.7084091491997, + "mean_forward_reward": 3493.673272858724, + "mean_return": 3136.964863709525, + "mean_x_position": 174.6659852655435, + "min_return": 2811.006284713854, + "policy_path": "submitted_artifact", + "score": 3136.964863709525 + }, + "hidden_robustness_weight": 0.3, + "normalized_score": -8.167582379, + "reference_score": 11487.256451471967, + "score": 3389.8660260183765, + "starter_score": 4001.2886158917113 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/metadata.json new file mode 100644 index 0000000..a046715 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:33650c821735e31eea00f2ba46ae09b5d19134820c2ec005e8e96d92cb0b66d0", + "digest_compatibility_note": null, + "finished_at": 1783935048.629668, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "task": "simulation_heuristics_montezuma_v1", + "task_digest": "sha256:33650c821735e31eea00f2ba46ae09b5d19134820c2ec005e8e96d92cb0b66d0", + "token_usage": { + "final_input_tokens": 542.0, + "final_output_tokens": 161.0, + "input_tokens": 542.0, + "output_tokens": 161.0, + "total_tokens": 12991.0 + }, + "tool_calls": 15 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/score.json new file mode 100644 index 0000000..3677853 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/score.json @@ -0,0 +1,216 @@ +{ + "normalized_score": 0.0, + "reference_score": 400.0, + "score": 0.0, + "starter_score": 0.0, + "suites": { + "hidden_seeds": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.15504447456735831, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10007, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + }, + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.1614375047099132, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10103, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.15824098963863575, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.1 + }, + "noop_variants": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.1589538881674977, + "policy_error": null, + "policy_steps": 2597, + "pre_policy_noops": 3, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "startup_noop_3" + }, + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.2664268070095333, + "policy_error": null, + "policy_steps": 2587, + "pre_policy_noops": 13, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "startup_noop_13" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.21269034758851552, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.1 + }, + "recovery": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 512, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.1545967847008737, + "policy_error": null, + "policy_steps": 2084, + "pre_policy_noops": 4, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_512_noop_4" + }, + { + "bootstrap_steps": 512, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.1483132426913541, + "policy_error": null, + "policy_steps": 2080, + "pre_policy_noops": 8, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_512_noop_8" + }, + { + "bootstrap_steps": 768, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.34355820039582385, + "policy_error": null, + "policy_steps": 1830, + "pre_policy_noops": 2, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_768_noop_2" + }, + { + "bootstrap_steps": 900, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.2770369280787075, + "policy_error": null, + "policy_steps": 1700, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_900" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.2308762889666898, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.55 + }, + "reproduction": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.15375497965858534, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.15375497965858534, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.25 + } + }, + "target_score": 400.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/metadata.json new file mode 100644 index 0000000..d7ba401 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:f0b6f90f1d8072e75f0e10f90f4b51e51e31b3590bb2c2736778ba155b23389f", + "digest_compatibility_note": null, + "finished_at": 1783935048.629668, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": -75.0, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T091939Z/gpt-5.4-mini", + "task": "simulation_heuristics_pong_ram_v1", + "task_digest": "sha256:f0b6f90f1d8072e75f0e10f90f4b51e51e31b3590bb2c2736778ba155b23389f", + "token_usage": { + "final_input_tokens": 568.0, + "final_output_tokens": 241.0, + "input_tokens": 568.0, + "output_tokens": 241.0, + "total_tokens": 14121.0 + }, + "tool_calls": 17 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/score.json new file mode 100644 index 0000000..bd2984d --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/score.json @@ -0,0 +1,133 @@ +{ + "normalized_score": -75.0, + "reference_score": 21.0, + "score": -21.0, + "starter_score": -2.9999999999999996, + "suite_weights": { + "hidden_nominal": 0.7, + "hidden_reset_robustness": 0.3 + }, + "suites": { + "hidden_nominal": { + "episodes": [ + { + "invalid_action": false, + "length": 3055, + "mean_action_latency_ms": 0.009747649163446724, + "points_against": 21, + "points_for": 0, + "policy_error": null, + "score": -21.0, + "seed": 101, + "target_reached": false, + "terminated": true, + "truncated": false, + "variant": "article_nominal", + "won": false + }, + { + "invalid_action": false, + "length": 3055, + "mean_action_latency_ms": 0.009773229286928224, + "points_against": 21, + "points_for": 0, + "policy_error": null, + "score": -21.0, + "seed": 211, + "target_reached": false, + "terminated": true, + "truncated": false, + "variant": "article_nominal", + "won": false + }, + { + "invalid_action": false, + "length": 3055, + "mean_action_latency_ms": 0.010469134549462463, + "points_against": 21, + "points_for": 0, + "policy_error": null, + "score": -21.0, + "seed": 307, + "target_reached": false, + "terminated": true, + "truncated": false, + "variant": "article_nominal", + "won": false + } + ], + "invalid_episode_rate": 0.0, + "max_score": -21.0, + "max_steps": 27000, + "mean_action_latency_ms": 0.009996670999945804, + "mean_score": -21.0, + "min_score": -21.0, + "policy_path": "submitted_artifact", + "score": -21.0, + "target_score": 21.0, + "target_score_rate": 0.0, + "win_rate": 0.0 + }, + "hidden_reset_robustness": { + "episodes": [ + { + "invalid_action": false, + "length": 3055, + "mean_action_latency_ms": 0.010082433564182975, + "points_against": 21, + "points_for": 0, + "policy_error": null, + "score": -21.0, + "seed": 503, + "target_reached": false, + "terminated": true, + "truncated": false, + "variant": "randomized_reset", + "won": false + }, + { + "invalid_action": false, + "length": 3051, + "mean_action_latency_ms": 0.012134256769949708, + "points_against": 21, + "points_for": 0, + "policy_error": null, + "score": -21.0, + "seed": 601, + "target_reached": false, + "terminated": true, + "truncated": false, + "variant": "randomized_reset", + "won": false + }, + { + "invalid_action": false, + "length": 3052, + "mean_action_latency_ms": 0.011448162823895951, + "points_against": 21, + "points_for": 0, + "policy_error": null, + "score": -21.0, + "seed": 701, + "target_reached": false, + "terminated": true, + "truncated": false, + "variant": "randomized_reset", + "won": false + } + ], + "invalid_episode_rate": 0.0, + "max_score": -21.0, + "max_steps": 27000, + "mean_action_latency_ms": 0.011221617719342878, + "mean_score": -21.0, + "min_score": -21.0, + "policy_path": "submitted_artifact", + "score": -21.0, + "target_score": 21.0, + "target_score_rate": 0.0, + "win_rate": 0.0 + } + }, + "target_score": 21.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/metadata.json new file mode 100644 index 0000000..a34d1f2 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:b3431b238bec4e66af6189d30bcd15d5cd227144dfe8bf0dd1011aa5416c1436", + "digest_compatibility_note": null, + "finished_at": 1783938535.150049, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": -161.116433137, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T102707Z/gpt-5.4-mini", + "task": "simulation_heuristics_vizdoom_d1_v1", + "task_digest": "sha256:b3431b238bec4e66af6189d30bcd15d5cd227144dfe8bf0dd1011aa5416c1436", + "token_usage": { + "final_input_tokens": 1327.0, + "final_output_tokens": 193.0, + "input_tokens": 1327.0, + "output_tokens": 193.0, + "total_tokens": 16368.0 + }, + "tool_calls": 14 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/score.json new file mode 100644 index 0000000..d960adc --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/score.json @@ -0,0 +1,211 @@ +{ + "anchor_calibration": { + "envpool_version": "1.1.1", + "mode": "numeric", + "platform": "linux-x86_64" + }, + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": true + }, + "normalized_score": -161.116433137, + "reference_score": 0.8178199748694897, + "resource_cleanup": { + "removed_bytes": 0, + "removed_resources": 0, + "runs": [ + { + "removed_bytes": 0, + "removed_resources": 0, + "terminated_processes": 0 + }, + { + "removed_bytes": 0, + "removed_resources": 0, + "terminated_processes": 0 + } + ], + "terminated_processes": 0 + }, + "score": -0.16450000800192355, + "starter_score": 0.44161998964846133, + "suites": { + "hidden_nominal": { + "batch_seed": 101, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 0, + "length": 768, + "mean_action_latency_ms": 0.439295722268677, + "policy_error": null, + "return": -0.10400001332163811, + "seed": 101, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 1, + "length": 480, + "mean_action_latency_ms": 0.38578944901625317, + "policy_error": null, + "return": -0.2160000056028366, + "seed": 102, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 2, + "length": 576, + "mean_action_latency_ms": 0.3996389051381912, + "policy_error": null, + "return": -0.1680000089108944, + "seed": 103, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 5.0 + }, + "invalid_action": false, + "lane": 3, + "length": 800, + "mean_action_latency_ms": 0.3952940576709807, + "policy_error": null, + "return": -0.08300001174211502, + "seed": 104, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 4, + "length": 448, + "mean_action_latency_ms": 0.3939860575233719, + "policy_error": null, + "return": -0.23200000450015068, + "seed": 105, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 1, + "invalid_episode_rate": 0.0, + "max_return": -0.08300001174211502, + "max_steps": 2100, + "mean_action_latency_ms": 0.4028008383234948, + "mean_return": -0.16060000881552697, + "min_return": -0.23200000450015068, + "policy_path": "submitted_artifact", + "render_height": 180, + "render_width": 240, + "scenario": "d1", + "score": -0.16060000881552697 + }, + "hidden_secondary": { + "batch_seed": 607, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 0, + "length": 576, + "mean_action_latency_ms": 0.42993988285565543, + "policy_error": null, + "return": -0.20000000670552254, + "seed": 607, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 1, + "length": 512, + "mean_action_latency_ms": 0.39957215631147847, + "policy_error": null, + "return": -0.2160000056028366, + "seed": 608, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 8.0 + }, + "invalid_action": false, + "lane": 2, + "length": 768, + "mean_action_latency_ms": 0.4345782654127106, + "policy_error": null, + "return": -0.052000004798173904, + "seed": 609, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 3, + "length": 480, + "mean_action_latency_ms": 0.3747861715964973, + "policy_error": null, + "return": -0.2160000056028366, + "seed": 610, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 4, + "length": 544, + "mean_action_latency_ms": 0.39103973026880445, + "policy_error": null, + "return": -0.18400000780820847, + "seed": 611, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 1, + "invalid_episode_rate": 0.0, + "max_return": -0.052000004798173904, + "max_steps": 2100, + "mean_action_latency_ms": 0.4059832412890293, + "mean_return": -0.17360000610351561, + "min_return": -0.2160000056028366, + "policy_path": "submitted_artifact", + "render_height": 180, + "render_width": 240, + "scenario": "d1", + "score": -0.17360000610351561 + } + } +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/metadata.json new file mode 100644 index 0000000..5cb7fd2 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:fcc6ed05673b7981aa17d60ca4e0d355fbcb57a6cbca293c0fa201ab97cdd081", + "digest_compatibility_note": null, + "finished_at": 1783939368.238364, + "harness": "opencode", + "model": { + "display_name": "GPT-5.4 Mini", + "id": "gpt-5.4-mini", + "model": "azure/gpt-5.4-mini", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": -9.859625668, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T104034Z/gpt-5.4-mini", + "task": "simulation_heuristics_vizdoom_d3_v1", + "task_digest": "sha256:fcc6ed05673b7981aa17d60ca4e0d355fbcb57a6cbca293c0fa201ab97cdd081", + "token_usage": { + "final_input_tokens": 610.0, + "final_output_tokens": 258.0, + "input_tokens": 610.0, + "output_tokens": 258.0, + "total_tokens": 13668.0 + }, + "tool_calls": 11 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/score.json new file mode 100644 index 0000000..c0dfc03 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/score.json @@ -0,0 +1,229 @@ +{ + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": true + }, + "normalized_score": -9.859625668, + "reference_score": 331.1, + "score": 2.4, + "starter_score": 31.9, + "suites": { + "hidden_nominal": { + "batch_seed": 101, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "AMMO2": 16.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 23.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 0, + "length": 414, + "mean_action_latency_ms": 5.564870929192518, + "policy_error": null, + "return": 0.0, + "seed": 101, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 36.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 1.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 1, + "length": 519, + "mean_action_latency_ms": 5.470503561256134, + "policy_error": null, + "return": 0.0, + "seed": 102, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 28.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 7.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 2, + "length": 236, + "mean_action_latency_ms": 5.521233540849161, + "policy_error": null, + "return": 0.0, + "seed": 103, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 14.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 1.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 3, + "length": 207, + "mean_action_latency_ms": 5.482445196541035, + "policy_error": null, + "return": 0.0, + "seed": 104, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 16.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 13.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 4, + "length": 166, + "mean_action_latency_ms": 5.500560115843293, + "policy_error": null, + "return": 0.0, + "seed": 105, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 2, + "invalid_episode_rate": 0.0, + "max_return": 0.0, + "max_steps": 1050, + "mean_action_latency_ms": 5.507922668736429, + "mean_return": 0.0, + "min_return": 0.0, + "policy_path": "submitted_artifact", + "render_height": 480, + "render_width": 640, + "scenario": "d3", + "score": 0.0 + }, + "hidden_secondary": { + "batch_seed": 607, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "AMMO2": 29.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 13.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 0, + "length": 88, + "mean_action_latency_ms": 5.623010484586385, + "policy_error": null, + "return": 0.0, + "seed": 607, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 20.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 100.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 1, + "length": 1050, + "mean_action_latency_ms": 5.629530707817702, + "policy_error": null, + "return": 0.0, + "seed": 608, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "AMMO2": 27.0, + "DAMAGECOUNT": 20.0, + "HEALTH": 4.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 + }, + "invalid_action": false, + "lane": 2, + "length": 762, + "mean_action_latency_ms": 5.546016788496355, + "policy_error": null, + "return": 40.0, + "seed": 609, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 15.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 12.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 3, + "length": 493, + "mean_action_latency_ms": 5.572860722352598, + "policy_error": null, + "return": 0.0, + "seed": 610, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 30.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 13.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 4, + "length": 224, + "mean_action_latency_ms": 5.576010119901704, + "policy_error": null, + "return": 0.0, + "seed": 611, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 2, + "invalid_episode_rate": 0.0, + "max_return": 40.0, + "max_steps": 1050, + "mean_action_latency_ms": 5.589485764630949, + "mean_return": 8.0, + "min_return": 0.0, + "policy_path": "submitted_artifact", + "render_height": 480, + "render_width": 640, + "scenario": "d3", + "score": 8.0 + } + } +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/metadata.json new file mode 100644 index 0000000..d79bc52 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:9da0e00147cf66804e6c2fc17869606bea8f260850c5447989a8880eef940d45", + "digest_compatibility_note": "Score-equivalent to the current task. The only later change adds a fail-closed internal timeout for candidates whose verifier would otherwise exceed BenchFlow's deadline.", + "finished_at": 1783939074.870018, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": -16.891796576, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T095807Z/gpt-5.5", + "task": "simulation_heuristics_ant_v1", + "task_digest": "sha256:bbb533da0cb86459f4d49dee667e6c73ac54c0188bc40e54e911d50ef3c3bc38", + "token_usage": { + "final_input_tokens": 1098.0, + "final_output_tokens": 223.0, + "input_tokens": 1098.0, + "output_tokens": 223.0, + "total_tokens": 31017.0 + }, + "tool_calls": 32 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/score.json new file mode 100644 index 0000000..d4f8a15 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/score.json @@ -0,0 +1,141 @@ +{ + "hidden_nominal": { + "episodes": [ + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.028467876109061763, + "policy_error": null, + "return": 1960.7456949187047, + "seed": 101, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 53.32761124336437, + "x_velocity": 0.9206927285512734 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.028923347155796364, + "policy_error": null, + "return": 877.2195257753193, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": -0.9485613953689006, + "x_velocity": -1.8377400222706064 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.029911679826909676, + "policy_error": null, + "return": 2424.075753957914, + "seed": 307, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 76.49771178866004, + "x_velocity": 1.573669182193953 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.031103011569939554, + "policy_error": null, + "return": 1210.9204369966183, + "seed": 401, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 15.788498223386553, + "x_velocity": -1.7502006181826246 + } + ], + "fall_rate": 0.0, + "invalid_episode_rate": 0.0, + "max_return": 2424.075753957914, + "max_steps": 1000, + "mean_action_latency_ms": 0.02960147866542684, + "mean_return": 1618.240352912139, + "mean_x_position": 36.166314965010514, + "min_return": 877.2195257753193, + "policy_path": "submitted_artifact", + "score": 1618.240352912139 + }, + "hidden_nominal_weight": 0.7, + "hidden_robustness": { + "episodes": [ + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.03534062029211782, + "policy_error": null, + "return": 2816.5356939643866, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 96.03292689641613, + "x_velocity": 1.711305903083371 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.028797698294511065, + "policy_error": null, + "return": 2494.2487539071335, + "seed": 601, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 80.00959841427752, + "x_velocity": 2.2284234213796594 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.02896833320846781, + "policy_error": null, + "return": 2413.6787614355176, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 76.42935346107426, + "x_velocity": 1.1236195076369881 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.029127599817002192, + "policy_error": null, + "return": 1876.672105142364, + "seed": 601, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 49.60350134337408, + "x_velocity": 1.681621357020333 + } + ], + "fall_rate": 0.0, + "invalid_episode_rate": 0.0, + "max_return": 2816.5356939643866, + "max_steps": 1000, + "mean_action_latency_ms": 0.030558562903024722, + "mean_return": 2400.2838286123506, + "mean_x_position": 75.51884502878549, + "min_return": 1876.672105142364, + "policy_path": "submitted_artifact", + "score": 2400.2838286123506 + }, + "hidden_robustness_weight": 0.3, + "normalized_score": -16.891796576, + "reference_score": 5942.333244755838, + "score": 1852.8533956222025, + "starter_score": 2443.815838992282, + "unique_policy_evaluations": 3 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/metadata.json new file mode 100644 index 0000000..36d1330 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:a3ec10ab61a65c03ca17e6460147a3178e222bd1c48964727ea9daae54717ebe", + "digest_compatibility_note": null, + "finished_at": 1783941481.68291, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T111604Z/gpt-5.5", + "task": "simulation_heuristics_atari57_v1", + "task_digest": "sha256:a3ec10ab61a65c03ca17e6460147a3178e222bd1c48964727ea9daae54717ebe", + "token_usage": { + "final_input_tokens": 5376.0, + "final_output_tokens": 324.0, + "input_tokens": 5376.0, + "output_tokens": 324.0, + "total_tokens": 19012.0 + }, + "tool_calls": 13 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/score.json new file mode 100644 index 0000000..7788108 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/score.json @@ -0,0 +1,15 @@ +{ + "disqualification_reason": "Official suite requires 342 completed search-ledger records; found 0.", + "evaluation": null, + "interaction_budget": { + "completed_trajectories": 0, + "counted_env_steps": 0, + "planned_trajectories": 342, + "target_env_steps": 6840000000 + }, + "normalized_score": 0.0, + "protocol_complete": false, + "reference_score": 0.8283015254994576, + "score": 0.0, + "starter_score": 0.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/metadata.json new file mode 100644 index 0000000..9c46b09 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:77215337f89f950f83b8f7436f799959c2ddccd4102552c6302b13d30cc57199", + "digest_compatibility_note": null, + "finished_at": 1783933771.613736, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 95.597484277, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "task": "simulation_heuristics_breakout_ram_v1", + "task_digest": "sha256:77215337f89f950f83b8f7436f799959c2ddccd4102552c6302b13d30cc57199", + "token_usage": { + "final_input_tokens": 2395.0, + "final_output_tokens": 237.0, + "input_tokens": 2395.0, + "output_tokens": 237.0, + "total_tokens": 21064.0 + }, + "tool_calls": 26 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/score.json new file mode 100644 index 0000000..7867301 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/score.json @@ -0,0 +1,80 @@ +{ + "normalized_score": 95.597484277, + "reference_score": 864.0, + "score": 843.0, + "starter_score": 387.0, + "suites": { + "hidden_nominal": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 2, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.004359432254676466, + "observation_mode": "ram", + "policy_error": null, + "return": 843.0, + "seed": 101, + "terminated": false, + "truncated": true, + "variant": "nominal", + "warmup_steps": 0 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 843.0, + "max_steps": 108000, + "mean_action_latency_ms": 0.004359432254676466, + "mean_return": 843.0, + "min_return": 843.0, + "observation_mode": "ram", + "policy_path": "submitted_artifact", + "score": 843.0 + }, + "hidden_shifted_start": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 2, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.004456932001091816, + "observation_mode": "ram", + "policy_error": null, + "return": 843.0, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "left_6", + "warmup_steps": 6 + }, + { + "final_lives": 2, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.00464326129467399, + "observation_mode": "ram", + "policy_error": null, + "return": 843.0, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "right_6", + "warmup_steps": 6 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 843.0, + "max_steps": 108000, + "mean_action_latency_ms": 0.004550096647882904, + "mean_return": 843.0, + "min_return": 843.0, + "observation_mode": "ram", + "policy_path": "submitted_artifact", + "score": 843.0 + } + }, + "target_gap": -21.0, + "target_score": 864.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/metadata.json new file mode 100644 index 0000000..44a7093 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:7e126380af5345a4bfa0c9d29e45e68b8f1aa20d5d655d12187744100b24441a", + "digest_compatibility_note": null, + "finished_at": 1783933771.613736, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 70.758122744, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "task": "simulation_heuristics_breakout_rgb_v1", + "task_digest": "sha256:7e126380af5345a4bfa0c9d29e45e68b8f1aa20d5d655d12187744100b24441a", + "token_usage": { + "final_input_tokens": 580.0, + "final_output_tokens": 197.0, + "input_tokens": 580.0, + "output_tokens": 197.0, + "total_tokens": 19209.0 + }, + "tool_calls": 25 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/score.json new file mode 100644 index 0000000..31e15c4 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/score.json @@ -0,0 +1,66 @@ +{ + "normalized_score": 70.758122744, + "reference_score": 864.0, + "score": 702.0, + "starter_score": 310.0, + "suites": { + "hidden_nominal": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 0, + "invalid_action": false, + "length": 18811, + "mean_action_latency_ms": 0.4204832990456023, + "observation_mode": "rgb", + "policy_error": null, + "return": 702.0, + "seed": 101, + "terminated": true, + "truncated": false, + "variant": "nominal", + "warmup_steps": 0 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 702.0, + "max_steps": 30000, + "mean_action_latency_ms": 0.4204832990456023, + "mean_return": 702.0, + "min_return": 702.0, + "observation_mode": "rgb", + "policy_path": "submitted_artifact", + "score": 702.0 + }, + "hidden_shifted_start": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 0, + "invalid_action": false, + "length": 18805, + "mean_action_latency_ms": 0.42227563768678744, + "observation_mode": "rgb", + "policy_error": null, + "return": 702.0, + "seed": 211, + "terminated": true, + "truncated": false, + "variant": "right_6", + "warmup_steps": 6 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 702.0, + "max_steps": 30000, + "mean_action_latency_ms": 0.42227563768678744, + "mean_return": 702.0, + "min_return": 702.0, + "observation_mode": "rgb", + "policy_path": "submitted_artifact", + "score": 702.0 + } + }, + "target_gap": -162.0, + "target_score": 864.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/metadata.json new file mode 100644 index 0000000..632f109 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:80c439f53e4ab964f9d7443cd7fb8f25cf6645a0bc288b0496b871c1800ebe78", + "digest_compatibility_note": null, + "finished_at": 1783941357.9386308, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 2.73755349, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T103810Z/gpt-5.5", + "task": "simulation_heuristics_halfcheetah_v1", + "task_digest": "sha256:80c439f53e4ab964f9d7443cd7fb8f25cf6645a0bc288b0496b871c1800ebe78", + "token_usage": { + "final_input_tokens": 1233.0, + "final_output_tokens": 233.0, + "input_tokens": 1233.0, + "output_tokens": 233.0, + "total_tokens": 18362.0 + }, + "tool_calls": 21 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/score.json new file mode 100644 index 0000000..36ef619 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/score.json @@ -0,0 +1,98 @@ +{ + "hidden_nominal": { + "episodes": [ + { + "control_cost": 330.4261677414179, + "forward_reward": 4605.109353712759, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.11520281340926886, + "policy_error": null, + "return": 4274.683185971341, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 230.2448098566455, + "x_velocity": 4.320877724728021 + }, + { + "control_cost": 331.2794958502054, + "forward_reward": 4739.932284020166, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.11275607440620661, + "policy_error": null, + "return": 4408.652788169957, + "seed": 307, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 237.02319826694526, + "x_velocity": 4.455080651779344 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 4408.652788169957, + "max_steps": 1000, + "mean_action_latency_ms": 0.11397944390773773, + "mean_control_cost": 330.85283179581165, + "mean_forward_reward": 4672.520818866462, + "mean_return": 4341.667987070648, + "mean_x_position": 233.6340040617954, + "min_return": 4274.683185971341, + "policy_path": "submitted_artifact", + "score": 4341.667987070648 + }, + "hidden_nominal_weight": 0.7, + "hidden_robustness": { + "episodes": [ + { + "control_cost": 328.19089092314243, + "forward_reward": 4836.63682109343, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.11922981264069676, + "policy_error": null, + "return": 4508.4459301702855, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 241.81416267727866, + "x_velocity": 4.403407883231224 + }, + { + "control_cost": 331.44420075416565, + "forward_reward": 3603.354261605028, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 0.11692689219489694, + "policy_error": null, + "return": 3271.9100608508643, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 180.15003470285865, + "x_velocity": 3.3799933403413216 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 4508.4459301702855, + "max_steps": 1000, + "mean_action_latency_ms": 0.11807835241779685, + "mean_control_cost": 329.81754583865404, + "mean_forward_reward": 4219.995541349229, + "mean_return": 3890.177995510575, + "mean_x_position": 210.98209869006865, + "min_return": 3271.9100608508643, + "policy_path": "submitted_artifact", + "score": 3890.177995510575 + }, + "hidden_robustness_weight": 0.3, + "normalized_score": 2.73755349, + "reference_score": 11487.256451471967, + "score": 4206.220989602626, + "starter_score": 4001.2886158917113 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/metadata.json new file mode 100644 index 0000000..b30e569 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:33650c821735e31eea00f2ba46ae09b5d19134820c2ec005e8e96d92cb0b66d0", + "digest_compatibility_note": null, + "finished_at": 1783933771.613736, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 100.0, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "task": "simulation_heuristics_montezuma_v1", + "task_digest": "sha256:33650c821735e31eea00f2ba46ae09b5d19134820c2ec005e8e96d92cb0b66d0", + "token_usage": { + "final_input_tokens": 2141.0, + "final_output_tokens": 318.0, + "input_tokens": 2141.0, + "output_tokens": 318.0, + "total_tokens": 43419.0 + }, + "tool_calls": 60 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/score.json new file mode 100644 index 0000000..9fda219 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/score.json @@ -0,0 +1,216 @@ +{ + "normalized_score": 100.0, + "reference_score": 400.0, + "score": 400.0, + "starter_score": 0.0, + "suites": { + "hidden_seeds": { + "evaluation": { + "capped_mean_score": 400.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 1769, + "mean_action_latency_ms": 0.003403249586821276, + "policy_error": null, + "policy_steps": 1769, + "pre_policy_noops": 0, + "return": 400.0, + "seed": 10007, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "canonical" + }, + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 1769, + "mean_action_latency_ms": 0.003197897434099843, + "policy_error": null, + "policy_steps": 1769, + "pre_policy_noops": 0, + "return": 400.0, + "seed": 10103, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "canonical" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.0033005735104605595, + "mean_return": 400.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 400.0, + "target_score": 400.0, + "target_success_rate": 1.0 + }, + "score": 400.0, + "weight": 0.1 + }, + "noop_variants": { + "evaluation": { + "capped_mean_score": 400.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 1773, + "mean_action_latency_ms": 0.003588716315347596, + "policy_error": null, + "policy_steps": 1770, + "pre_policy_noops": 3, + "return": 400.0, + "seed": 10001, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "startup_noop_3" + }, + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 1769, + "mean_action_latency_ms": 0.0032576125598849904, + "policy_error": null, + "policy_steps": 1756, + "pre_policy_noops": 13, + "return": 400.0, + "seed": 10001, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "startup_noop_13" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.0034231644376162933, + "mean_return": 400.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 1.0, + "score": 400.0, + "target_score": 400.0, + "target_success_rate": 1.0 + }, + "score": 400.0, + "weight": 0.1 + }, + "recovery": { + "evaluation": { + "capped_mean_score": 400.0, + "episodes": [ + { + "bootstrap_steps": 512, + "invalid_action": false, + "length": 1773, + "mean_action_latency_ms": 0.0038068125123719705, + "policy_error": null, + "policy_steps": 1257, + "pre_policy_noops": 4, + "return": 400.0, + "seed": 10001, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "recover_512_noop_4" + }, + { + "bootstrap_steps": 512, + "invalid_action": false, + "length": 1769, + "mean_action_latency_ms": 0.0031122399435890877, + "policy_error": null, + "policy_steps": 1249, + "pre_policy_noops": 8, + "return": 400.0, + "seed": 10001, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "recover_512_noop_8" + }, + { + "bootstrap_steps": 768, + "invalid_action": false, + "length": 1773, + "mean_action_latency_ms": 0.005221254343644215, + "policy_error": null, + "policy_steps": 1003, + "pre_policy_noops": 2, + "return": 400.0, + "seed": 10001, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "recover_768_noop_2" + }, + { + "bootstrap_steps": 900, + "invalid_action": false, + "length": 2173, + "mean_action_latency_ms": 0.002957832211597829, + "policy_error": null, + "policy_steps": 1273, + "pre_policy_noops": 0, + "return": 400.0, + "seed": 10001, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "recover_900" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.003774534752800776, + "mean_return": 400.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 1.0, + "score": 400.0, + "target_score": 400.0, + "target_success_rate": 1.0 + }, + "score": 400.0, + "weight": 0.55 + }, + "reproduction": { + "evaluation": { + "capped_mean_score": 400.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 1769, + "mean_action_latency_ms": 0.0035391858299281247, + "policy_error": null, + "policy_steps": 1769, + "pre_policy_noops": 0, + "return": 400.0, + "seed": 10001, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "canonical" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.0035391858299281247, + "mean_return": 400.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 400.0, + "target_score": 400.0, + "target_success_rate": 1.0 + }, + "score": 400.0, + "weight": 0.25 + } + }, + "target_score": 400.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/metadata.json new file mode 100644 index 0000000..1ee464b --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:f0b6f90f1d8072e75f0e10f90f4b51e51e31b3590bb2c2736778ba155b23389f", + "digest_compatibility_note": null, + "finished_at": 1783933771.613736, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 45.833333333, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "task": "simulation_heuristics_pong_ram_v1", + "task_digest": "sha256:f0b6f90f1d8072e75f0e10f90f4b51e51e31b3590bb2c2736778ba155b23389f", + "token_usage": { + "final_input_tokens": 739.0, + "final_output_tokens": 248.0, + "input_tokens": 739.0, + "output_tokens": 248.0, + "total_tokens": 18907.0 + }, + "tool_calls": 31 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/score.json new file mode 100644 index 0000000..4135a2c --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/score.json @@ -0,0 +1,133 @@ +{ + "normalized_score": 45.833333333, + "reference_score": 21.0, + "score": 8.0, + "starter_score": -2.9999999999999996, + "suite_weights": { + "hidden_nominal": 0.7, + "hidden_reset_robustness": 0.3 + }, + "suites": { + "hidden_nominal": { + "episodes": [ + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.003850699022964195, + "points_against": 1, + "points_for": 9, + "policy_error": null, + "score": 8.0, + "seed": 101, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "article_nominal", + "won": true + }, + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.0040996284396560105, + "points_against": 1, + "points_for": 9, + "policy_error": null, + "score": 8.0, + "seed": 211, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "article_nominal", + "won": true + }, + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.004162275025414096, + "points_against": 1, + "points_for": 9, + "policy_error": null, + "score": 8.0, + "seed": 307, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "article_nominal", + "won": true + } + ], + "invalid_episode_rate": 0.0, + "max_score": 8.0, + "max_steps": 27000, + "mean_action_latency_ms": 0.004037534162678101, + "mean_score": 8.0, + "min_score": 8.0, + "policy_path": "submitted_artifact", + "score": 8.0, + "target_score": 21.0, + "target_score_rate": 0.0, + "win_rate": 1.0 + }, + "hidden_reset_robustness": { + "episodes": [ + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.003967505638246183, + "points_against": 1, + "points_for": 9, + "policy_error": null, + "score": 8.0, + "seed": 503, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "randomized_reset", + "won": true + }, + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.0037646840705915733, + "points_against": 1, + "points_for": 9, + "policy_error": null, + "score": 8.0, + "seed": 601, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "randomized_reset", + "won": true + }, + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.0034993514073667705, + "points_against": 1, + "points_for": 9, + "policy_error": null, + "score": 8.0, + "seed": 701, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "randomized_reset", + "won": true + } + ], + "invalid_episode_rate": 0.0, + "max_score": 8.0, + "max_steps": 27000, + "mean_action_latency_ms": 0.003743847038734842, + "mean_score": 8.0, + "min_score": 8.0, + "policy_path": "submitted_artifact", + "score": 8.0, + "target_score": 21.0, + "target_score_rate": 0.0, + "win_rate": 1.0 + } + }, + "target_score": 21.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/metadata.json new file mode 100644 index 0000000..d436810 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:b3431b238bec4e66af6189d30bcd15d5cd227144dfe8bf0dd1011aa5416c1436", + "digest_compatibility_note": null, + "finished_at": 1783938341.5167592, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 85.199359782, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T102218Z/gpt-5.5", + "task": "simulation_heuristics_vizdoom_d1_v1", + "task_digest": "sha256:b3431b238bec4e66af6189d30bcd15d5cd227144dfe8bf0dd1011aa5416c1436", + "token_usage": { + "final_input_tokens": 1575.0, + "final_output_tokens": 196.0, + "input_tokens": 1575.0, + "output_tokens": 196.0, + "total_tokens": 22251.0 + }, + "tool_calls": 25 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/score.json new file mode 100644 index 0000000..abc6f9f --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/score.json @@ -0,0 +1,211 @@ +{ + "anchor_calibration": { + "envpool_version": "1.1.1", + "mode": "numeric", + "platform": "linux-x86_64" + }, + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": true + }, + "normalized_score": 85.199359782, + "reference_score": 0.8178199748694897, + "resource_cleanup": { + "removed_bytes": 0, + "removed_resources": 0, + "runs": [ + { + "removed_bytes": 0, + "removed_resources": 0, + "terminated_processes": 0 + }, + { + "removed_bytes": 0, + "removed_resources": 0, + "terminated_processes": 0 + } + ], + "terminated_processes": 0 + }, + "score": 0.7621399685554205, + "starter_score": 0.44161998964846133, + "suites": { + "hidden_nominal": { + "batch_seed": 101, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "HEALTH": 37.0 + }, + "invalid_action": false, + "lane": 0, + "length": 2100, + "mean_action_latency_ms": 0.12978367596155121, + "policy_error": null, + "return": 0.7009999677538872, + "seed": 101, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 52.0 + }, + "invalid_action": false, + "lane": 1, + "length": 2100, + "mean_action_latency_ms": 0.12354861146637372, + "policy_error": null, + "return": 0.7919999733567238, + "seed": 102, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 52.0 + }, + "invalid_action": false, + "lane": 2, + "length": 2100, + "mean_action_latency_ms": 0.12090237012931279, + "policy_error": null, + "return": 0.8079999694600701, + "seed": 103, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 100.0 + }, + "invalid_action": false, + "lane": 3, + "length": 2100, + "mean_action_latency_ms": 0.12010271705332257, + "policy_error": null, + "return": 0.9839999675750732, + "seed": 104, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 60.0 + }, + "invalid_action": false, + "lane": 4, + "length": 2100, + "mean_action_latency_ms": 0.1198997339677243, + "policy_error": null, + "return": 0.8319999575614929, + "seed": 105, + "terminated": false, + "truncated": true + } + ], + "frame_skip": 1, + "invalid_episode_rate": 0.0, + "max_return": 0.9839999675750732, + "max_steps": 2100, + "mean_action_latency_ms": 0.12284742171565691, + "mean_return": 0.8233999671414495, + "min_return": 0.7009999677538872, + "policy_path": "submitted_artifact", + "render_height": 180, + "render_width": 240, + "scenario": "d1", + "score": 0.8233999671414495 + }, + "hidden_secondary": { + "batch_seed": 607, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "HEALTH": 60.0 + }, + "invalid_action": false, + "lane": 0, + "length": 2100, + "mean_action_latency_ms": 0.13575854578188487, + "policy_error": null, + "return": 0.7359999641776085, + "seed": 607, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 52.0 + }, + "invalid_action": false, + "lane": 1, + "length": 2100, + "mean_action_latency_ms": 0.12331685051321983, + "policy_error": null, + "return": 0.7759999595582485, + "seed": 608, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 44.0 + }, + "invalid_action": false, + "lane": 2, + "length": 2100, + "mean_action_latency_ms": 0.12160280719399452, + "policy_error": null, + "return": 0.7199999764561653, + "seed": 609, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 100.0 + }, + "invalid_action": false, + "lane": 3, + "length": 2100, + "mean_action_latency_ms": 0.11989811878828775, + "policy_error": null, + "return": 1.063999965786934, + "seed": 610, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 4.0 + }, + "invalid_action": false, + "lane": 4, + "length": 512, + "mean_action_latency_ms": 0.11529984476510435, + "policy_error": null, + "return": -0.20000000670552254, + "seed": 611, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 1, + "invalid_episode_rate": 0.0, + "max_return": 1.063999965786934, + "max_steps": 2100, + "mean_action_latency_ms": 0.12317523340849826, + "mean_return": 0.6191999718546868, + "min_return": -0.20000000670552254, + "policy_path": "submitted_artifact", + "render_height": 180, + "render_width": 240, + "scenario": "d1", + "score": 0.6191999718546868 + } + } +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/metadata.json new file mode 100644 index 0000000..ff9a0de --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:fcc6ed05673b7981aa17d60ca4e0d355fbcb57a6cbca293c0fa201ab97cdd081", + "digest_compatibility_note": null, + "finished_at": 1783933771.613736, + "harness": "opencode", + "model": { + "display_name": "GPT-5.5", + "id": "gpt-5.5", + "model": "azure/gpt-5.5", + "provider": "azure", + "provider_reasoning_effort": "xhigh" + }, + "normalized_score": 5.481283422, + "provider_reasoning_effort": "xhigh", + "source_run_id": "leaderboard/runs/article_suite/20260713T084111Z/gpt-5.5", + "task": "simulation_heuristics_vizdoom_d3_v1", + "task_digest": "sha256:fcc6ed05673b7981aa17d60ca4e0d355fbcb57a6cbca293c0fa201ab97cdd081", + "token_usage": { + "final_input_tokens": 1848.0, + "final_output_tokens": 220.0, + "input_tokens": 1848.0, + "output_tokens": 220.0, + "total_tokens": 17940.0 + }, + "tool_calls": 14 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/score.json new file mode 100644 index 0000000..f7ec5f1 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/score.json @@ -0,0 +1,229 @@ +{ + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": true + }, + "normalized_score": 5.481283422, + "reference_score": 331.1, + "score": 48.3, + "starter_score": 31.9, + "suites": { + "hidden_nominal": { + "batch_seed": 101, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "AMMO2": 1.0, + "DAMAGECOUNT": 15.0, + "HEALTH": 19.0, + "HITCOUNT": 1.0, + "KILLCOUNT": 1.0 + }, + "invalid_action": false, + "lane": 0, + "length": 147, + "mean_action_latency_ms": 1.9462588211508836, + "policy_error": null, + "return": 25.0, + "seed": 101, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 20.0, + "HEALTH": 4.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 + }, + "invalid_action": false, + "lane": 1, + "length": 257, + "mean_action_latency_ms": 1.9732051119498242, + "policy_error": null, + "return": 40.0, + "seed": 102, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 2.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 16.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 2, + "length": 130, + "mean_action_latency_ms": 1.8855262404451003, + "policy_error": null, + "return": 0.0, + "seed": 103, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 40.0, + "HEALTH": 7.0, + "HITCOUNT": 4.0, + "KILLCOUNT": 4.0 + }, + "invalid_action": false, + "lane": 3, + "length": 505, + "mean_action_latency_ms": 2.334480810135898, + "policy_error": null, + "return": 80.0, + "seed": 104, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 30.0, + "HEALTH": 12.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 + }, + "invalid_action": false, + "lane": 4, + "length": 379, + "mean_action_latency_ms": 1.9117984675008577, + "policy_error": null, + "return": 50.0, + "seed": 105, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 2, + "invalid_episode_rate": 0.0, + "max_return": 80.0, + "max_steps": 1050, + "mean_action_latency_ms": 2.0102538902365126, + "mean_return": 39.0, + "min_return": 0.0, + "policy_path": "submitted_artifact", + "render_height": 480, + "render_width": 640, + "scenario": "d3", + "score": 39.0 + }, + "hidden_secondary": { + "batch_seed": 607, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 90.0, + "HEALTH": 10.0, + "HITCOUNT": 7.0, + "KILLCOUNT": 7.0 + }, + "invalid_action": false, + "lane": 0, + "length": 328, + "mean_action_latency_ms": 1.9002374625060616, + "policy_error": null, + "return": 160.0, + "seed": 607, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 15.0, + "HEALTH": 10.0, + "HITCOUNT": 1.0, + "KILLCOUNT": 1.0 + }, + "invalid_action": false, + "lane": 1, + "length": 295, + "mean_action_latency_ms": 1.9393673419194706, + "policy_error": null, + "return": 25.0, + "seed": 608, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 10.0, + "HEALTH": 9.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 + }, + "invalid_action": false, + "lane": 2, + "length": 648, + "mean_action_latency_ms": 2.3452048074959975, + "policy_error": null, + "return": 30.0, + "seed": 609, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 16.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 + }, + "invalid_action": false, + "lane": 3, + "length": 415, + "mean_action_latency_ms": 1.9054093229842473, + "policy_error": null, + "return": 0.0, + "seed": 610, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 75.0, + "HEALTH": 19.0, + "HITCOUNT": 6.0, + "KILLCOUNT": 6.0 + }, + "invalid_action": false, + "lane": 4, + "length": 422, + "mean_action_latency_ms": 1.8646283432735369, + "policy_error": null, + "return": 135.0, + "seed": 611, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 2, + "invalid_episode_rate": 0.0, + "max_return": 160.0, + "max_steps": 1050, + "mean_action_latency_ms": 1.990969455635863, + "mean_return": 70.0, + "min_return": 0.0, + "policy_path": "submitted_artifact", + "render_height": 480, + "render_width": 640, + "scenario": "d3", + "score": 70.0 + } + } +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/metadata.json new file mode 100644 index 0000000..aaab68a --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:9da0e00147cf66804e6c2fc17869606bea8f260850c5447989a8880eef940d45", + "digest_compatibility_note": "Score-equivalent to the current task. The only later change adds a fail-closed internal timeout for candidates whose verifier would otherwise exceed BenchFlow's deadline.", + "finished_at": 1783939823.3358562, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 39.203600385, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T093815Z/gpt-5.6-sol", + "task": "simulation_heuristics_ant_v1", + "task_digest": "sha256:bbb533da0cb86459f4d49dee667e6c73ac54c0188bc40e54e911d50ef3c3bc38", + "token_usage": { + "final_input_tokens": 416.0, + "final_output_tokens": 196.0, + "input_tokens": 416.0, + "output_tokens": 196.0, + "total_tokens": 36964.0 + }, + "tool_calls": 37 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/score.json new file mode 100644 index 0000000..3582d4f --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/score.json @@ -0,0 +1,141 @@ +{ + "hidden_nominal": { + "episodes": [ + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 133.12361360079376, + "policy_error": null, + "return": 2944.6798391671878, + "seed": 101, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 112.93341277596396, + "x_velocity": -0.05774537124608514 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 130.92819425201742, + "policy_error": null, + "return": 4189.938284380654, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 176.7363723697072, + "x_velocity": 5.390771978175053 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 133.01312479103217, + "policy_error": null, + "return": 4054.5561540797858, + "seed": 307, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 169.74913616212385, + "x_velocity": 3.097238615771971 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 128.64087908767397, + "policy_error": null, + "return": 4052.711831067477, + "seed": 401, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 169.84904705908204, + "x_velocity": 2.148484367220931 + } + ], + "fall_rate": 0.0, + "invalid_episode_rate": 0.0, + "max_return": 4189.938284380654, + "max_steps": 1000, + "mean_action_latency_ms": 131.42645293287933, + "mean_return": 3810.471527173776, + "mean_x_position": 157.31699209171927, + "min_return": 2944.6798391671878, + "policy_path": "submitted_artifact", + "score": 3810.471527173776 + }, + "hidden_nominal_weight": 0.7, + "hidden_robustness": { + "episodes": [ + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 129.90234417081228, + "policy_error": null, + "return": 4416.461572320053, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 187.29214144677127, + "x_velocity": 3.390203653403887 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 128.21582087071147, + "policy_error": null, + "return": 4370.866265431624, + "seed": 601, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 185.1144372257778, + "x_velocity": 3.11121379282838 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 129.91995375533588, + "policy_error": null, + "return": 3731.5669383240584, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 153.54017346584874, + "x_velocity": 3.9302016156398167 + }, + { + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 131.77032847257215, + "policy_error": null, + "return": 2788.179265576913, + "seed": 601, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 106.42804578295254, + "x_velocity": 2.7356958324236302 + } + ], + "fall_rate": 0.0, + "invalid_episode_rate": 0.0, + "max_return": 4416.461572320053, + "max_steps": 1000, + "mean_action_latency_ms": 129.95211181735795, + "mean_return": 3826.768510413162, + "mean_x_position": 158.0936994803376, + "min_return": 2788.179265576913, + "policy_path": "submitted_artifact", + "score": 3826.768510413162 + }, + "hidden_robustness_weight": 0.3, + "normalized_score": 39.203600385, + "reference_score": 5942.333244755838, + "score": 3815.360622145591, + "starter_score": 2443.815838992282, + "unique_policy_evaluations": 3 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/metadata.json new file mode 100644 index 0000000..acdbc6b --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:a3ec10ab61a65c03ca17e6460147a3178e222bd1c48964727ea9daae54717ebe", + "digest_compatibility_note": null, + "finished_at": 1783941310.238979, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T111156Z/gpt-5.6-sol", + "task": "simulation_heuristics_atari57_v1", + "task_digest": "sha256:a3ec10ab61a65c03ca17e6460147a3178e222bd1c48964727ea9daae54717ebe", + "token_usage": { + "final_input_tokens": 708.0, + "final_output_tokens": 205.0, + "input_tokens": 708.0, + "output_tokens": 205.0, + "total_tokens": 31121.0 + }, + "tool_calls": 26 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/score.json new file mode 100644 index 0000000..7788108 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/score.json @@ -0,0 +1,15 @@ +{ + "disqualification_reason": "Official suite requires 342 completed search-ledger records; found 0.", + "evaluation": null, + "interaction_budget": { + "completed_trajectories": 0, + "counted_env_steps": 0, + "planned_trajectories": 342, + "target_env_steps": 6840000000 + }, + "normalized_score": 0.0, + "protocol_complete": false, + "reference_score": 0.8283015254994576, + "score": 0.0, + "starter_score": 0.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/metadata.json new file mode 100644 index 0000000..7241d05 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:77215337f89f950f83b8f7436f799959c2ddccd4102552c6302b13d30cc57199", + "digest_compatibility_note": null, + "finished_at": 1783931627.734036, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 100.0, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "task": "simulation_heuristics_breakout_ram_v1", + "task_digest": "sha256:77215337f89f950f83b8f7436f799959c2ddccd4102552c6302b13d30cc57199", + "token_usage": { + "final_input_tokens": 536.0, + "final_output_tokens": 151.0, + "input_tokens": 536.0, + "output_tokens": 151.0, + "total_tokens": 34479.0 + }, + "tool_calls": 53 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/score.json new file mode 100644 index 0000000..806d20b --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/score.json @@ -0,0 +1,80 @@ +{ + "normalized_score": 100.0, + "reference_score": 864.0, + "score": 864.0, + "starter_score": 387.0, + "suites": { + "hidden_nominal": { + "completion_rate": 1.0, + "episodes": [ + { + "final_lives": 5, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.021162176155485213, + "observation_mode": "ram", + "policy_error": null, + "return": 864.0, + "seed": 101, + "terminated": false, + "truncated": true, + "variant": "nominal", + "warmup_steps": 0 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 864.0, + "max_steps": 108000, + "mean_action_latency_ms": 0.021162176155485213, + "mean_return": 864.0, + "min_return": 864.0, + "observation_mode": "ram", + "policy_path": "submitted_artifact", + "score": 864.0 + }, + "hidden_shifted_start": { + "completion_rate": 1.0, + "episodes": [ + { + "final_lives": 5, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.022908275248482823, + "observation_mode": "ram", + "policy_error": null, + "return": 864.0, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "left_6", + "warmup_steps": 6 + }, + { + "final_lives": 5, + "invalid_action": false, + "length": 108000, + "mean_action_latency_ms": 0.02272433058048288, + "observation_mode": "ram", + "policy_error": null, + "return": 864.0, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "right_6", + "warmup_steps": 6 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 864.0, + "max_steps": 108000, + "mean_action_latency_ms": 0.022816302914482854, + "mean_return": 864.0, + "min_return": 864.0, + "observation_mode": "ram", + "policy_path": "submitted_artifact", + "score": 864.0 + } + }, + "target_gap": 0.0, + "target_score": 864.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/metadata.json new file mode 100644 index 0000000..80d6c8c --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:7e126380af5345a4bfa0c9d29e45e68b8f1aa20d5d655d12187744100b24441a", + "digest_compatibility_note": null, + "finished_at": 1783931627.734036, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 70.758122744, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "task": "simulation_heuristics_breakout_rgb_v1", + "task_digest": "sha256:7e126380af5345a4bfa0c9d29e45e68b8f1aa20d5d655d12187744100b24441a", + "token_usage": { + "final_input_tokens": 708.0, + "final_output_tokens": 118.0, + "input_tokens": 708.0, + "output_tokens": 118.0, + "total_tokens": 21306.0 + }, + "tool_calls": 33 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/score.json new file mode 100644 index 0000000..51a1a30 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/score.json @@ -0,0 +1,66 @@ +{ + "normalized_score": 70.758122744, + "reference_score": 864.0, + "score": 702.0, + "starter_score": 310.0, + "suites": { + "hidden_nominal": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 0, + "invalid_action": false, + "length": 18811, + "mean_action_latency_ms": 0.48869839894144823, + "observation_mode": "rgb", + "policy_error": null, + "return": 702.0, + "seed": 101, + "terminated": true, + "truncated": false, + "variant": "nominal", + "warmup_steps": 0 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 702.0, + "max_steps": 30000, + "mean_action_latency_ms": 0.48869839894144823, + "mean_return": 702.0, + "min_return": 702.0, + "observation_mode": "rgb", + "policy_path": "submitted_artifact", + "score": 702.0 + }, + "hidden_shifted_start": { + "completion_rate": 0.0, + "episodes": [ + { + "final_lives": 0, + "invalid_action": false, + "length": 18805, + "mean_action_latency_ms": 0.5275690000493359, + "observation_mode": "rgb", + "policy_error": null, + "return": 702.0, + "seed": 211, + "terminated": true, + "truncated": false, + "variant": "right_6", + "warmup_steps": 6 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 702.0, + "max_steps": 30000, + "mean_action_latency_ms": 0.5275690000493359, + "mean_return": 702.0, + "min_return": 702.0, + "observation_mode": "rgb", + "policy_path": "submitted_artifact", + "score": 702.0 + } + }, + "target_gap": -162.0, + "target_score": 864.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/metadata.json new file mode 100644 index 0000000..ee6e268 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:80c439f53e4ab964f9d7443cd7fb8f25cf6645a0bc288b0496b871c1800ebe78", + "digest_compatibility_note": null, + "finished_at": 1783935158.328714, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 15.873072595, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T084340Z/gpt-5.6-sol", + "task": "simulation_heuristics_halfcheetah_v1", + "task_digest": "sha256:80c439f53e4ab964f9d7443cd7fb8f25cf6645a0bc288b0496b871c1800ebe78", + "token_usage": { + "final_input_tokens": 670.0, + "final_output_tokens": 192.0, + "input_tokens": 670.0, + "output_tokens": 192.0, + "total_tokens": 19806.0 + }, + "tool_calls": 28 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/score.json new file mode 100644 index 0000000..47fb6eb --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/score.json @@ -0,0 +1,98 @@ +{ + "hidden_nominal": { + "episodes": [ + { + "control_cost": 356.07088327407837, + "forward_reward": 5632.4692681876395, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 88.74183517647907, + "policy_error": null, + "return": 5276.398384913561, + "seed": 211, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 281.61280558038936, + "x_velocity": 6.870092442627187 + }, + { + "control_cost": 354.76760932803154, + "forward_reward": 5644.934512193883, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 87.7370754101139, + "policy_error": null, + "return": 5290.16690286585, + "seed": 307, + "terminated": false, + "truncated": true, + "variant": "nominal", + "x_position": 282.2733096756311, + "x_velocity": 5.513374251299842 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 5290.16690286585, + "max_steps": 1000, + "mean_action_latency_ms": 88.23945529329649, + "mean_control_cost": 355.41924630105495, + "mean_forward_reward": 5638.701890190761, + "mean_return": 5283.2826438897055, + "mean_x_position": 281.94305762801025, + "min_return": 5276.398384913561, + "policy_path": "submitted_artifact", + "score": 5283.2826438897055 + }, + "hidden_nominal_weight": 0.7, + "hidden_robustness": { + "episodes": [ + { + "control_cost": 356.7619220018387, + "forward_reward": 6306.763750673376, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 87.32058335322654, + "policy_error": null, + "return": 5950.001828671538, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "light_low_friction", + "x_position": 315.32050915627565, + "x_velocity": 6.34920209270735 + }, + { + "control_cost": 354.2664739340544, + "forward_reward": 4345.89047307381, + "invalid_action": false, + "length": 1000, + "mean_action_latency_ms": 89.54298742787796, + "policy_error": null, + "return": 3991.623999139752, + "seed": 503, + "terminated": false, + "truncated": true, + "variant": "heavy_weak_actuators", + "x_position": 217.2768452762976, + "x_velocity": 4.8277047961250705 + } + ], + "invalid_episode_rate": 0.0, + "max_return": 5950.001828671538, + "max_steps": 1000, + "mean_action_latency_ms": 88.43178539055225, + "mean_control_cost": 355.51419796794653, + "mean_forward_reward": 5326.327111873594, + "mean_return": 4970.812913905645, + "mean_x_position": 266.2986772162866, + "min_return": 3991.623999139752, + "policy_path": "submitted_artifact", + "score": 4970.812913905645 + }, + "hidden_robustness_weight": 0.3, + "normalized_score": 15.873072595, + "reference_score": 11487.256451471967, + "score": 5189.541724894487, + "starter_score": 4001.2886158917113 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/metadata.json new file mode 100644 index 0000000..5033390 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:33650c821735e31eea00f2ba46ae09b5d19134820c2ec005e8e96d92cb0b66d0", + "digest_compatibility_note": null, + "finished_at": 1783931627.734036, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 0.0, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "task": "simulation_heuristics_montezuma_v1", + "task_digest": "sha256:33650c821735e31eea00f2ba46ae09b5d19134820c2ec005e8e96d92cb0b66d0", + "token_usage": { + "final_input_tokens": 892.0, + "final_output_tokens": 35.0, + "input_tokens": 892.0, + "output_tokens": 35.0, + "total_tokens": 13215.0 + }, + "tool_calls": 18 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/score.json new file mode 100644 index 0000000..bbc2347 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/score.json @@ -0,0 +1,216 @@ +{ + "normalized_score": 0.0, + "reference_score": 400.0, + "score": 0.0, + "starter_score": 0.0, + "suites": { + "hidden_seeds": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.10412707590148784, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10007, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + }, + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.10631888453365089, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10103, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.10522298021756937, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.1 + }, + "noop_variants": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.10265886916421801, + "policy_error": null, + "policy_steps": 2597, + "pre_policy_noops": 3, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "startup_noop_3" + }, + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.10074814882796873, + "policy_error": null, + "policy_steps": 2587, + "pre_policy_noops": 13, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "startup_noop_13" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.10170350899609337, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.1 + }, + "recovery": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 512, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.0991505951992229, + "policy_error": null, + "policy_steps": 2084, + "pre_policy_noops": 4, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_512_noop_4" + }, + { + "bootstrap_steps": 512, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.09907003707037522, + "policy_error": null, + "policy_steps": 2080, + "pre_policy_noops": 8, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_512_noop_8" + }, + { + "bootstrap_steps": 768, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.11110870922086098, + "policy_error": null, + "policy_steps": 1830, + "pre_policy_noops": 2, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_768_noop_2" + }, + { + "bootstrap_steps": 900, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.10168849173864788, + "policy_error": null, + "policy_steps": 1700, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "recover_900" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.10275445830727675, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.55 + }, + "reproduction": { + "evaluation": { + "capped_mean_score": 0.0, + "episodes": [ + { + "bootstrap_steps": 0, + "invalid_action": false, + "length": 2600, + "mean_action_latency_ms": 0.10425583515405798, + "policy_error": null, + "policy_steps": 2600, + "pre_policy_noops": 0, + "return": 0.0, + "seed": 10001, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "canonical" + } + ], + "invalid_episode_rate": 0.0, + "max_steps": 2600, + "mean_action_latency_ms": 0.10425583515405798, + "mean_return": 0.0, + "policy_path": "submitted_artifact", + "recovery_success_rate": 0.0, + "score": 0.0, + "target_score": 400.0, + "target_success_rate": 0.0 + }, + "score": 0.0, + "weight": 0.25 + } + }, + "target_score": 400.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/metadata.json new file mode 100644 index 0000000..19ace4e --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:f0b6f90f1d8072e75f0e10f90f4b51e51e31b3590bb2c2736778ba155b23389f", + "digest_compatibility_note": null, + "finished_at": 1783931627.734036, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 52.916666667, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T080935Z/gpt-5.6-sol", + "task": "simulation_heuristics_pong_ram_v1", + "task_digest": "sha256:f0b6f90f1d8072e75f0e10f90f4b51e51e31b3590bb2c2736778ba155b23389f", + "token_usage": { + "final_input_tokens": 530.0, + "final_output_tokens": 124.0, + "input_tokens": 530.0, + "output_tokens": 124.0, + "total_tokens": 19598.0 + }, + "tool_calls": 39 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/score.json new file mode 100644 index 0000000..8c61862 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/score.json @@ -0,0 +1,133 @@ +{ + "normalized_score": 52.916666667, + "reference_score": 21.0, + "score": 9.7, + "starter_score": -2.9999999999999996, + "suite_weights": { + "hidden_nominal": 0.7, + "hidden_reset_robustness": 0.3 + }, + "suites": { + "hidden_nominal": { + "episodes": [ + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.0035600767867570676, + "points_against": 2, + "points_for": 11, + "policy_error": null, + "score": 9.0, + "seed": 101, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "article_nominal", + "won": true + }, + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.0040982644573819855, + "points_against": 2, + "points_for": 11, + "policy_error": null, + "score": 9.0, + "seed": 211, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "article_nominal", + "won": true + }, + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.004250124277960923, + "points_against": 2, + "points_for": 11, + "policy_error": null, + "score": 9.0, + "seed": 307, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "article_nominal", + "won": true + } + ], + "invalid_episode_rate": 0.0, + "max_score": 9.0, + "max_steps": 27000, + "mean_action_latency_ms": 0.003969488507366658, + "mean_score": 9.0, + "min_score": 9.0, + "policy_path": "submitted_artifact", + "score": 9.0, + "target_score": 21.0, + "target_score_rate": 0.0, + "win_rate": 1.0 + }, + "hidden_reset_robustness": { + "episodes": [ + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.004065527891119321, + "points_against": 2, + "points_for": 11, + "policy_error": null, + "score": 9.0, + "seed": 503, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "randomized_reset", + "won": true + }, + { + "invalid_action": false, + "length": 27000, + "mean_action_latency_ms": 0.004138178468026497, + "points_against": 0, + "points_for": 4, + "policy_error": null, + "score": 4.0, + "seed": 601, + "target_reached": false, + "terminated": false, + "truncated": true, + "variant": "randomized_reset", + "won": true + }, + { + "invalid_action": false, + "length": 26459, + "mean_action_latency_ms": 0.003632126502959365, + "points_against": 0, + "points_for": 21, + "policy_error": null, + "score": 21.0, + "seed": 701, + "target_reached": true, + "terminated": true, + "truncated": false, + "variant": "randomized_reset", + "won": true + } + ], + "invalid_episode_rate": 0.0, + "max_score": 21.0, + "max_steps": 27000, + "mean_action_latency_ms": 0.0039452776207017275, + "mean_score": 11.333333333333334, + "min_score": 4.0, + "policy_path": "submitted_artifact", + "score": 11.333333333333334, + "target_score": 21.0, + "target_score_rate": 0.3333333333333333, + "win_rate": 1.0 + } + }, + "target_score": 21.0 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/metadata.json new file mode 100644 index 0000000..cacb6ff --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:b3431b238bec4e66af6189d30bcd15d5cd227144dfe8bf0dd1011aa5416c1436", + "digest_compatibility_note": null, + "finished_at": 1783938003.487764, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 49.409889366, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T101636Z/gpt-5.6-sol", + "task": "simulation_heuristics_vizdoom_d1_v1", + "task_digest": "sha256:b3431b238bec4e66af6189d30bcd15d5cd227144dfe8bf0dd1011aa5416c1436", + "token_usage": { + "final_input_tokens": 589.0, + "final_output_tokens": 136.0, + "input_tokens": 589.0, + "output_tokens": 136.0, + "total_tokens": 22229.0 + }, + "tool_calls": 25 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/score.json new file mode 100644 index 0000000..d210cac --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/score.json @@ -0,0 +1,211 @@ +{ + "anchor_calibration": { + "envpool_version": "1.1.1", + "mode": "numeric", + "platform": "linux-x86_64" + }, + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": true + }, + "normalized_score": 49.409889366, + "reference_score": 0.8178199748694897, + "resource_cleanup": { + "removed_bytes": 0, + "removed_resources": 0, + "runs": [ + { + "removed_bytes": 0, + "removed_resources": 0, + "terminated_processes": 0 + }, + { + "removed_bytes": 0, + "removed_resources": 0, + "terminated_processes": 0 + } + ], + "terminated_processes": 0 + }, + "score": 0.62749998614192, + "starter_score": 0.44161998964846133, + "suites": { + "hidden_nominal": { + "batch_seed": 101, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "HEALTH": 19.0 + }, + "invalid_action": false, + "lane": 0, + "length": 2100, + "mean_action_latency_ms": 0.17381823754736356, + "policy_error": null, + "return": 0.6429999805986881, + "seed": 101, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 3.0 + }, + "invalid_action": false, + "lane": 1, + "length": 1088, + "mean_action_latency_ms": 0.14961455770603874, + "policy_error": null, + "return": 0.08299999311566353, + "seed": 102, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "HEALTH": 22.0 + }, + "invalid_action": false, + "lane": 2, + "length": 2100, + "mean_action_latency_ms": 0.15601739109981627, + "policy_error": null, + "return": 0.6899999864399433, + "seed": 103, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 71.0 + }, + "invalid_action": false, + "lane": 3, + "length": 2100, + "mean_action_latency_ms": 0.1531444631871723, + "policy_error": null, + "return": 0.9349999837577343, + "seed": 104, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 47.0 + }, + "invalid_action": false, + "lane": 4, + "length": 2100, + "mean_action_latency_ms": 0.15155099775819553, + "policy_error": null, + "return": 0.8149999864399433, + "seed": 105, + "terminated": false, + "truncated": true + } + ], + "frame_skip": 1, + "invalid_episode_rate": 0.0, + "max_return": 0.9349999837577343, + "max_steps": 2100, + "mean_action_latency_ms": 0.15682912945971728, + "mean_return": 0.6331999860703945, + "min_return": 0.08299999311566353, + "policy_path": "submitted_artifact", + "render_height": 180, + "render_width": 240, + "scenario": "d1", + "score": 0.6331999860703945 + }, + "hidden_secondary": { + "batch_seed": 607, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "HEALTH": 38.0 + }, + "invalid_action": false, + "lane": 0, + "length": 2100, + "mean_action_latency_ms": 0.14928735703939483, + "policy_error": null, + "return": 0.769999984651804, + "seed": 607, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 47.0 + }, + "invalid_action": false, + "lane": 1, + "length": 2100, + "mean_action_latency_ms": 0.13680784031748772, + "policy_error": null, + "return": 0.7989999875426292, + "seed": 608, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 37.0 + }, + "invalid_action": false, + "lane": 2, + "length": 2100, + "mean_action_latency_ms": 0.13356554454990796, + "policy_error": null, + "return": 0.764999981969595, + "seed": 609, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 47.0 + }, + "invalid_action": false, + "lane": 3, + "length": 2100, + "mean_action_latency_ms": 0.1317734359985306, + "policy_error": null, + "return": 0.8149999864399433, + "seed": 610, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "HEALTH": 6.0 + }, + "invalid_action": false, + "lane": 4, + "length": 736, + "mean_action_latency_ms": 0.12118354662442984, + "policy_error": null, + "return": -0.078000009059906, + "seed": 611, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 1, + "invalid_episode_rate": 0.0, + "max_return": 0.8149999864399433, + "max_steps": 2100, + "mean_action_latency_ms": 0.1345235449059502, + "mean_return": 0.6141999863088131, + "min_return": -0.078000009059906, + "policy_path": "submitted_artifact", + "render_height": 180, + "render_width": 240, + "scenario": "d1", + "score": 0.6141999863088131 + } + } +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/metadata.json new file mode 100644 index 0000000..15f1fe9 --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/metadata.json @@ -0,0 +1,27 @@ +{ + "benchmark": "learning_beyond_gradients_article_suite", + "current_task_digest": "sha256:fcc6ed05673b7981aa17d60ca4e0d355fbcb57a6cbca293c0fa201ab97cdd081", + "digest_compatibility_note": null, + "finished_at": 1783932419.4689522, + "harness": "opencode", + "model": { + "display_name": "GPT-5.6 Sol", + "id": "gpt-5.6-sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max" + }, + "normalized_score": 26.236631016, + "provider_reasoning_effort": "max", + "source_run_id": "leaderboard/runs/article_suite/20260713T083940Z/gpt-5.6-sol", + "task": "simulation_heuristics_vizdoom_d3_v1", + "task_digest": "sha256:fcc6ed05673b7981aa17d60ca4e0d355fbcb57a6cbca293c0fa201ab97cdd081", + "token_usage": { + "final_input_tokens": 425.0, + "final_output_tokens": 156.0, + "input_tokens": 425.0, + "output_tokens": 156.0, + "total_tokens": 23109.0 + }, + "tool_calls": 29 +} diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/score.json new file mode 100644 index 0000000..54904fa --- /dev/null +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/score.json @@ -0,0 +1,229 @@ +{ + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": true + }, + "normalized_score": 26.236631016, + "reference_score": 331.1, + "score": 110.39999999999998, + "starter_score": 31.9, + "suites": { + "hidden_nominal": { + "batch_seed": 101, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 70.0, + "HEALTH": 10.0, + "HITCOUNT": 8.0, + "KILLCOUNT": 7.0 + }, + "invalid_action": false, + "lane": 0, + "length": 1027, + "mean_action_latency_ms": 3.1377216359813986, + "policy_error": null, + "return": 140.0, + "seed": 101, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 22.0, + "DAMAGECOUNT": 40.0, + "HEALTH": 64.0, + "HITCOUNT": 5.0, + "KILLCOUNT": 5.0 + }, + "invalid_action": false, + "lane": 1, + "length": 1050, + "mean_action_latency_ms": 3.0818451521918178, + "policy_error": null, + "return": 90.0, + "seed": 102, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 40.0, + "HEALTH": 2.0, + "HITCOUNT": 5.0, + "KILLCOUNT": 4.0 + }, + "invalid_action": false, + "lane": 2, + "length": 1009, + "mean_action_latency_ms": 3.0280790251979237, + "policy_error": null, + "return": 80.0, + "seed": 103, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 10.0, + "DAMAGECOUNT": 45.0, + "HEALTH": 78.0, + "HITCOUNT": 4.0, + "KILLCOUNT": 4.0 + }, + "invalid_action": false, + "lane": 3, + "length": 1050, + "mean_action_latency_ms": 3.0736763611535673, + "policy_error": null, + "return": 85.0, + "seed": 104, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "AMMO2": 2.0, + "DAMAGECOUNT": 90.0, + "HEALTH": 14.0, + "HITCOUNT": 7.0, + "KILLCOUNT": 7.0 + }, + "invalid_action": false, + "lane": 4, + "length": 929, + "mean_action_latency_ms": 3.054685383848144, + "policy_error": null, + "return": 160.0, + "seed": 105, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 2, + "invalid_episode_rate": 0.0, + "max_return": 160.0, + "max_steps": 1050, + "mean_action_latency_ms": 3.07520151167457, + "mean_return": 111.0, + "min_return": 80.0, + "policy_path": "submitted_artifact", + "render_height": 480, + "render_width": 640, + "scenario": "d3", + "score": 111.0 + }, + "hidden_secondary": { + "batch_seed": 607, + "envpool_version": "1.1.1", + "episodes": [ + { + "final_variables": { + "AMMO2": 14.0, + "DAMAGECOUNT": 30.0, + "HEALTH": 19.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 + }, + "invalid_action": false, + "lane": 0, + "length": 240, + "mean_action_latency_ms": 3.0935825085180113, + "policy_error": null, + "return": 50.0, + "seed": 607, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 6.0, + "DAMAGECOUNT": 55.0, + "HEALTH": 1.0, + "HITCOUNT": 4.0, + "KILLCOUNT": 4.0 + }, + "invalid_action": false, + "lane": 1, + "length": 715, + "mean_action_latency_ms": 3.0578368642185065, + "policy_error": null, + "return": 95.0, + "seed": 608, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 0.0, + "DAMAGECOUNT": 20.0, + "HEALTH": 9.0, + "HITCOUNT": 3.0, + "KILLCOUNT": 2.0 + }, + "invalid_action": false, + "lane": 2, + "length": 956, + "mean_action_latency_ms": 3.0342543454878284, + "policy_error": null, + "return": 40.0, + "seed": 609, + "terminated": true, + "truncated": false + }, + { + "final_variables": { + "AMMO2": 9.0, + "DAMAGECOUNT": 35.0, + "HEALTH": 61.0, + "HITCOUNT": 4.0, + "KILLCOUNT": 4.0 + }, + "invalid_action": false, + "lane": 3, + "length": 1050, + "mean_action_latency_ms": 3.052990203418414, + "policy_error": null, + "return": 75.0, + "seed": 610, + "terminated": false, + "truncated": true + }, + { + "final_variables": { + "AMMO2": 4.0, + "DAMAGECOUNT": 145.0, + "HEALTH": 2.0, + "HITCOUNT": 14.0, + "KILLCOUNT": 14.0 + }, + "invalid_action": false, + "lane": 4, + "length": 992, + "mean_action_latency_ms": 3.0088798643367727, + "policy_error": null, + "return": 285.0, + "seed": 611, + "terminated": true, + "truncated": false + } + ], + "frame_skip": 2, + "invalid_episode_rate": 0.0, + "max_return": 285.0, + "max_steps": 1050, + "mean_action_latency_ms": 3.0495087571959067, + "mean_return": 109.0, + "min_return": 40.0, + "policy_path": "submitted_artifact", + "render_height": 480, + "render_width": 640, + "scenario": "d3", + "score": 109.0 + } + } +} diff --git a/pyproject.toml b/pyproject.toml index 183041a..7f8a7e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,9 @@ dev = [ "pytest>=8.4,<9", "ruff>=0.12,<1", ] +sandbox-daytona = [ + "benchflow[sandbox-daytona]>=0.6.5,<0.7", +] [project.scripts] genesisbench-eval-simulation-heuristics-ant-v1 = "genesisbench.cli:main" diff --git a/scripts/bench_opencode.py b/scripts/bench_opencode.py new file mode 100644 index 0000000..1ca6559 --- /dev/null +++ b/scripts/bench_opencode.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Run BenchFlow with OpenCode's provider-native protocol enabled. + +BenchFlow 0.6.5 normally routes OpenCode through a Chat Completions LiteLLM +gateway. Current Azure GPT-5.6 Sol tool use requires OpenCode's native Azure +Responses-API path, so this entrypoint keeps BenchFlow's sandbox, ACP, and +verifier lifecycle while bypassing only that incompatible model gateway. +""" + +from __future__ import annotations + +from dataclasses import replace + +from benchflow.agents.registry import AGENTS, CredentialFile +from benchflow.providers import litellm_runtime + + +def apply_opencode_direct_provider_mode() -> None: + """Let OpenCode speak its provider-native protocol without LiteLLM. + + Azure GPT-5.6 Sol currently rejects the function-call transformation made + by BenchFlow 0.6.5's chat-completions gateway. OpenCode's native Azure + Responses-API path is verified separately and preserves tool use. + """ + + litellm_runtime._NATIVE_PROTOCOL_AGENTS = frozenset( # noqa: SLF001 + {*litellm_runtime._NATIVE_PROTOCOL_AGENTS, "opencode"} # noqa: SLF001 + ) + + +def apply_opencode_claude_oauth_support() -> None: + """Materialize the provided Claude OAuth token for the OpenCode plugin.""" + + config = AGENTS["opencode"] + credential_path = "{home}/.claude/.credentials.json" + if any(item.path == credential_path for item in config.credential_files): + return + credential = CredentialFile( + path=credential_path, + env_source="CLAUDE_CODE_OAUTH_TOKEN", + template=( + '{{"claudeAiOauth":{{"accessToken":"{value}",' + '"refreshToken":"","expiresAt":4102444800000}}}}' + ), + ) + AGENTS["opencode"] = replace( + config, + credential_files=[*config.credential_files, credential], + ) + + +def main() -> None: + apply_opencode_direct_provider_mode() + apply_opencode_claude_oauth_support() + from benchflow.cli.main import app + + app() + + +if __name__ == "__main__": + main() diff --git a/scripts/build_article_suite_leaderboard.py b/scripts/build_article_suite_leaderboard.py new file mode 100644 index 0000000..5265b68 --- /dev/null +++ b/scripts/build_article_suite_leaderboard.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import tomllib +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from benchflow._utils.task_authoring import task_digest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +MODELS_PATH = REPO_ROOT / "experiments" / "article_suite" / "models.toml" +TASKS = ( + "simulation_heuristics_ant_v1", + "simulation_heuristics_pong_ram_v1", + "simulation_heuristics_breakout_ram_v1", + "simulation_heuristics_breakout_rgb_v1", + "simulation_heuristics_halfcheetah_v1", + "simulation_heuristics_vizdoom_d1_v1", + "simulation_heuristics_vizdoom_d3_v1", + "simulation_heuristics_atari57_v1", + "simulation_heuristics_montezuma_v1", +) +TASK_DIGEST_COMPATIBILITY = { + "simulation_heuristics_ant_v1": { + "sha256:bbb533da0cb86459f4d49dee667e6c73ac54c0188bc40e54e911d50ef3c3bc38": ( + "Score-equivalent to the current task. The only later change adds " + "a fail-closed internal timeout for candidates whose verifier " + "would otherwise exceed BenchFlow's deadline." + ) + } +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build the aggregate nine-task article-suite leaderboard." + ) + parser.add_argument( + "--runs-root", + type=Path, + default=REPO_ROOT / "leaderboard" / "runs" / "article_suite", + ) + parser.add_argument( + "--output", + type=Path, + default=REPO_ROOT / "leaderboard" / "article_suite.json", + ) + return parser.parse_args() + + +def _result_files(model_root: Path) -> list[Path]: + jobs = model_root / "jobs" + candidates = sorted( + path + for path in jobs.glob("*/results.jsonl") + if path.parent.name != jobs.name + ) + if not candidates: + candidates = sorted(jobs.glob("results.jsonl")) + return candidates + + +def _load_results( + model_root: Path, + *, + expected_tasks: tuple[str, ...] = TASKS, + allow_partial_errors: bool = False, +) -> dict[str, dict[str, Any]]: + files = _result_files(model_root) + if len(files) != 1: + raise RuntimeError( + f"{model_root} must contain exactly one top-level results.jsonl; " + f"found {len(files)}" + ) + rows: dict[str, dict[str, Any]] = {} + for line in files[0].read_text().splitlines(): + if not line.strip(): + continue + row = json.loads(line) + task_name = row.get("info", {}).get("task_name") + if task_name not in expected_tasks: + continue + error = row.get("error") + rollout_dir = row.get("info", {}).get("rollout_dir") + score_details = ( + Path(rollout_dir) / "verifier" / "genesis-score.json" + if isinstance(rollout_dir, str) + else None + ) + has_score_details = bool( + score_details is not None and score_details.is_file() + ) + if error is not None and not ( + isinstance(error, dict) + and error.get("error") == "missing_llm_trajectory" + ) and not has_score_details: + if allow_partial_errors: + continue + raise RuntimeError(f"{model_root.name}/{task_name} contains an error") + reward = row.get("reward") + if ( + not isinstance(reward, int | float) + or isinstance(reward, bool) + or not math.isfinite(float(reward)) + ): + raise RuntimeError( + f"{model_root.name}/{task_name} has invalid reward {reward!r}" + ) + rows[task_name] = row + missing = sorted(set(expected_tasks) - set(rows)) + if missing and not allow_partial_errors: + raise RuntimeError( + f"{model_root.name} is missing article-suite tasks: " + + ", ".join(missing) + ) + return rows + + +def _normalized_task_score(row: dict[str, Any]) -> float: + rollout_dir = row.get("info", {}).get("rollout_dir") + if isinstance(rollout_dir, str): + details_path = Path(rollout_dir) / "verifier" / "genesis-score.json" + if details_path.is_file(): + details = json.loads(details_path.read_text()) + normalized = details.get("normalized_score") + if ( + isinstance(normalized, int | float) + and not isinstance(normalized, bool) + and math.isfinite(float(normalized)) + ): + return float(normalized) + return 100.0 * float(row["reward"]) + + +def _sanitize_score_paths(value: Any) -> Any: + if isinstance(value, dict): + return { + key: ( + "submitted_artifact" + if key in {"policy_path", "artifact_path"} + else _sanitize_score_paths(item) + ) + for key, item in value.items() + } + if isinstance(value, list): + return [_sanitize_score_paths(item) for item in value] + return value + + +def _digest_compatibility_note( + task: str, + recorded_digest: str, + current_digest: str, +) -> str | None: + if recorded_digest == current_digest: + return None + return TASK_DIGEST_COMPATIBILITY.get(task, {}).get(recorded_digest) + + +def _expected_models() -> dict[str, dict[str, Any]]: + payload = tomllib.loads(MODELS_PATH.read_text()) + return {model["id"]: model for model in payload["models"]} + + +def _latest_model_runs(runs_root: Path) -> dict[str, Path]: + selected: dict[str, tuple[float, Path]] = {} + for metadata_path in runs_root.glob("*/*/run_metadata.json"): + metadata = json.loads(metadata_path.read_text()) + if metadata.get("dry_run"): + continue + model_id = metadata.get("model", {}).get("id") + finished_at = metadata.get("finished_at") + if not isinstance(model_id, str) or not isinstance( + finished_at, + int | float, + ): + continue + current = selected.get(model_id) + if current is None or float(finished_at) > current[0]: + selected[model_id] = (float(finished_at), metadata_path.parent) + return {model_id: item[1] for model_id, item in selected.items()} + + +def _latest_task_results( + runs_root: Path, +) -> dict[ + str, + dict[str, tuple[float, dict[str, Any], Path, str]], +]: + selected: dict[ + str, + dict[str, tuple[float, dict[str, Any], Path, str]], + ] = {} + for metadata_path in runs_root.glob("*/*/run_metadata.json"): + metadata = json.loads(metadata_path.read_text()) + if metadata.get("dry_run"): + continue + model_id = metadata.get("model", {}).get("id") + finished_at = metadata.get("finished_at") + configured_tasks = metadata.get("tasks") + if ( + not isinstance(model_id, str) + or not isinstance(finished_at, int | float) + or not isinstance(configured_tasks, list) + or not configured_tasks + or not all(task in TASKS for task in configured_tasks) + ): + continue + model_root = metadata_path.parent + manifest_path = model_root / "task_manifest.json" + if not manifest_path.is_file(): + continue + manifest = json.loads(manifest_path.read_text()) + manifest_digests = { + item.get("task_id"): item.get("digest") + for item in manifest.get("tasks", []) + if isinstance(item, dict) + } + if any( + not isinstance(manifest_digests.get(task), str) + for task in configured_tasks + ): + continue + try: + rows = _load_results( + model_root, + expected_tasks=tuple(configured_tasks), + allow_partial_errors=True, + ) + except RuntimeError: + continue + model_rows = selected.setdefault(model_id, {}) + for task, row in rows.items(): + current = model_rows.get(task) + if current is None or float(finished_at) > current[0]: + model_rows[task] = ( + float(finished_at), + row, + model_root, + manifest_digests[task], + ) + return selected + + +def main() -> None: + args = parse_args() + runs_root = args.runs_root.resolve() + expected_models = _expected_models() + task_results = _latest_task_results(runs_root) + missing_models = sorted(set(expected_models) - set(task_results)) + if missing_models: + raise RuntimeError( + "Missing completed article-suite model runs: " + + ", ".join(missing_models) + ) + ranked: list[dict[str, Any]] = [] + submissions_root = args.output.parent / "article_suite_submissions" + for model_id in expected_models: + model_results = task_results[model_id] + missing_tasks = sorted(set(TASKS) - set(model_results)) + if missing_tasks: + raise RuntimeError( + f"{model_id} is missing completed task results: " + + ", ".join(missing_tasks) + ) + stale_tasks = [ + task + for task in TASKS + if _digest_compatibility_note( + task, + model_results[task][3], + task_digest(REPO_ROOT / "tasks" / task), + ) + is None + and model_results[task][3] + != task_digest(REPO_ROOT / "tasks" / task) + ] + if stale_tasks: + raise RuntimeError( + f"{model_id} has stale task digests: " + + ", ".join(stale_tasks) + ) + latest_task = max( + model_results.values(), + key=lambda item: item[0], + ) + metadata = json.loads( + (latest_task[2] / "run_metadata.json").read_text() + ) + task_scores = { + task: _normalized_task_score(model_results[task][1]) + for task in TASKS + } + submission_details: dict[str, str] = {} + for task in TASKS: + _, result_row, model_root, digest = model_results[task] + current_digest = task_digest(REPO_ROOT / "tasks" / task) + compatibility_note = _digest_compatibility_note( + task, + digest, + current_digest, + ) + task_metadata = json.loads( + (model_root / "run_metadata.json").read_text() + ) + rollout_dir = Path(result_row["info"]["rollout_dir"]) + source_score = rollout_dir / "verifier" / "genesis-score.json" + if not source_score.is_file(): + raise RuntimeError( + f"{model_id}/{task} has no verifier genesis-score.json" + ) + destination = submissions_root / model_id / task + destination.mkdir(parents=True, exist_ok=True) + sanitized_score = _sanitize_score_paths( + json.loads(source_score.read_text()) + ) + (destination / "score.json").write_text( + json.dumps(sanitized_score, indent=2, sort_keys=True) + "\n" + ) + source_run = str(model_root.relative_to(REPO_ROOT)) + metadata_payload = { + "benchmark": "learning_beyond_gradients_article_suite", + "task": task, + "task_digest": digest, + "current_task_digest": current_digest, + "digest_compatibility_note": compatibility_note, + "model": task_metadata["model"], + "harness": task_metadata["harness"], + "provider_reasoning_effort": task_metadata[ + "provider_reasoning_effort" + ], + "normalized_score": task_scores[task], + "source_run_id": source_run, + "finished_at": task_metadata["finished_at"], + "tool_calls": result_row.get("metrics", {}).get( + "n_tool_calls", + result_row.get("total_tool_calls"), + ), + "token_usage": result_row.get("token_usage"), + } + (destination / "metadata.json").write_text( + json.dumps(metadata_payload, indent=2, sort_keys=True) + "\n" + ) + submission_details[task] = str( + (destination / "score.json").relative_to(REPO_ROOT) + ) + average = sum(task_scores.values()) / len(TASKS) + ranked.append( + { + "model_id": metadata["model"]["id"], + "model": metadata["model"]["display_name"], + "harness": metadata["harness"], + "provider_reasoning_effort": metadata[ + "provider_reasoning_effort" + ], + "average_normalized_score": average, + "task_scores": task_scores, + "submission_details": submission_details, + "source_runs": { + task: str( + model_results[task][2].relative_to(REPO_ROOT) + ) + for task in TASKS + }, + } + ) + ranked.sort( + key=lambda row: row["average_normalized_score"], + reverse=True, + ) + for rank, row in enumerate(ranked, start=1): + row["rank"] = rank + + payload = { + "benchmark": "learning_beyond_gradients_article_suite", + "task_count": len(TASKS), + "tasks": list(TASKS), + "aggregation": "arithmetic_mean_of_normalized_task_scores", + "task_digest_compatibility": TASK_DIGEST_COMPATIBILITY, + "source_runs": { + model_id: { + task: str(task_results[model_id][task][2].relative_to(REPO_ROOT)) + for task in TASKS + } + for model_id in expected_models + }, + "generated_at": datetime.fromtimestamp( + max( + item[0] + for model_results in task_results.values() + for item in model_results.values() + ), + UTC, + ).isoformat(), + "rows": ranked, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + labels = { + "simulation_heuristics_ant_v1": "Ant", + "simulation_heuristics_pong_ram_v1": "Pong", + "simulation_heuristics_breakout_ram_v1": "Breakout RAM", + "simulation_heuristics_breakout_rgb_v1": "Breakout RGB", + "simulation_heuristics_halfcheetah_v1": "HalfCheetah", + "simulation_heuristics_vizdoom_d1_v1": "Doom D1", + "simulation_heuristics_vizdoom_d3_v1": "Doom D3", + "simulation_heuristics_atari57_v1": "Atari57", + "simulation_heuristics_montezuma_v1": "Montezuma", + } + header = [ + "Rank", + "Model", + "Harness", + "Effort", + "Average", + *(labels[task] for task in TASKS), + ] + alignment = ["---:", "---", "---", "---", "---:", *(["---:"] * len(TASKS))] + markdown = [ + "# GenesisBench Learning Beyond Gradients Article Suite", + "", + "The aggregate score is the arithmetic mean of nine normalized task " + "scores. Starter policies map to 0 and trusted article-level references " + "map to 100.", + "", + "| " + " | ".join(header) + " |", + "| " + " | ".join(alignment) + " |", + ] + for row in ranked: + values = [ + str(row["rank"]), + row["model"], + row["harness"], + row["provider_reasoning_effort"], + f"{row['average_normalized_score']:.2f}", + *(f"{row['task_scores'][task]:.2f}" for task in TASKS), + ] + markdown.append("| " + " | ".join(values) + " |") + (args.output.parent / "ARTICLE_SUITE.md").write_text( + "\n".join(markdown) + "\n" + ) + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_article_suite.py b/scripts/run_article_suite.py new file mode 100644 index 0000000..032286d --- /dev/null +++ b/scripts/run_article_suite.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import time +import tomllib +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + + +REPO_ROOT = Path(__file__).resolve().parents[1] +MODELS_PATH = REPO_ROOT / "experiments" / "article_suite" / "models.toml" +TASKS = ( + "simulation_heuristics_ant_v1", + "simulation_heuristics_pong_ram_v1", + "simulation_heuristics_breakout_ram_v1", + "simulation_heuristics_breakout_rgb_v1", + "simulation_heuristics_halfcheetah_v1", + "simulation_heuristics_vizdoom_d1_v1", + "simulation_heuristics_vizdoom_d3_v1", + "simulation_heuristics_atari57_v1", + "simulation_heuristics_montezuma_v1", +) +OPENCODE_AGENT_ENV = { + "OPENCODE_DISABLE_AUTOUPDATE": "1", + "OPENCODE_DISABLE_CLAUDE_CODE": "1", + "OPENCODE_DISABLE_DEFAULT_PLUGINS": "1", + "OPENCODE_DISABLE_EXTERNAL_SKILLS": "1", + "OPENCODE_DISABLE_LSP_DOWNLOAD": "1", +} + + +def _default_env_path() -> Path: + configured = os.environ.get("GENESISBENCH_ENV_FILE") + if configured: + return Path(configured).expanduser() + return REPO_ROOT / ".env" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the nine-task article suite through OpenCode." + ) + selection = parser.add_mutually_exclusive_group(required=True) + selection.add_argument("--model") + selection.add_argument("--all-models", action="store_true") + parser.add_argument( + "--task", + action="append", + choices=TASKS, + help="Run only this task; repeat for multiple tasks. Defaults to all nine.", + ) + parser.add_argument("--env-file", type=Path, default=_default_env_path()) + parser.add_argument( + "--output-root", + type=Path, + default=REPO_ROOT / "leaderboard" / "runs" / "article_suite", + ) + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument( + "--sandbox", + choices=("docker", "daytona"), + default="docker", + ) + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +def _read_env(path: Path) -> dict[str, str]: + result: dict[str, str] = {} + if not path.is_file(): + return result + for raw_line in path.read_text().splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + value = value.strip() + if ( + len(value) >= 2 + and value[0] == value[-1] + and value[0] in {'"', "'"} + ): + value = value[1:-1] + result[key] = value + return result + + +def _models() -> list[dict[str, Any]]: + payload = tomllib.loads(MODELS_PATH.read_text()) + return list(payload["models"]) + + +def _select_models(model_id: str | None, all_models: bool) -> list[dict[str, Any]]: + models = _models() + if all_models: + return models + selected = [model for model in models if model["id"] == model_id] + if not selected: + choices = ", ".join(model["id"] for model in models) + raise ValueError(f"Unknown model {model_id!r}; choose one of: {choices}") + return selected + + +def _validate_tasks(tasks: tuple[str, ...]) -> None: + missing = [ + task + for task in tasks + if not (REPO_ROOT / "tasks" / task / "task.md").is_file() + ] + if missing: + raise RuntimeError( + "Article suite task packages are missing: " + ", ".join(missing) + ) + + +def _require_credentials(model: dict[str, Any], env: dict[str, str]) -> None: + required = { + "azure": ("AZURE_API_ENDPOINT", "AZURE_API_KEY"), + "claude_oauth": ("CLAUDE_CODE_OAUTH_TOKEN",), + }[model["provider"]] + missing = [name for name in required if not env.get(name)] + if missing: + raise RuntimeError( + f"Missing credentials for {model['id']}: {', '.join(missing)}" + ) + + +def _scoped_provider_env( + model: dict[str, Any], + provider_env: dict[str, str], +) -> dict[str, str]: + scoped = provider_env.copy() + if model["provider"] == "azure": + for key in ( + "ANTHROPIC_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "CLAUDE_OAUTH_TOKEN", + ): + scoped.pop(key, None) + elif model["provider"] == "claude_oauth": + for key in ( + "AZURE_API_ENDPOINT", + "AZURE_API_KEY", + "AZURE_API_VERSION", + "AZURE_RESOURCE", + "AZURE_RESOURCE_NAME", + "CLAUDE_OAUTH_TOKEN", + ): + scoped.pop(key, None) + # BenchFlow's provider resolver expects the standard Anthropic slot. + # The OpenCode plugin intercepts the request and uses the separately + # materialized OAuth credential, so a non-secret placeholder is enough. + scoped["ANTHROPIC_API_KEY"] = "oauth-plugin" + return scoped + + +def build_command( + *, + model: dict[str, Any], + jobs_dir: Path, + sandbox: str, + concurrency: int, + artifact_dir: Path, + tasks: tuple[str, ...] = TASKS, + azure_resource_name: str | None = None, +) -> list[str]: + command = [ + "uv", + "run", + "python", + "scripts/bench_opencode.py", + "eval", + "run", + "--tasks-dir", + "tasks", + "--agent", + "opencode", + "--model", + model["model"], + "--sandbox", + sandbox, + "--skill-mode", + "no-skill", + "--loop-strategy", + "single-shot", + "--usage-tracking", + "off", + "--concurrency", + str(concurrency), + "--build-concurrency", + "1", + "--agent-idle-timeout", + str(model.get("agent_idle_timeout_sec", 600)), + "--expected-tasks", + str(len(tasks)), + "--jobs-dir", + str(jobs_dir), + "--health-summary-out", + str(artifact_dir / "health.json"), + "--task-manifest-out", + str(artifact_dir / "task_manifest.json"), + "--run-config-out", + str(artifact_dir / "run_config.json"), + "--context-root", + ".", + ] + command.extend( + [ + "--agent-env", + "OPENCODE_CONFIG_CONTENT=" + + json.dumps( + _opencode_config(model), + separators=(",", ":"), + sort_keys=True, + ), + ] + ) + for key, value in OPENCODE_AGENT_ENV.items(): + command.extend(["--agent-env", f"{key}={value}"]) + if azure_resource_name: + command.extend( + ["--agent-env", f"AZURE_RESOURCE_NAME={azure_resource_name}"] + ) + for task in tasks: + command.extend(["--include", task]) + return command + + +def _opencode_config(model: dict[str, Any]) -> dict[str, Any]: + provider = model["model"].split("/", 1)[0] + bare_model = model["model"].split("/", 1)[1] + effort = model["provider_reasoning_effort"] + definition: dict[str, Any] = { + "name": model["display_name"], + "reasoning": True, + } + if provider == "azure": + definition["options"] = { + "reasoningEffort": effort, + "reasoningSummary": "auto", + } + definition["variants"] = { + effort: { + "reasoningEffort": effort, + "reasoningSummary": "auto", + "include": ["reasoning.encrypted_content"], + } + } + elif provider == "anthropic": + budget = 31_999 if effort == "max" else 16_000 + definition["options"] = { + "thinking": { + "type": "enabled", + "budgetTokens": budget, + } + } + definition["variants"] = { + effort: { + "thinking": { + "type": "enabled", + "budgetTokens": budget, + } + } + } + config: dict[str, Any] = { + "$schema": "https://opencode.ai/config.json", + "provider": { + provider: { + "models": { + bare_model: definition, + } + } + }, + } + if model["provider"] == "claude_oauth": + config["plugin"] = ["opencode-claude-auth@2.0.0"] + return config + + +def _azure_resource_name(env: dict[str, str]) -> str: + explicit = env.get("AZURE_RESOURCE_NAME", "").strip() + if explicit: + return explicit + endpoint = env.get("AZURE_API_ENDPOINT", "").strip() + if not endpoint: + raise RuntimeError("AZURE_API_ENDPOINT is required") + parsed = urlparse( + endpoint if "://" in endpoint else f"https://{endpoint}" + ) + host = (parsed.hostname or "").strip() + suffix = ".openai.azure.com" + if not host.endswith(suffix): + raise RuntimeError( + "AZURE_API_ENDPOINT must use an *.openai.azure.com host" + ) + resource = host[: -len(suffix)] + if not resource: + raise RuntimeError("Could not derive Azure resource name") + return resource + + +def _docker_ready() -> bool: + result = subprocess.run( + ["docker", "info"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + return result.returncode == 0 + + +def main() -> None: + args = parse_args() + tasks = tuple(args.task or TASKS) + _validate_tasks(tasks) + models = _select_models(args.model, args.all_models) + provider_env = os.environ.copy() + provider_env.update(_read_env(args.env_file.expanduser())) + if args.sandbox == "daytona": + if not provider_env.get("DAYTONA_API_KEY"): + raise RuntimeError("DAYTONA_API_KEY is required for --sandbox daytona") + provider_env.setdefault("DAYTONA_TARGET", "us") + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + batch_root = args.output_root.resolve() / timestamp + batch_root.mkdir(parents=True, exist_ok=True) + + if args.sandbox == "docker" and not args.dry_run and not _docker_ready(): + raise RuntimeError( + "Docker is not ready. Start Docker Desktop before running the " + "authoritative article-suite evaluation." + ) + + batch_manifest: dict[str, Any] = { + "suite": "learning_beyond_gradients_article", + "harness": "opencode", + "tasks": list(tasks), + "created_at": datetime.now(UTC).isoformat(), + "models": [], + } + for model in models: + _require_credentials(model, provider_env) + model_root = batch_root / model["id"] + jobs_dir = model_root / "jobs" + model_root.mkdir(parents=True, exist_ok=True) + run_env = _scoped_provider_env(model, provider_env) + azure_resource_name = None + if model["provider"] == "azure": + azure_resource_name = _azure_resource_name(run_env) + run_env["AZURE_RESOURCE_NAME"] = azure_resource_name + command = build_command( + model=model, + jobs_dir=jobs_dir, + sandbox=args.sandbox, + concurrency=args.concurrency, + artifact_dir=model_root, + tasks=tasks, + azure_resource_name=azure_resource_name, + ) + run_env["BENCHFLOW_REASONING_EFFORT"] = model[ + "provider_reasoning_effort" + ] + if "daytona_pty_readline_timeout_sec" in model: + run_env["BENCHFLOW_DAYTONA_PTY_READLINE_TIMEOUT"] = str( + model["daytona_pty_readline_timeout_sec"] + ) + started_at = time.time() + metadata = { + "model": model, + "harness": "opencode", + "sandbox": args.sandbox, + "tasks": list(tasks), + "command": command, + "provider_env_keys": sorted( + key + for key in ( + "AZURE_API_ENDPOINT", + "AZURE_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + ) + if run_env.get(key) + ), + "provider_reasoning_effort": model["provider_reasoning_effort"], + "started_at": started_at, + "finished_at": None, + "elapsed_seconds": None, + "return_code": None, + "status": "running", + "dry_run": args.dry_run, + } + metadata_path = model_root / "run_metadata.json" + metadata_path.write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n" + ) + batch_entry = { + "id": model["id"], + "run_metadata": str(metadata_path.relative_to(batch_root)), + "return_code": None, + "status": "running", + } + batch_manifest["models"].append(batch_entry) + (batch_root / "batch_manifest.json").write_text( + json.dumps(batch_manifest, indent=2, sort_keys=True) + "\n" + ) + return_code = 0 + status = "completed" + try: + if args.dry_run: + print(" ".join(command)) + else: + return_code = subprocess.run( + command, + cwd=REPO_ROOT, + env=run_env, + check=False, + ).returncode + if return_code != 0: + status = "failed" + except KeyboardInterrupt: + return_code = 130 + status = "interrupted" + raise + finally: + metadata.update( + { + "finished_at": time.time(), + "elapsed_seconds": time.time() - started_at, + "return_code": return_code, + "status": status, + } + ) + metadata_path.write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n" + ) + batch_entry["return_code"] = return_code + batch_entry["status"] = status + (batch_root / "batch_manifest.json").write_text( + json.dumps(batch_manifest, indent=2, sort_keys=True) + "\n" + ) + if return_code != 0: + raise SystemExit(return_code) + + (batch_root / "batch_manifest.json").write_text( + json.dumps(batch_manifest, indent=2, sort_keys=True) + "\n" + ) + print(batch_root) + + +if __name__ == "__main__": + main() diff --git a/src/genesisbench/ant.py b/src/genesisbench/ant.py index ecdfda3..4e5b3ad 100644 --- a/src/genesisbench/ant.py +++ b/src/genesisbench/ant.py @@ -4,6 +4,8 @@ import inspect import json import math +import shutil +import sys import tempfile import time import xml.etree.ElementTree as ET @@ -87,9 +89,7 @@ def invalid_episode_rate(self) -> float: @property def mean_action_latency_ms(self) -> float: return float( - np.mean( - [episode.mean_action_latency_ms for episode in self.episodes] - ) + np.mean([episode.mean_action_latency_ms for episode in self.episodes]) ) def to_dict(self) -> dict[str, Any]: @@ -117,14 +117,20 @@ def to_json(self) -> str: def _load_policy_module(policy_path: Path) -> ModuleType: + module_name = f"genesisbench_submission_{abs(hash(policy_path.resolve()))}" spec = importlib.util.spec_from_file_location( - f"genesisbench_submission_{abs(hash(policy_path.resolve()))}", + module_name, policy_path, ) if spec is None or spec.loader is None: raise RuntimeError(f"Unable to import policy from {policy_path}") module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise return module @@ -144,6 +150,42 @@ def _instantiate_policy(module: ModuleType, seed: int) -> Any: raise AttributeError("Submission must define Policy or make_policy") +def _configure_policy( + policy: Any, + *, + model_xml_path: Path, + frame_skip: int, +) -> None: + configure = getattr(policy, "configure_simulator", None) + if configure is None: + return + + available = { + "model_xml_path": str(model_xml_path), + "frame_skip": int(frame_skip), + } + try: + signature = inspect.signature(configure) + except (TypeError, ValueError): + configure(**available) + return + + accepts_kwargs = any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + kwargs = ( + available + if accepts_kwargs + else { + name: value + for name, value in available.items() + if name in signature.parameters + } + ) + configure(**kwargs) + + def _reset_policy(policy: Any, seed: int) -> None: reset = getattr(policy, "reset", None) if reset is None: @@ -205,14 +247,21 @@ def _make_ant_env( *, variant: DynamicsVariant, max_steps: int, -) -> tuple[gym.Env, tempfile.TemporaryDirectory[str] | None]: - temporary_directory: tempfile.TemporaryDirectory[str] | None = None +) -> tuple[ + gym.Env, + tempfile.TemporaryDirectory[str] | None, + Path, +]: + temporary_directory = tempfile.TemporaryDirectory( + prefix="genesisbench-simulation-heuristics-ant-v1-" + ) + model_xml_path = Path(temporary_directory.name) / f"{variant.name}.xml" kwargs: dict[str, Any] = { "include_cfrc_ext_in_observation": False, "contact_cost_weight": 0.0, "max_episode_steps": max_steps, } - if variant.name != "nominal" or any( + is_variant = variant.name != "nominal" or any( not math.isclose(value, 1.0) for value in ( variant.density_scale, @@ -220,15 +269,18 @@ def _make_ant_env( variant.damping_scale, variant.actuator_scale, ) - ): - temporary_directory = tempfile.TemporaryDirectory( - prefix="genesisbench-simulation-heuristics-ant-v1-" - ) - xml_path = Path(temporary_directory.name) / f"{variant.name}.xml" - _write_variant_xml(variant, xml_path) - kwargs["xml_file"] = str(xml_path) + ) + if is_variant: + _write_variant_xml(variant, model_xml_path) + else: + shutil.copy2(_ant_xml_path(), model_xml_path) + kwargs["xml_file"] = str(model_xml_path) - return gym.make("Ant-v5", **kwargs), temporary_directory + return ( + gym.make("Ant-v5", **kwargs), + temporary_directory, + model_xml_path, + ) def _validate_action(action: Any) -> np.ndarray: @@ -256,17 +308,23 @@ def evaluate_ant_policy( module = _load_policy_module(path) configured_variants = tuple(variants or (DynamicsVariant(),)) + configured_seeds = tuple(int(seed) for seed in seeds) episodes: list[AntEpisode] = [] for variant in configured_variants: - for seed in seeds: - env, temporary_directory = _make_ant_env( + for seed in configured_seeds: + env, temporary_directory, model_xml_path = _make_ant_env( variant=variant, max_steps=max_steps, ) + observation, _ = env.reset(seed=seed) policy = _instantiate_policy(module, seed) + _configure_policy( + policy, + model_xml_path=model_xml_path, + frame_skip=int(env.unwrapped.frame_skip), + ) _reset_policy(policy, seed) - observation, _ = env.reset(seed=seed) episode_return = 0.0 terminated = False truncated = False @@ -287,9 +345,7 @@ def evaluate_ant_policy( episode_return = failure_return break latencies.append((time.perf_counter() - started_at) * 1000) - observation, reward, terminated, truncated, info = env.step( - action - ) + observation, reward, terminated, truncated, info = env.step(action) episode_return += float(reward) if terminated or truncated: break diff --git a/src/genesisbench/atari57.py b/src/genesisbench/atari57.py new file mode 100644 index 0000000..947e74e --- /dev/null +++ b/src/genesisbench/atari57.py @@ -0,0 +1,1004 @@ +from __future__ import annotations + +import csv +import importlib.util +import inspect +import json +import statistics +import sys +import time +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType +from typing import Any, Callable, Iterable + +import numpy as np + +ATARI57_GAMES = ( + "Alien-v5", + "Amidar-v5", + "Assault-v5", + "Asterix-v5", + "Asteroids-v5", + "Atlantis-v5", + "BankHeist-v5", + "BattleZone-v5", + "BeamRider-v5", + "Berzerk-v5", + "Bowling-v5", + "Boxing-v5", + "Breakout-v5", + "Centipede-v5", + "ChopperCommand-v5", + "CrazyClimber-v5", + "Defender-v5", + "DemonAttack-v5", + "DoubleDunk-v5", + "Enduro-v5", + "FishingDerby-v5", + "Freeway-v5", + "Frostbite-v5", + "Gopher-v5", + "Gravitar-v5", + "Hero-v5", + "IceHockey-v5", + "Jamesbond-v5", + "Kangaroo-v5", + "Krull-v5", + "KungFuMaster-v5", + "MontezumaRevenge-v5", + "MsPacman-v5", + "NameThisGame-v5", + "Phoenix-v5", + "Pitfall-v5", + "Pong-v5", + "PrivateEye-v5", + "Qbert-v5", + "Riverraid-v5", + "RoadRunner-v5", + "Robotank-v5", + "Seaquest-v5", + "Skiing-v5", + "Solaris-v5", + "SpaceInvaders-v5", + "StarGunner-v5", + "Surround-v5", + "Tennis-v5", + "TimePilot-v5", + "Tutankham-v5", + "UpNDown-v5", + "Venture-v5", + "VideoPinball-v5", + "WizardOfWor-v5", + "YarsRevenge-v5", + "Zaxxon-v5", +) +OBSERVATION_MODES = ("ram", "native_obs") +SEARCH_REPEAT_INDICES = (0, 1, 2) +FRAME_BUDGET_PER_SEARCH = 20_000_000 +ENVPOOL_VERSION = "1.1.1" +ARTICLE_SEARCH_EVIDENCE_FILES = ( + "policy.py", + "trials.jsonl", + "summary.csv", + "sample_efficiency.png", + "README.md", +) +ATARI_SETTINGS: dict[str, int | bool | float] = { + "img_height": 210, + "img_width": 160, + "stack_num": 1, + "gray_scale": False, + "frame_skip": 1, + "noop_max": 1, + "use_fire_reset": True, + "episodic_life": False, + "reward_clip": False, + "repeat_action_probability": 0.0, + "full_action_space": False, +} + + +class Atari57ArtifactError(ValueError): + """Raised when an aggregate Atari57 artifact violates the public contract.""" + + +@dataclass(frozen=True) +class SearchTrajectory: + env_id: str + obs_mode: str + repeat_index: int + frame_budget: int = FRAME_BUDGET_PER_SEARCH + + +@dataclass(frozen=True) +class AtariPolicySpec: + env_id: str + obs_mode: str + repeat_index: int + module_path: Path + config: dict[str, Any] + + +@dataclass(frozen=True) +class InteractionRecord: + env_id: str + obs_mode: str + repeat_index: int + cumulative_env_steps: int + cumulative_episodes: int + status: str + evidence_path: Path | None + + +@dataclass(frozen=True) +class InteractionBudget: + planned_trajectories: int + completed_trajectories: int + counted_env_steps: int + target_env_steps: int + + +@dataclass(frozen=True) +class Atari57Artifact: + root: Path + manifest_path: Path + policies: tuple[AtariPolicySpec, ...] + interaction_records: tuple[InteractionRecord, ...] + interaction_budget: InteractionBudget + + +@dataclass(frozen=True) +class AtariEpisode: + env_id: str + obs_mode: str + repeat_index: int + seed: int + return_: float + hns: float + length: int + terminated: bool + truncated: bool + invalid_action: bool + policy_error: str | None + mean_action_latency_ms: float + + +@dataclass(frozen=True) +class HNSReference: + env_id: str + known_best_score: float + random_score: float + human_score: float + + def normalize(self, score: float) -> float: + denominator = self.human_score - self.random_score + if denominator == 0: + raise ValueError(f"Human and random anchors are equal for {self.env_id}") + return (float(score) - self.random_score) / denominator + + +@dataclass(frozen=True) +class Atari57Evaluation: + episodes: tuple[AtariEpisode, ...] + per_game: dict[str, dict[str, Any]] + score: float + mean_best_input_mean_hns: float + median_best_single_run_hns: float + mean_best_single_run_hns: float + + @property + def evaluation_trajectories(self) -> int: + return len(self.episodes) + + @property + def games_evaluated(self) -> int: + return len(self.per_game) + + @property + def invalid_episode_rate(self) -> float: + if not self.episodes: + return 0.0 + return statistics.fmean( + episode.invalid_action or episode.policy_error is not None + for episode in self.episodes + ) + + @property + def counted_evaluation_steps(self) -> int: + return sum(episode.length for episode in self.episodes) + + def to_dict(self) -> dict[str, Any]: + return { + "score": self.score, + "median_best_input_mean_hns": self.score, + "mean_best_input_mean_hns": self.mean_best_input_mean_hns, + "median_best_single_run_hns": self.median_best_single_run_hns, + "mean_best_single_run_hns": self.mean_best_single_run_hns, + "games_evaluated": self.games_evaluated, + "evaluation_trajectories": self.evaluation_trajectories, + "counted_evaluation_steps": self.counted_evaluation_steps, + "invalid_episode_rate": self.invalid_episode_rate, + "per_game": self.per_game, + "episodes": [ + { + "env_id": episode.env_id, + "obs_mode": episode.obs_mode, + "repeat_index": episode.repeat_index, + "seed": episode.seed, + "return": episode.return_, + "hns": episode.hns, + "length": episode.length, + "terminated": episode.terminated, + "truncated": episode.truncated, + "invalid_action": episode.invalid_action, + "policy_error": episode.policy_error, + "mean_action_latency_ms": (episode.mean_action_latency_ms), + } + for episode in self.episodes + ], + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + + +def expected_search_trajectories() -> tuple[SearchTrajectory, ...]: + return tuple( + SearchTrajectory( + env_id=env_id, + obs_mode=obs_mode, + repeat_index=repeat_index, + ) + for env_id in ATARI57_GAMES + for obs_mode in OBSERVATION_MODES + for repeat_index in SEARCH_REPEAT_INDICES + ) + + +def load_hns_references( + path: str | Path, +) -> dict[str, HNSReference]: + table_path = Path(path) + with table_path.open(newline="") as handle: + rows = list(csv.DictReader(handle)) + required_fields = { + "env_id", + "known_best_score", + "random_score", + "human_score", + } + if not rows or not required_fields.issubset(rows[0]): + raise ValueError(f"{table_path} must contain {sorted(required_fields)}") + + references: dict[str, HNSReference] = {} + for row in rows: + env_id = row["env_id"] + if env_id not in ATARI57_GAMES: + raise ValueError(f"Unknown Atari57 game in HNS table: {env_id}") + if env_id in references: + raise ValueError(f"Duplicate HNS row for {env_id}") + reference = HNSReference( + env_id=env_id, + known_best_score=float(row["known_best_score"]), + random_score=float(row["random_score"]), + human_score=float(row["human_score"]), + ) + if reference.human_score == reference.random_score: + raise ValueError(f"Degenerate HNS anchors for {env_id}") + references[env_id] = reference + + missing = set(ATARI57_GAMES) - set(references) + if missing: + raise ValueError( + "HNS table is missing Atari57 games: " + ", ".join(sorted(missing)) + ) + return references + + +def aggregate_atari57_episodes( + episodes: tuple[AtariEpisode, ...] | list[AtariEpisode], +) -> Atari57Evaluation: + configured_episodes = tuple(episodes) + if not configured_episodes: + raise ValueError("At least one Atari episode is required") + + grouped: dict[tuple[str, str], list[AtariEpisode]] = defaultdict(list) + seen: set[tuple[str, str, int]] = set() + for episode in configured_episodes: + if episode.env_id not in ATARI57_GAMES: + raise ValueError(f"Unknown Atari57 game: {episode.env_id}") + if episode.obs_mode not in OBSERVATION_MODES: + raise ValueError(f"Unknown observation mode: {episode.obs_mode}") + key = (episode.env_id, episode.obs_mode, episode.repeat_index) + if key in seen: + raise ValueError(f"Duplicate evaluation trajectory: {key}") + seen.add(key) + grouped[(episode.env_id, episode.obs_mode)].append(episode) + + games: dict[str, dict[str, Any]] = {} + for env_id in ATARI57_GAMES: + modes: dict[str, dict[str, Any]] = {} + game_episodes: list[AtariEpisode] = [] + for obs_mode in OBSERVATION_MODES: + mode_episodes = grouped.get((env_id, obs_mode), []) + if not mode_episodes: + continue + game_episodes.extend(mode_episodes) + modes[obs_mode] = { + "mean_return": statistics.fmean( + episode.return_ for episode in mode_episodes + ), + "mean_hns": statistics.fmean(episode.hns for episode in mode_episodes), + "repeat_count": len(mode_episodes), + } + if not modes: + continue + best_input_mode = max( + modes, + key=lambda mode: modes[mode]["mean_hns"], + ) + games[env_id] = { + "modes": modes, + "best_input_mode": best_input_mode, + "best_input_mean_hns": modes[best_input_mode]["mean_hns"], + "best_single_run_hns": max(episode.hns for episode in game_episodes), + } + + best_input_values = [game["best_input_mean_hns"] for game in games.values()] + best_single_values = [game["best_single_run_hns"] for game in games.values()] + return Atari57Evaluation( + episodes=configured_episodes, + per_game=games, + score=float(statistics.median(best_input_values)), + mean_best_input_mean_hns=statistics.fmean(best_input_values), + median_best_single_run_hns=float(statistics.median(best_single_values)), + mean_best_single_run_hns=statistics.fmean(best_single_values), + ) + + +def _load_policy_module(path: Path) -> ModuleType: + module_name = f"genesisbench_atari57_submission_{abs(hash(path.resolve()))}" + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to import policy from {path}") + module = importlib.util.module_from_spec(spec) + previous = sys.dont_write_bytecode + sys.dont_write_bytecode = True + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + finally: + sys.dont_write_bytecode = previous + return module + + +def _call_with_supported_kwargs(callable_: Any, kwargs: dict[str, Any]) -> Any: + try: + signature = inspect.signature(callable_) + except (TypeError, ValueError): + return callable_(**kwargs) + if any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ): + return callable_(**kwargs) + supported = { + name: value for name, value in kwargs.items() if name in signature.parameters + } + return callable_(**supported) + + +def _instantiate_policy( + module: ModuleType, + *, + spec: AtariPolicySpec, + action_count: int, + seed: int, +) -> Any: + kwargs = { + "env_id": spec.env_id, + "obs_mode": spec.obs_mode, + "repeat_index": spec.repeat_index, + "action_count": action_count, + "seed": seed, + "config": dict(spec.config), + } + if hasattr(module, "make_policy"): + return _call_with_supported_kwargs(module.make_policy, kwargs) + if hasattr(module, "Policy"): + return _call_with_supported_kwargs(module.Policy, kwargs) + raise AttributeError(f"{spec.module_path} must define Policy or make_policy") + + +def _reset_policy(policy: Any, seed: int) -> None: + reset = getattr(policy, "reset", None) + if reset is not None: + _call_with_supported_kwargs(reset, {"seed": seed}) + + +def _policy_action(policy: Any, observation: Any, info: Any) -> Any: + act = getattr(policy, "act", None) + if act is None: + raise AttributeError("Policy must define act") + try: + signature = inspect.signature(act) + except (TypeError, ValueError): + return act(observation, info) + parameters = signature.parameters + if "info" in parameters or any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in parameters.values() + ): + return act(observation, info=info) + positional = [ + parameter + for parameter in parameters.values() + if parameter.kind + in { + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + } + ] + if len(positional) >= 2: + return act(observation, info) + return act(observation) + + +def _unbatch(value: Any) -> Any: + array = np.asarray(value) + if array.ndim >= 1 and array.shape[0] == 1: + return array[0] + if array.ndim == 0: + return array.item() + return value + + +def _unbatch_info(value: object) -> dict[str, Any]: + if not isinstance(value, dict): + return {} + return {key: _unbatch(item) for key, item in value.items()} + + +def _parse_reset(value: Any) -> tuple[Any, dict[str, Any]]: + if isinstance(value, tuple) and len(value) == 2 and isinstance(value[1], dict): + return _unbatch(value[0]), _unbatch_info(value[1]) + return _unbatch(value), {} + + +def _parse_step( + value: Any, +) -> tuple[Any, float, bool, bool, dict[str, Any]]: + if not isinstance(value, tuple): + raise TypeError("Environment step must return a tuple") + if len(value) == 4: + observation, reward, done, info = value + return ( + _unbatch(observation), + float(_unbatch(reward)), + bool(_unbatch(done)), + False, + _unbatch_info(info), + ) + if len(value) == 5: + observation, reward, terminated, truncated, info = value + return ( + _unbatch(observation), + float(_unbatch(reward)), + bool(_unbatch(terminated)), + bool(_unbatch(truncated)), + _unbatch_info(info), + ) + raise TypeError("Environment step must return 4 (Gym) or 5 (Gymnasium) values") + + +def _action_count(action_space: Any) -> int: + if hasattr(action_space, "n"): + value = np.asarray(action_space.n) + return int(value.reshape(-1)[0]) + if hasattr(action_space, "nvec"): + value = np.asarray(action_space.nvec) + return int(value.reshape(-1)[0]) + raise TypeError("Atari action space must expose n or nvec") + + +def _validate_action(action: Any, action_count: int) -> int: + array = np.asarray(action) + if array.size != 1: + raise ValueError(f"Expected one discrete action, got shape {array.shape}") + scalar = array.reshape(-1)[0] + if isinstance(scalar, (np.floating, float)) and not float(scalar).is_integer(): + raise ValueError(f"Action must be an integer, got {scalar!r}") + integer = int(scalar) + if integer < 0 or integer >= action_count: + raise ValueError(f"Action {integer} is outside [0, {action_count})") + return integer + + +def _default_env_factory( + env_id: str, + seed: int, + settings: dict[str, object], +) -> Any: + try: + import envpool + except ImportError as error: + raise RuntimeError( + "EnvPool is required for Atari execution. Use the task Dockerfile " + "or install envpool==1.1.1." + ) from error + if envpool.__version__ != ENVPOOL_VERSION: + raise RuntimeError( + f"Expected envpool=={ENVPOOL_VERSION}, got {envpool.__version__}" + ) + return envpool.make_gym( + env_id, + num_envs=1, + batch_size=1, + seed=seed, + **settings, + ) + + +EnvFactory = Callable[[str, int, dict[str, object]], Any] + + +class _DeterministicActionSpace: + n = 4 + + +class DeterministicAtariTestEnv: + """Tiny deterministic vector environment used only for contract tests.""" + + action_space = _DeterministicActionSpace() + + def __init__(self, env_id: str, seed: int) -> None: + self.env_id = env_id + self.seed = seed + self.steps = 0 + + def _observation(self) -> np.ndarray: + game_index = ATARI57_GAMES.index(self.env_id) + return np.asarray( + [[game_index, self.seed % 997, self.steps]], + dtype=np.int64, + ) + + def _info(self) -> dict[str, np.ndarray]: + return { + "ram": np.asarray( + [[self.steps % 256, self.seed % 256]], + dtype=np.uint8, + ) + } + + def reset(self) -> tuple[np.ndarray, dict[str, np.ndarray]]: + self.steps = 0 + return self._observation(), self._info() + + def step( + self, + action: np.ndarray, + ) -> tuple[ + np.ndarray, + np.ndarray, + np.ndarray, + dict[str, np.ndarray], + ]: + integer = int(np.asarray(action).reshape(-1)[0]) + self.steps += 1 + return ( + self._observation(), + np.asarray([float(integer)]), + np.asarray([False]), + self._info(), + ) + + def close(self) -> None: + return None + + +def deterministic_test_env_factory( + env_id: str, + seed: int, + settings: dict[str, object], +) -> DeterministicAtariTestEnv: + if settings != ATARI_SETTINGS: + raise ValueError("Deterministic test backend requires fixed Atari settings") + return DeterministicAtariTestEnv(env_id, seed) + + +def evaluate_atari57_artifact( + artifact: str | Path | Atari57Artifact, + *, + games: Iterable[str], + obs_modes: Iterable[str], + seeds: Iterable[int], + max_steps: int, + hns_references: dict[str, HNSReference], + env_factory: EnvFactory | None = None, +) -> Atari57Evaluation: + configured_artifact = ( + artifact + if isinstance(artifact, Atari57Artifact) + else load_atari57_artifact(artifact) + ) + configured_games = tuple(games) + configured_modes = tuple(obs_modes) + configured_seeds = tuple(seeds) + if not configured_games: + raise ValueError("At least one game is required") + if not configured_modes: + raise ValueError("At least one observation mode is required") + if not configured_seeds: + raise ValueError("At least one seed is required") + if len(configured_seeds) > len(SEARCH_REPEAT_INDICES): + raise ValueError("Atari57 evaluation supports at most three repeat seeds") + if max_steps <= 0: + raise ValueError("max_steps must be positive") + for env_id in configured_games: + if env_id not in ATARI57_GAMES: + raise ValueError(f"Unknown Atari57 game: {env_id}") + if env_id not in hns_references: + raise ValueError(f"Missing HNS reference for {env_id}") + for obs_mode in configured_modes: + if obs_mode not in OBSERVATION_MODES: + raise ValueError(f"Unknown observation mode: {obs_mode}") + + policies = { + (policy.env_id, policy.obs_mode, policy.repeat_index): policy + for policy in configured_artifact.policies + } + make_env = env_factory or _default_env_factory + episodes: list[AtariEpisode] = [] + + for env_id in configured_games: + reference = hns_references[env_id] + for obs_mode in configured_modes: + for repeat_index, seed in enumerate(configured_seeds): + policy_spec = policies[(env_id, obs_mode, repeat_index)] + module = _load_policy_module(policy_spec.module_path) + env = make_env(env_id, seed, dict(ATARI_SETTINGS)) + action_count = _action_count(env.action_space) + policy = _instantiate_policy( + module, + spec=policy_spec, + action_count=action_count, + seed=seed, + ) + _reset_policy(policy, seed) + observation, info = _parse_reset(env.reset()) + episode_return = 0.0 + terminated = False + truncated = False + invalid_action = False + policy_error: str | None = None + latencies: list[float] = [] + length = 0 + + try: + for _ in range(max_steps): + policy_info = ( + {"ram": info["ram"]} + if obs_mode == "ram" and "ram" in info + else ({} if obs_mode == "ram" else None) + ) + started = time.perf_counter() + try: + action = _policy_action( + policy, + observation, + policy_info, + ) + action = _validate_action( + action, + action_count, + ) + except Exception as error: + invalid_action = isinstance(error, ValueError) + policy_error = f"{type(error).__name__}: {error}" + episode_return = reference.random_score + break + finally: + latencies.append((time.perf_counter() - started) * 1000.0) + + ( + observation, + reward, + terminated, + truncated, + info, + ) = _parse_step(env.step(np.asarray([action], dtype=np.int32))) + length += 1 + episode_return += reward + if terminated or truncated: + break + else: + truncated = True + finally: + close = getattr(env, "close", None) + if close is not None: + close() + + episodes.append( + AtariEpisode( + env_id=env_id, + obs_mode=obs_mode, + repeat_index=repeat_index, + seed=seed, + return_=episode_return, + hns=reference.normalize(episode_return), + length=length, + terminated=terminated, + truncated=truncated, + invalid_action=invalid_action, + policy_error=policy_error, + mean_action_latency_ms=( + statistics.fmean(latencies) if latencies else 0.0 + ), + ) + ) + + return aggregate_atari57_episodes(episodes) + + +def _mapping(value: object, field: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise Atari57ArtifactError(f"{field} must be an object") + return value + + +def _safe_relative_path(value: object, field: str) -> Path: + if not isinstance(value, str) or not value: + raise Atari57ArtifactError(f"{field} must be a non-empty string") + path = Path(value) + if path.is_absolute() or ".." in path.parts: + raise Atari57ArtifactError(f"{field} must be a safe relative path") + return path + + +def _resolve_under_root(root: Path, relative: Path, field: str) -> Path: + resolved_root = root.resolve() + resolved = (resolved_root / relative).resolve() + if not resolved.is_relative_to(resolved_root): + raise Atari57ArtifactError(f"{field} resolves outside the artifact") + return resolved + + +def _load_manifest(path: str | Path) -> tuple[Path, Path, dict[str, Any]]: + candidate = Path(path).resolve() + manifest_path = candidate / "manifest.json" if candidate.is_dir() else candidate + if not manifest_path.is_file(): + raise FileNotFoundError(manifest_path) + try: + manifest = json.loads(manifest_path.read_text()) + except json.JSONDecodeError as error: + raise Atari57ArtifactError( + f"manifest.json is not valid JSON: {error}" + ) from error + return manifest_path.parent, manifest_path, _mapping(manifest, "manifest") + + +def _validate_protocol(manifest: dict[str, Any]) -> None: + if manifest.get("schema_version") != "1.0": + raise Atari57ArtifactError("schema_version must be '1.0'") + protocol = _mapping(manifest.get("protocol"), "protocol") + expected = { + "suite": "Atari57", + "envpool_version": ENVPOOL_VERSION, + "observation_modes": list(OBSERVATION_MODES), + "search_repeats": len(SEARCH_REPEAT_INDICES), + "frame_budget_per_search": FRAME_BUDGET_PER_SEARCH, + "expected_search_trajectories": len(expected_search_trajectories()), + "expected_policy_slots": len(expected_search_trajectories()), + "expected_total_frame_target": ( + len(expected_search_trajectories()) * FRAME_BUDGET_PER_SEARCH + ), + "atari_settings": ATARI_SETTINGS, + } + for field, expected_value in expected.items(): + if protocol.get(field) != expected_value: + raise Atari57ArtifactError( + f"protocol.{field} must equal {expected_value!r}" + ) + + +def _policy_specs( + root: Path, + manifest: dict[str, Any], +) -> tuple[AtariPolicySpec, ...]: + defaults = _mapping(manifest.get("policy_defaults"), "policy_defaults") + overrides_value = manifest.get("policies", []) + if not isinstance(overrides_value, list): + raise Atari57ArtifactError("policies must be an array") + + overrides: dict[tuple[str, str, int], dict[str, Any]] = {} + for index, value in enumerate(overrides_value): + override = _mapping(value, f"policies[{index}]") + env_id = override.get("env_id") + obs_mode = override.get("obs_mode") + repeat_index = override.get("repeat_index") + key = (env_id, obs_mode, repeat_index) + if env_id not in ATARI57_GAMES: + raise Atari57ArtifactError(f"policies[{index}].env_id is not in Atari57") + if obs_mode not in OBSERVATION_MODES: + raise Atari57ArtifactError(f"policies[{index}].obs_mode is invalid") + if repeat_index not in SEARCH_REPEAT_INDICES: + raise Atari57ArtifactError(f"policies[{index}].repeat_index is invalid") + if key in overrides: + raise Atari57ArtifactError(f"duplicate policy override for {key}") + overrides[key] = override + + policies: list[AtariPolicySpec] = [] + for env_id in ATARI57_GAMES: + for obs_mode in OBSERVATION_MODES: + for repeat_index in SEARCH_REPEAT_INDICES: + override = overrides.get((env_id, obs_mode, repeat_index), {}) + field = f"policy module for {env_id}/{obs_mode}/{repeat_index}" + module = _safe_relative_path( + override.get("module", defaults.get("module")), + field, + ) + module_path = _resolve_under_root(root, module, field) + if not module_path.is_file(): + raise Atari57ArtifactError( + f"policy module does not exist: {module}" + ) + default_config = defaults.get("config", {}) + override_config = override.get("config", {}) + config = { + **_mapping(default_config, "policy_defaults.config"), + **_mapping( + override_config, + f"policy config for {env_id}/{obs_mode}/{repeat_index}", + ), + } + policies.append( + AtariPolicySpec( + env_id=env_id, + obs_mode=obs_mode, + repeat_index=repeat_index, + module_path=module_path, + config=config, + ) + ) + return tuple(policies) + + +def _interaction_records( + root: Path, + manifest: dict[str, Any], +) -> tuple[InteractionRecord, ...]: + ledger_path = _resolve_under_root( + root, + _safe_relative_path( + manifest.get("interaction_ledger"), + "interaction_ledger", + ), + "interaction_ledger", + ) + if not ledger_path.is_file(): + raise Atari57ArtifactError( + f"interaction ledger does not exist: {ledger_path.name}" + ) + try: + ledger = json.loads(ledger_path.read_text()) + except json.JSONDecodeError as error: + raise Atari57ArtifactError( + f"interaction ledger is not valid JSON: {error}" + ) from error + ledger_mapping = _mapping(ledger, "interaction ledger") + if ledger_mapping.get("schema_version") != "1.0": + raise Atari57ArtifactError("interaction ledger schema_version must be '1.0'") + values = ledger_mapping.get("records") + if not isinstance(values, list): + raise Atari57ArtifactError("interaction ledger records must be an array") + + expected_keys = { + (trajectory.env_id, trajectory.obs_mode, trajectory.repeat_index) + for trajectory in expected_search_trajectories() + } + records: dict[tuple[str, str, int], InteractionRecord] = {} + completed_evidence_paths: set[Path] = set() + for index, value in enumerate(values): + item = _mapping(value, f"interaction ledger records[{index}]") + env_id = item.get("env_id") + obs_mode = item.get("obs_mode") + repeat_index = item.get("repeat_index") + key = (env_id, obs_mode, repeat_index) + if key not in expected_keys: + raise Atari57ArtifactError( + f"interaction ledger records[{index}] has invalid trajectory" + ) + if key in records: + raise Atari57ArtifactError(f"duplicate interaction ledger record for {key}") + steps = item.get("cumulative_env_steps") + episodes = item.get("cumulative_episodes") + status = item.get("status") + if not isinstance(steps, int) or steps < 0: + raise Atari57ArtifactError( + "cumulative_env_steps must be a non-negative integer" + ) + if not isinstance(episodes, int) or episodes < 0: + raise Atari57ArtifactError( + "cumulative_episodes must be a non-negative integer" + ) + if status not in {"not_run", "running", "complete", "failed"}: + raise Atari57ArtifactError( + "status must be not_run, running, complete, or failed" + ) + if status == "complete" and steps < FRAME_BUDGET_PER_SEARCH: + raise Atari57ArtifactError( + "complete search trajectories must count at least " + f"{FRAME_BUDGET_PER_SEARCH} environment steps" + ) + evidence_path: Path | None = None + evidence_value = item.get("evidence_path") + if evidence_value is not None: + evidence_path = _resolve_under_root( + root, + _safe_relative_path( + evidence_value, + f"interaction ledger records[{index}].evidence_path", + ), + f"interaction ledger records[{index}].evidence_path", + ) + if status == "complete": + missing = ( + list(ARTICLE_SEARCH_EVIDENCE_FILES) + if evidence_path is None + else [ + filename + for filename in ARTICLE_SEARCH_EVIDENCE_FILES + if not (evidence_path / filename).is_file() + or not (evidence_path / filename) + .resolve() + .is_relative_to(root.resolve()) + ] + ) + if missing: + raise Atari57ArtifactError( + "complete search evidence is missing: " + ", ".join(missing) + ) + if evidence_path in completed_evidence_paths: + raise Atari57ArtifactError( + "complete search trajectories must use distinct evidence paths" + ) + completed_evidence_paths.add(evidence_path) + records[key] = InteractionRecord( + env_id=env_id, + obs_mode=obs_mode, + repeat_index=repeat_index, + cumulative_env_steps=steps, + cumulative_episodes=episodes, + status=status, + evidence_path=evidence_path, + ) + + return tuple(records.values()) + + +def load_atari57_artifact(path: str | Path) -> Atari57Artifact: + root, manifest_path, manifest = _load_manifest(path) + _validate_protocol(manifest) + policies = _policy_specs(root, manifest) + records = _interaction_records(root, manifest) + expected_count = len(expected_search_trajectories()) + counted_steps = sum(record.cumulative_env_steps for record in records) + budget = InteractionBudget( + planned_trajectories=expected_count, + completed_trajectories=sum(record.status == "complete" for record in records), + counted_env_steps=counted_steps, + target_env_steps=expected_count * FRAME_BUDGET_PER_SEARCH, + ) + return Atari57Artifact( + root=root, + manifest_path=manifest_path, + policies=policies, + interaction_records=records, + interaction_budget=budget, + ) diff --git a/src/genesisbench/breakout.py b/src/genesisbench/breakout.py new file mode 100644 index 0000000..88b482c --- /dev/null +++ b/src/genesisbench/breakout.py @@ -0,0 +1,384 @@ +from __future__ import annotations + +import importlib.util +import inspect +import json +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from types import ModuleType +from typing import Any, Iterable, Literal + +import numpy as np + + +ObservationMode = Literal["ram", "rgb"] + + +@dataclass(frozen=True) +class BreakoutVariant: + name: str = "nominal" + repeat_action_probability: float = 0.0 + initial_actions: tuple[int, ...] = () + + +@dataclass(frozen=True) +class BreakoutEpisode: + seed: int + variant: str + observation_mode: ObservationMode + return_: float + length: int + warmup_steps: int + final_lives: int + terminated: bool + truncated: bool + invalid_action: bool + policy_error: str | None + mean_action_latency_ms: float + + +@dataclass(frozen=True) +class BreakoutEvaluation: + policy_path: str + observation_mode: ObservationMode + max_steps: int + episodes: tuple[BreakoutEpisode, ...] + + @property + def mean_return(self) -> float: + return float(np.mean([episode.return_ for episode in self.episodes])) + + @property + def min_return(self) -> float: + return float(np.min([episode.return_ for episode in self.episodes])) + + @property + def max_return(self) -> float: + return float(np.max([episode.return_ for episode in self.episodes])) + + @property + def completion_rate(self) -> float: + return float( + np.mean([episode.return_ >= 864.0 for episode in self.episodes]) + ) + + @property + def invalid_episode_rate(self) -> float: + return float( + np.mean( + [ + episode.invalid_action or episode.policy_error is not None + for episode in self.episodes + ] + ) + ) + + @property + def mean_action_latency_ms(self) -> float: + return float( + np.mean( + [episode.mean_action_latency_ms for episode in self.episodes] + ) + ) + + def to_dict(self) -> dict[str, Any]: + rendered_episodes = [] + for episode in self.episodes: + rendered_episode = asdict(episode) + rendered_episode["return"] = rendered_episode.pop("return_") + rendered_episodes.append(rendered_episode) + return { + "policy_path": self.policy_path, + "observation_mode": self.observation_mode, + "max_steps": self.max_steps, + "score": self.mean_return, + "mean_return": self.mean_return, + "min_return": self.min_return, + "max_return": self.max_return, + "completion_rate": self.completion_rate, + "invalid_episode_rate": self.invalid_episode_rate, + "mean_action_latency_ms": self.mean_action_latency_ms, + "episodes": rendered_episodes, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + + +def _load_policy_module(policy_path: Path) -> ModuleType: + module_name = ( + f"genesisbench_breakout_submission_{abs(hash(policy_path.resolve()))}" + ) + spec = importlib.util.spec_from_file_location(module_name, policy_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to import policy from {policy_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + return module + + +def _instantiate_policy(module: ModuleType, seed: int) -> Any: + if hasattr(module, "make_policy"): + factory = module.make_policy + try: + return factory(seed=seed) + except TypeError: + return factory() + if hasattr(module, "Policy"): + policy_class = module.Policy + try: + return policy_class(seed=seed) + except TypeError: + return policy_class() + raise AttributeError("Submission must define Policy or make_policy") + + +def _reset_policy(policy: Any, seed: int) -> None: + reset = getattr(policy, "reset", None) + if reset is None: + return + try: + signature = inspect.signature(reset) + except (TypeError, ValueError): + signature = None + if signature is not None and "seed" in signature.parameters: + reset(seed=seed) + else: + reset() + + +def _validate_action(action: Any) -> int: + array = np.asarray(action) + if array.shape == (1,): + array = array.reshape(()) + if array.shape != (): + raise ValueError(f"Expected a scalar action, got shape {array.shape}") + value = float(array) + if not np.isfinite(value): + raise ValueError("Action is NaN or infinite") + if not value.is_integer(): + raise ValueError(f"Action must be an integer, got {value}") + action_id = int(value) + if action_id not in (0, 1, 2, 3): + raise ValueError(f"Action must be one of 0, 1, 2, 3; got {action_id}") + return action_id + + +def _make_breakout_env( + *, + variant: BreakoutVariant, + max_steps: int, + seed: int, +) -> Any: + try: + import envpool + from importlib.metadata import version + except ImportError as error: + raise RuntimeError( + "Breakout evaluation requires envpool==1.1.1" + ) from error + installed_version = version("envpool") + if installed_version != "1.1.1": + raise RuntimeError( + f"Breakout evaluation requires envpool==1.1.1, got {installed_version}" + ) + + env = envpool.make_gym( + "Breakout-v5", + num_envs=1, + batch_size=1, + seed=seed, + max_episode_steps=max_steps + len(variant.initial_actions), + img_height=210, + img_width=160, + stack_num=1, + gray_scale=False, + frame_skip=1, + noop_max=1, + use_fire_reset=True, + episodic_life=False, + reward_clip=False, + repeat_action_probability=variant.repeat_action_probability, + full_action_space=False, + ) + if int(env.action_space.n) != 4: + raise RuntimeError(f"Expected four Breakout actions, got {env.action_space}") + return env + + +def _reset_env(env: Any) -> tuple[np.ndarray, dict[str, Any]]: + result = env.reset() + if not isinstance(result, tuple) or len(result) != 2: + raise RuntimeError("EnvPool 1.1.1 reset must return (observation, info)") + observation, info = result + return np.asarray(observation), info + + +def _step_env( + env: Any, + action: int, +) -> tuple[np.ndarray, float, bool, bool, dict[str, Any]]: + result = env.step(np.asarray([action], dtype=np.int32)) + if len(result) != 5: + raise RuntimeError("EnvPool 1.1.1 step must return five values") + observation, reward, terminated, truncated, info = result + return ( + np.asarray(observation), + float(np.asarray(reward)[0]), + bool(np.asarray(terminated)[0]), + bool(np.asarray(truncated)[0]), + info, + ) + + +def _policy_observation( + observation: np.ndarray, + info: dict[str, Any], + mode: ObservationMode, +) -> np.ndarray: + if mode == "ram": + ram = np.asarray(info["ram"], dtype=np.uint8) + if ram.shape != (1, 128): + raise RuntimeError(f"Unexpected EnvPool RAM shape: {ram.shape}") + return ram[0].copy() + + pixels = np.asarray(observation, dtype=np.uint8) + if pixels.shape != (1, 3, 210, 160): + raise RuntimeError(f"Unexpected EnvPool RGB shape: {pixels.shape}") + return pixels[0].copy() + + +def _info_lives(info: dict[str, Any]) -> int: + lives = np.asarray(info.get("lives", (0,))) + return int(lives.reshape(-1)[0]) if lives.size else 0 + + +def _apply_initial_actions( + env: Any, + observation: np.ndarray, + info: dict[str, Any], + actions: tuple[int, ...], +) -> tuple[np.ndarray, dict[str, Any], float]: + warmup_return = 0.0 + for action in actions: + action_id = _validate_action(action) + observation, reward, terminated, truncated, info = _step_env( + env, + action_id, + ) + warmup_return += reward + if terminated or truncated: + raise RuntimeError( + "Breakout variant terminated during its initial action prefix" + ) + return observation, info, warmup_return + + +def evaluate_breakout_policy( + policy_path: str | Path, + *, + observation_mode: ObservationMode, + seeds: Iterable[int], + max_steps: int = 108000, + variants: Iterable[BreakoutVariant] | None = None, + failure_return: float = -1.0, +) -> BreakoutEvaluation: + path = Path(policy_path).resolve() + if path.is_dir(): + path = path / "policy.py" + if not path.is_file(): + raise FileNotFoundError(path) + if observation_mode not in ("ram", "rgb"): + raise ValueError(f"Unsupported observation mode: {observation_mode}") + + module = _load_policy_module(path) + configured_variants = tuple(variants or (BreakoutVariant(),)) + configured_seeds = tuple(int(seed) for seed in seeds) + episodes: list[BreakoutEpisode] = [] + + for variant in configured_variants: + for seed in configured_seeds: + env = _make_breakout_env( + variant=variant, + max_steps=max_steps, + seed=seed, + ) + observation, info = _reset_env(env) + observation, info, episode_return = _apply_initial_actions( + env, + observation, + info, + variant.initial_actions, + ) + policy = _instantiate_policy(module, seed) + _reset_policy(policy, seed) + terminated = False + truncated = False + invalid_action = False + policy_error: str | None = None + latencies: list[float] = [] + length = 0 + + try: + for length in range(1, max_steps + 1): + started_at = time.perf_counter() + try: + action = _validate_action( + policy.act( + _policy_observation( + observation, + info, + observation_mode, + ) + ) + ) + except Exception as error: + policy_error = f"{type(error).__name__}: {error}" + invalid_action = isinstance(error, ValueError) + episode_return = failure_return + break + latencies.append((time.perf_counter() - started_at) * 1000) + observation, reward, terminated, truncated, info = _step_env( + env, + action, + ) + episode_return += reward + if terminated or truncated: + break + finally: + close = getattr(env, "close", None) + if close is not None: + close() + + episodes.append( + BreakoutEpisode( + seed=seed, + variant=variant.name, + observation_mode=observation_mode, + return_=float(episode_return), + length=length, + warmup_steps=len(variant.initial_actions), + final_lives=_info_lives(info), + terminated=bool(terminated), + truncated=bool(truncated), + invalid_action=invalid_action, + policy_error=policy_error, + mean_action_latency_ms=float(np.mean(latencies)) + if latencies + else 0.0, + ) + ) + + return BreakoutEvaluation( + policy_path=str(path), + observation_mode=observation_mode, + max_steps=max_steps, + episodes=tuple(episodes), + ) diff --git a/src/genesisbench/halfcheetah.py b/src/genesisbench/halfcheetah.py new file mode 100644 index 0000000..da05b58 --- /dev/null +++ b/src/genesisbench/halfcheetah.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import importlib.util +import inspect +import json +import math +import sys +import tempfile +import time +import xml.etree.ElementTree as ET +from dataclasses import asdict, dataclass +from pathlib import Path +from types import ModuleType +from typing import Any, Iterable + +import gymnasium as gym +import numpy as np + + +@dataclass(frozen=True) +class DynamicsVariant: + name: str = "nominal" + mass_scale: float = 1.0 + friction_scale: float = 1.0 + damping_scale: float = 1.0 + actuator_scale: float = 1.0 + + +@dataclass(frozen=True) +class HalfCheetahEpisode: + seed: int + variant: str + return_: float + length: int + x_position: float + x_velocity: float + forward_reward: float + control_cost: float + terminated: bool + truncated: bool + invalid_action: bool + policy_error: str | None + mean_action_latency_ms: float + + +@dataclass(frozen=True) +class HalfCheetahEvaluation: + policy_path: str + max_steps: int + episodes: tuple[HalfCheetahEpisode, ...] + + @property + def mean_return(self) -> float: + return float(np.mean([episode.return_ for episode in self.episodes])) + + @property + def min_return(self) -> float: + return float(np.min([episode.return_ for episode in self.episodes])) + + @property + def max_return(self) -> float: + return float(np.max([episode.return_ for episode in self.episodes])) + + @property + def mean_x_position(self) -> float: + return float(np.mean([episode.x_position for episode in self.episodes])) + + @property + def mean_forward_reward(self) -> float: + return float(np.mean([episode.forward_reward for episode in self.episodes])) + + @property + def mean_control_cost(self) -> float: + return float(np.mean([episode.control_cost for episode in self.episodes])) + + @property + def invalid_episode_rate(self) -> float: + return float( + np.mean( + [ + episode.invalid_action or episode.policy_error is not None + for episode in self.episodes + ] + ) + ) + + @property + def mean_action_latency_ms(self) -> float: + return float( + np.mean([episode.mean_action_latency_ms for episode in self.episodes]) + ) + + def to_dict(self) -> dict[str, Any]: + rendered_episodes: list[dict[str, Any]] = [] + for episode in self.episodes: + rendered = asdict(episode) + rendered["return"] = rendered.pop("return_") + rendered_episodes.append(rendered) + return { + "policy_path": self.policy_path, + "max_steps": self.max_steps, + "score": self.mean_return, + "mean_return": self.mean_return, + "min_return": self.min_return, + "max_return": self.max_return, + "mean_x_position": self.mean_x_position, + "mean_forward_reward": self.mean_forward_reward, + "mean_control_cost": self.mean_control_cost, + "invalid_episode_rate": self.invalid_episode_rate, + "mean_action_latency_ms": self.mean_action_latency_ms, + "episodes": rendered_episodes, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + + +def _load_policy_module(policy_path: Path) -> ModuleType: + module_name = ( + f"genesisbench_halfcheetah_submission_{abs(hash(policy_path.resolve()))}" + ) + spec = importlib.util.spec_from_file_location(module_name, policy_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to import policy from {policy_path}") + + module = importlib.util.module_from_spec(spec) + parent = str(policy_path.parent) + added_to_path = parent not in sys.path + if added_to_path: + sys.path.insert(0, parent) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + finally: + if added_to_path: + sys.path.remove(parent) + return module + + +def _instantiate_policy(module: ModuleType, seed: int) -> Any: + if hasattr(module, "make_policy"): + factory = module.make_policy + try: + return factory(seed=seed) + except TypeError: + return factory() + if hasattr(module, "Policy"): + policy_class = module.Policy + try: + return policy_class(seed=seed) + except TypeError: + return policy_class() + raise AttributeError("Submission must define Policy or make_policy") + + +def _configure_policy( + policy: Any, + *, + model_xml_path: Path, + frame_skip: int, +) -> None: + configure = getattr(policy, "configure_simulator", None) + if configure is None: + return + + available = { + "model_xml_path": str(model_xml_path), + "frame_skip": int(frame_skip), + } + try: + signature = inspect.signature(configure) + except (TypeError, ValueError): + configure(**available) + return + + accepts_kwargs = any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in signature.parameters.values() + ) + kwargs = ( + available + if accepts_kwargs + else { + name: value + for name, value in available.items() + if name in signature.parameters + } + ) + configure(**kwargs) + + +def _reset_policy(policy: Any, seed: int) -> None: + reset = getattr(policy, "reset", None) + if reset is None: + return + try: + signature = inspect.signature(reset) + except (TypeError, ValueError): + signature = None + if signature is not None and "seed" in signature.parameters: + reset(seed=seed) + else: + reset() + + +def _halfcheetah_xml_path() -> Path: + import gymnasium.envs.mujoco + + return ( + Path(gymnasium.envs.mujoco.__file__).resolve().parent + / "assets" + / "half_cheetah.xml" + ) + + +def _scale_vector(value: str, scale: float) -> str: + return " ".join(f"{float(item) * scale:.12g}" for item in value.split()) + + +def _write_variant_xml( + variant: DynamicsVariant, + output_path: Path, +) -> None: + root = ET.parse(_halfcheetah_xml_path()).getroot() + + compiler = root.find("compiler") + if compiler is not None and "settotalmass" in compiler.attrib: + compiler.set( + "settotalmass", + f"{float(compiler.attrib['settotalmass']) * variant.mass_scale:.12g}", + ) + + for geom in root.findall(".//geom"): + if "friction" in geom.attrib: + geom.set( + "friction", + _scale_vector(geom.attrib["friction"], variant.friction_scale), + ) + + for joint in root.findall(".//joint"): + if "damping" in joint.attrib: + joint.set( + "damping", + f"{float(joint.attrib['damping']) * variant.damping_scale:.12g}", + ) + + for motor in root.findall(".//motor"): + if "gear" in motor.attrib: + motor.set( + "gear", + _scale_vector(motor.attrib["gear"], variant.actuator_scale), + ) + + ET.ElementTree(root).write(output_path, encoding="unicode") + + +def _is_nominal(variant: DynamicsVariant) -> bool: + return variant.name == "nominal" and all( + math.isclose(value, 1.0) + for value in ( + variant.mass_scale, + variant.friction_scale, + variant.damping_scale, + variant.actuator_scale, + ) + ) + + +def _make_halfcheetah_env( + *, + variant: DynamicsVariant, + max_steps: int, +) -> tuple[gym.Env, tempfile.TemporaryDirectory[str] | None, Path]: + temporary_directory: tempfile.TemporaryDirectory[str] | None = None + model_xml_path = _halfcheetah_xml_path() + kwargs: dict[str, Any] = {"max_episode_steps": max_steps} + + if not _is_nominal(variant): + temporary_directory = tempfile.TemporaryDirectory( + prefix="genesisbench-simulation-heuristics-halfcheetah-v1-" + ) + model_xml_path = Path(temporary_directory.name) / f"{variant.name}.xml" + _write_variant_xml(variant, model_xml_path) + kwargs["xml_file"] = str(model_xml_path) + + env = gym.make("HalfCheetah-v5", **kwargs) + return env, temporary_directory, model_xml_path + + +def _validate_action(action: Any) -> np.ndarray: + array = np.asarray(action, dtype=np.float64) + if array.shape != (6,): + raise ValueError(f"Expected action shape (6,), got {array.shape}") + if not np.all(np.isfinite(array)): + raise ValueError("Action contains NaN or infinite values") + return np.clip(array, -1.0, 1.0).astype(np.float32) + + +def evaluate_halfcheetah_policy( + policy_path: str | Path, + *, + seeds: Iterable[int], + max_steps: int = 1000, + variants: Iterable[DynamicsVariant] | None = None, + failure_return: float = -1000.0, +) -> HalfCheetahEvaluation: + path = Path(policy_path).resolve() + if path.is_dir(): + path = path / "policy.py" + if not path.is_file(): + raise FileNotFoundError(path) + if max_steps <= 0: + raise ValueError("max_steps must be positive") + + seed_values = tuple(int(seed) for seed in seeds) + if not seed_values: + raise ValueError("At least one seed is required") + + module = _load_policy_module(path) + configured_variants = tuple(variants or (DynamicsVariant(),)) + episodes: list[HalfCheetahEpisode] = [] + + for variant in configured_variants: + for seed in seed_values: + env, temporary_directory, model_xml_path = _make_halfcheetah_env( + variant=variant, + max_steps=max_steps, + ) + observation, _ = env.reset(seed=seed) + episode_return = 0.0 + forward_reward = 0.0 + control_cost = 0.0 + terminated = False + truncated = False + invalid_action = False + policy_error: str | None = None + latencies: list[float] = [] + info: dict[str, Any] = {} + length = 0 + + try: + try: + policy = _instantiate_policy(module, seed) + _configure_policy( + policy, + model_xml_path=model_xml_path, + frame_skip=int(env.unwrapped.frame_skip), + ) + _reset_policy(policy, seed) + except Exception as error: + policy_error = f"{type(error).__name__}: {error}" + episode_return = failure_return + else: + for length in range(1, max_steps + 1): + started_at = time.perf_counter() + try: + action = _validate_action(policy.act(observation)) + except Exception as error: + policy_error = f"{type(error).__name__}: {error}" + invalid_action = isinstance(error, ValueError) + episode_return = failure_return + break + latencies.append((time.perf_counter() - started_at) * 1000) + ( + observation, + reward, + terminated, + truncated, + info, + ) = env.step(action) + episode_return += float(reward) + forward_reward += float(info.get("reward_forward", 0.0)) + control_cost += -float(info.get("reward_ctrl", 0.0)) + if terminated or truncated: + break + finally: + x_position = float(env.unwrapped.data.qpos[0]) + x_velocity = float(env.unwrapped.data.qvel[0]) + env.close() + if temporary_directory is not None: + temporary_directory.cleanup() + + episodes.append( + HalfCheetahEpisode( + seed=seed, + variant=variant.name, + return_=float(episode_return), + length=length, + x_position=x_position, + x_velocity=x_velocity, + forward_reward=float(forward_reward), + control_cost=float(control_cost), + terminated=bool(terminated), + truncated=bool(truncated), + invalid_action=invalid_action, + policy_error=policy_error, + mean_action_latency_ms=float(np.mean(latencies)) + if latencies + else 0.0, + ) + ) + + return HalfCheetahEvaluation( + policy_path=str(path), + max_steps=max_steps, + episodes=tuple(episodes), + ) diff --git a/src/genesisbench/montezuma.py b/src/genesisbench/montezuma.py new file mode 100644 index 0000000..612ed40 --- /dev/null +++ b/src/genesisbench/montezuma.py @@ -0,0 +1,419 @@ +from __future__ import annotations + +import importlib.util +import inspect +import json +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from types import ModuleType +from typing import Any, Iterable + +import numpy as np + + +MONTEZUMA_ENV_ID = "MontezumaRevenge-v5" +MONTEZUMA_ACTION_COUNT = 18 + + +@dataclass(frozen=True) +class MontezumaVariant: + name: str = "canonical" + bootstrap_steps: int = 0 + pre_policy_noops: int = 0 + + @property + def is_recovery(self) -> bool: + return self.bootstrap_steps > 0 or self.pre_policy_noops > 0 + + +@dataclass(frozen=True) +class MontezumaEpisode: + seed: int + variant: str + bootstrap_steps: int + pre_policy_noops: int + return_: float + length: int + policy_steps: int + target_reached: bool + terminated: bool + truncated: bool + invalid_action: bool + policy_error: str | None + mean_action_latency_ms: float + + +@dataclass(frozen=True) +class MontezumaEvaluation: + policy_path: str + max_steps: int + target_score: float + episodes: tuple[MontezumaEpisode, ...] + + @property + def mean_return(self) -> float: + return float(np.mean([episode.return_ for episode in self.episodes])) + + @property + def capped_mean_score(self) -> float: + return float( + np.mean( + [ + np.clip(episode.return_, 0.0, self.target_score) + for episode in self.episodes + ] + ) + ) + + @property + def target_success_rate(self) -> float: + return float(np.mean([episode.target_reached for episode in self.episodes])) + + @property + def recovery_success_rate(self) -> float: + recovery_episodes = [ + episode + for episode in self.episodes + if episode.bootstrap_steps > 0 or episode.pre_policy_noops > 0 + ] + if not recovery_episodes: + return 0.0 + return float(np.mean([episode.target_reached for episode in recovery_episodes])) + + @property + def invalid_episode_rate(self) -> float: + return float( + np.mean( + [ + episode.invalid_action or episode.policy_error is not None + for episode in self.episodes + ] + ) + ) + + @property + def mean_action_latency_ms(self) -> float: + return float( + np.mean([episode.mean_action_latency_ms for episode in self.episodes]) + ) + + def to_dict(self) -> dict[str, Any]: + rendered_episodes = [] + for episode in self.episodes: + rendered_episode = asdict(episode) + rendered_episode["return"] = rendered_episode.pop("return_") + rendered_episodes.append(rendered_episode) + return { + "policy_path": self.policy_path, + "max_steps": self.max_steps, + "target_score": self.target_score, + "score": self.capped_mean_score, + "mean_return": self.mean_return, + "capped_mean_score": self.capped_mean_score, + "target_success_rate": self.target_success_rate, + "recovery_success_rate": self.recovery_success_rate, + "invalid_episode_rate": self.invalid_episode_rate, + "mean_action_latency_ms": self.mean_action_latency_ms, + "episodes": rendered_episodes, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + + +def _resolve_policy_path(policy_path: str | Path) -> Path: + path = Path(policy_path).resolve() + if path.is_dir(): + path = path / "policy.py" + if not path.is_file(): + raise FileNotFoundError(path) + return path + + +def _load_policy_module(policy_path: Path) -> ModuleType: + module_name = f"genesisbench_montezuma_submission_{abs(hash(policy_path))}" + spec = importlib.util.spec_from_file_location(module_name, policy_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to import policy from {policy_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + return module + + +def _instantiate_policy(module: ModuleType, seed: int) -> Any: + if hasattr(module, "make_policy"): + factory = module.make_policy + try: + return factory(seed=seed) + except TypeError: + return factory() + if hasattr(module, "Policy"): + policy_class = module.Policy + try: + return policy_class(seed=seed) + except TypeError: + return policy_class() + raise AttributeError("Submission must define Policy or make_policy") + + +def _reset_policy(policy: Any, seed: int) -> None: + reset = getattr(policy, "reset", None) + if reset is None: + return + try: + signature = inspect.signature(reset) + except (TypeError, ValueError): + signature = None + if signature is not None and "seed" in signature.parameters: + reset(seed=seed) + else: + reset() + + +def _make_montezuma_env(*, seed: int, max_steps: int) -> Any: + try: + import envpool + except ImportError as error: + raise RuntimeError("Montezuma evaluation requires envpool==1.1.1") from error + + return envpool.make_gym( + MONTEZUMA_ENV_ID, + num_envs=1, + batch_size=1, + num_threads=1, + seed=seed, + max_episode_steps=max_steps, + img_height=210, + img_width=160, + stack_num=1, + gray_scale=False, + frame_skip=1, + noop_max=1, + use_fire_reset=True, + episodic_life=False, + reward_clip=False, + repeat_action_probability=0.0, + full_action_space=False, + ) + + +def _reset_env(env: Any) -> np.ndarray: + result = env.reset() + observation = result[0] if isinstance(result, tuple) else result + return _native_frame(observation) + + +def _native_frame(observation: Any) -> np.ndarray: + array = np.asarray(observation) + if array.ndim == 4 and array.shape[0] == 1: + array = array[0] + if array.ndim != 3: + raise ValueError( + "Expected one native RGB image with three dimensions, " + f"got shape {array.shape}" + ) + if array.dtype != np.uint8: + array = array.astype(np.uint8) + return np.array(array, copy=True) + + +def _step_env( + env: Any, + action: int, +) -> tuple[np.ndarray, float, bool, bool]: + result = env.step(np.asarray([action], dtype=np.int32)) + if len(result) == 5: + observation, reward, terminated, truncated, _ = result + terminated_value = bool(np.asarray(terminated).reshape(-1)[0]) + truncated_value = bool(np.asarray(truncated).reshape(-1)[0]) + else: + observation, reward, done, _ = result + terminated_value = bool(np.asarray(done).reshape(-1)[0]) + truncated_value = False + return ( + _native_frame(observation), + float(np.asarray(reward).reshape(-1)[0]), + terminated_value, + truncated_value, + ) + + +def _validate_action(action: Any) -> int: + array = np.asarray(action) + if array.shape != (): + raise ValueError(f"Expected one scalar action, got shape {array.shape}") + value = array.item() + if isinstance(value, bool) or not isinstance(value, (int, np.integer)): + raise ValueError(f"Expected an integer action, got {value!r}") + action_value = int(value) + if not 0 <= action_value < MONTEZUMA_ACTION_COUNT: + raise ValueError( + f"Action must be in [0, {MONTEZUMA_ACTION_COUNT - 1}], got {action_value}" + ) + return action_value + + +def _trusted_action(policy: Any, observation: np.ndarray) -> int: + try: + return _validate_action(policy.act(observation)) + except Exception as error: + raise RuntimeError( + f"Trusted bootstrap policy failed: {type(error).__name__}: {error}" + ) from error + + +def _validate_variant(variant: MontezumaVariant) -> None: + if variant.bootstrap_steps < 0: + raise ValueError("bootstrap_steps must be non-negative") + if variant.pre_policy_noops < 0: + raise ValueError("pre_policy_noops must be non-negative") + + +def evaluate_montezuma_policy( + policy_path: str | Path, + *, + seeds: Iterable[int], + max_steps: int = 2600, + variants: Iterable[MontezumaVariant] | None = None, + bootstrap_policy_path: str | Path | None = None, + target_score: float = 400.0, + failure_score: float = 0.0, +) -> MontezumaEvaluation: + if max_steps <= 0: + raise ValueError("max_steps must be positive") + if target_score <= 0: + raise ValueError("target_score must be positive") + + path = _resolve_policy_path(policy_path) + module = _load_policy_module(path) + configured_seeds = tuple(int(seed) for seed in seeds) + configured_variants = tuple(variants or (MontezumaVariant(),)) + if not configured_seeds: + raise ValueError("At least one seed is required") + if not configured_variants: + raise ValueError("At least one variant is required") + for variant in configured_variants: + _validate_variant(variant) + + needs_bootstrap = any( + variant.bootstrap_steps > 0 for variant in configured_variants + ) + bootstrap_module: ModuleType | None = None + if needs_bootstrap: + if bootstrap_policy_path is None: + raise ValueError("bootstrap_policy_path is required for recovery variants") + bootstrap_module = _load_policy_module( + _resolve_policy_path(bootstrap_policy_path) + ) + + episodes: list[MontezumaEpisode] = [] + for variant in configured_variants: + for seed in configured_seeds: + env = _make_montezuma_env(seed=seed, max_steps=max_steps) + episode_return = 0.0 + length = 0 + policy_steps = 0 + terminated = False + truncated = False + invalid_action = False + policy_error: str | None = None + latencies: list[float] = [] + + try: + observation = _reset_env(env) + + if variant.bootstrap_steps > 0: + assert bootstrap_module is not None + bootstrap_policy = _instantiate_policy( + bootstrap_module, + seed, + ) + _reset_policy(bootstrap_policy, seed) + for _ in range(variant.bootstrap_steps): + if length >= max_steps or terminated or truncated: + break + action = _trusted_action( + bootstrap_policy, + observation, + ) + ( + observation, + reward, + terminated, + truncated, + ) = _step_env(env, action) + episode_return += reward + length += 1 + + for _ in range(variant.pre_policy_noops): + if length >= max_steps or terminated or truncated: + break + ( + observation, + reward, + terminated, + truncated, + ) = _step_env(env, 0) + episode_return += reward + length += 1 + + policy = _instantiate_policy(module, seed) + _reset_policy(policy, seed) + while length < max_steps and not (terminated or truncated): + started_at = time.perf_counter() + try: + action = _validate_action(policy.act(observation)) + except Exception as error: + policy_error = f"{type(error).__name__}: {error}" + invalid_action = isinstance(error, ValueError) + episode_return = failure_score + break + latencies.append((time.perf_counter() - started_at) * 1000.0) + ( + observation, + reward, + terminated, + truncated, + ) = _step_env(env, action) + episode_return += reward + length += 1 + policy_steps += 1 + + if length >= max_steps and not terminated and policy_error is None: + truncated = True + finally: + env.close() + + episodes.append( + MontezumaEpisode( + seed=seed, + variant=variant.name, + bootstrap_steps=variant.bootstrap_steps, + pre_policy_noops=variant.pre_policy_noops, + return_=float(episode_return), + length=length, + policy_steps=policy_steps, + target_reached=episode_return >= target_score, + terminated=terminated, + truncated=truncated, + invalid_action=invalid_action, + policy_error=policy_error, + mean_action_latency_ms=float(np.mean(latencies)) + if latencies + else 0.0, + ) + ) + + return MontezumaEvaluation( + policy_path=str(path), + max_steps=max_steps, + target_score=target_score, + episodes=tuple(episodes), + ) diff --git a/src/genesisbench/pong.py b/src/genesisbench/pong.py new file mode 100644 index 0000000..fb81a01 --- /dev/null +++ b/src/genesisbench/pong.py @@ -0,0 +1,426 @@ +from __future__ import annotations + +import importlib.util +import inspect +import json +import math +import sys +import time +from collections.abc import Callable, Iterable +from dataclasses import asdict, dataclass +from pathlib import Path +from types import ModuleType +from typing import Any + +import numpy as np + + +PONG_TARGET_SCORE = 21.0 + + +@dataclass(frozen=True) +class PongVariant: + """Trusted Atari settings for one Pong evaluation condition.""" + + name: str = "nominal" + noop_max: int = 1 + frame_skip: int = 1 + repeat_action_probability: float = 0.0 + use_fire_reset: bool = True + + +@dataclass(frozen=True) +class PongEpisode: + seed: int + variant: str + score: float + length: int + points_for: int + points_against: int + terminated: bool + truncated: bool + invalid_action: bool + policy_error: str | None + mean_action_latency_ms: float + + @property + def won(self) -> bool: + return self.score > 0.0 + + @property + def target_reached(self) -> bool: + return math.isclose(self.score, PONG_TARGET_SCORE) + + +@dataclass(frozen=True) +class PongEvaluation: + policy_path: str + max_steps: int + episodes: tuple[PongEpisode, ...] + + @property + def mean_score(self) -> float: + return float(np.mean([episode.score for episode in self.episodes])) + + @property + def min_score(self) -> float: + return float(np.min([episode.score for episode in self.episodes])) + + @property + def max_score(self) -> float: + return float(np.max([episode.score for episode in self.episodes])) + + @property + def win_rate(self) -> float: + return float(np.mean([episode.won for episode in self.episodes])) + + @property + def target_score_rate(self) -> float: + return float(np.mean([episode.target_reached for episode in self.episodes])) + + @property + def invalid_episode_rate(self) -> float: + return float( + np.mean( + [ + episode.invalid_action or episode.policy_error is not None + for episode in self.episodes + ] + ) + ) + + @property + def mean_action_latency_ms(self) -> float: + return float( + np.mean([episode.mean_action_latency_ms for episode in self.episodes]) + ) + + def to_dict(self) -> dict[str, Any]: + rendered_episodes = [] + for episode in self.episodes: + rendered = asdict(episode) + rendered["won"] = episode.won + rendered["target_reached"] = episode.target_reached + rendered_episodes.append(rendered) + return { + "policy_path": self.policy_path, + "max_steps": self.max_steps, + "target_score": PONG_TARGET_SCORE, + "score": self.mean_score, + "mean_score": self.mean_score, + "min_score": self.min_score, + "max_score": self.max_score, + "win_rate": self.win_rate, + "target_score_rate": self.target_score_rate, + "invalid_episode_rate": self.invalid_episode_rate, + "mean_action_latency_ms": self.mean_action_latency_ms, + "episodes": rendered_episodes, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + + +EnvironmentFactory = Callable[[int, PongVariant], Any] + + +def _load_policy_module(policy_path: Path) -> ModuleType: + module_name = f"genesisbench_pong_submission_{abs(hash(policy_path.resolve()))}" + spec = importlib.util.spec_from_file_location(module_name, policy_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to import policy from {policy_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + return module + + +def _instantiate_policy(module: ModuleType, seed: int) -> Any: + if hasattr(module, "make_policy"): + factory = module.make_policy + try: + return factory(seed=seed) + except TypeError: + return factory() + if hasattr(module, "Policy"): + policy_class = module.Policy + try: + return policy_class(seed=seed) + except TypeError: + return policy_class() + raise AttributeError("Submission must define Policy or make_policy") + + +def _reset_policy(policy: Any, seed: int) -> None: + reset = getattr(policy, "reset", None) + if reset is None: + return + try: + signature = inspect.signature(reset) + except (TypeError, ValueError): + signature = None + if signature is not None and "seed" in signature.parameters: + reset(seed=seed) + else: + reset() + + +def _make_pong_env(seed: int, variant: PongVariant) -> Any: + try: + import envpool + except ImportError as error: + raise RuntimeError( + "Pong evaluation requires envpool==1.1.1. " + "Use the task environment or install that package locally." + ) from error + + return envpool.make_gym( + "Pong-v5", + num_envs=1, + batch_size=1, + seed=seed, + img_height=210, + img_width=160, + stack_num=1, + gray_scale=True, + frame_skip=variant.frame_skip, + noop_max=variant.noop_max, + use_fire_reset=variant.use_fire_reset, + episodic_life=False, + reward_clip=False, + repeat_action_probability=variant.repeat_action_probability, + full_action_space=False, + ) + + +def _reset_env(environment: Any) -> tuple[Any, dict[str, Any]]: + result = environment.reset() + if not isinstance(result, tuple) or len(result) != 2: + raise RuntimeError("Pong environment reset did not return (obs, info)") + observation, info = result + if not isinstance(info, dict): + raise RuntimeError("Pong environment reset info must be a mapping") + return observation, info + + +def _step_env( + environment: Any, + action: int, +) -> tuple[Any, float, bool, bool, dict[str, Any]]: + result = environment.step(np.asarray([action], dtype=np.int32)) + if not isinstance(result, tuple): + raise RuntimeError("Pong environment step returned an invalid result") + if len(result) == 5: + observation, reward, terminated, truncated, info = result + elif len(result) == 4: + observation, reward, done, info = result + terminated = done + truncated = False + else: + raise RuntimeError(f"Pong environment step returned {len(result)} values") + return ( + observation, + float(np.asarray(reward).reshape(-1)[0]), + bool(np.asarray(terminated).reshape(-1)[0]), + bool(np.asarray(truncated).reshape(-1)[0]), + info, + ) + + +def _extract_ram(info: dict[str, Any]) -> np.ndarray: + if "ram" not in info: + raise RuntimeError("Pong environment did not expose info['ram']") + ram = np.asarray(info["ram"]) + if ram.shape == (1, 128): + ram = ram[0] + if ram.shape != (128,): + raise RuntimeError(f"Expected Pong RAM shape (128,), got {ram.shape}") + return np.asarray(ram, dtype=np.uint8).copy() + + +def _validate_action(action: Any, action_count: int) -> int: + array = np.asarray(action) + if array.shape not in ((), (1,)): + raise ValueError(f"Expected one discrete Pong action, got shape {array.shape}") + value = array.reshape(-1)[0] if array.shape else array.item() + try: + numeric = float(value) + except (TypeError, ValueError) as error: + raise ValueError(f"Pong action must be numeric, got {value!r}") from error + if not math.isfinite(numeric) or not numeric.is_integer(): + raise ValueError(f"Pong action must be a finite integer, got {value!r}") + integer = int(numeric) + if not 0 <= integer < action_count: + raise ValueError( + f"Pong action must be in [0, {action_count - 1}], got {integer}" + ) + return integer + + +def _failure_episode( + *, + seed: int, + variant: PongVariant, + failure_score: float, + length: int, + error: Exception, + invalid_action: bool, + latencies: list[float], +) -> PongEpisode: + return PongEpisode( + seed=seed, + variant=variant.name, + score=failure_score, + length=length, + points_for=0, + points_against=21, + terminated=False, + truncated=False, + invalid_action=invalid_action, + policy_error=f"{type(error).__name__}: {error}", + mean_action_latency_ms=float(np.mean(latencies)) if latencies else 0.0, + ) + + +def evaluate_pong_policy( + policy_path: str | Path, + *, + seeds: Iterable[int], + max_steps: int = 27_000, + variants: Iterable[PongVariant] | None = None, + failure_score: float = -21.0, + environment_factory: EnvironmentFactory | None = None, +) -> PongEvaluation: + """Evaluate a policy using only the current 128-byte Pong RAM state.""" + + path = Path(policy_path).resolve() + if path.is_dir(): + path = path / "policy.py" + if not path.is_file(): + raise FileNotFoundError(path) + if max_steps <= 0: + raise ValueError("max_steps must be positive") + + module = _load_policy_module(path) + configured_seeds = tuple(int(seed) for seed in seeds) + if not configured_seeds: + raise ValueError("At least one evaluation seed is required") + configured_variants = tuple(variants or (PongVariant(),)) + if not configured_variants: + raise ValueError("At least one Pong variant is required") + make_environment = environment_factory or _make_pong_env + episodes: list[PongEpisode] = [] + + for variant in configured_variants: + for seed in configured_seeds: + environment = make_environment(seed, variant) + latencies: list[float] = [] + length = 0 + try: + try: + policy = _instantiate_policy(module, seed) + _reset_policy(policy, seed) + _, info = _reset_env(environment) + except Exception as error: + episodes.append( + _failure_episode( + seed=seed, + variant=variant, + failure_score=failure_score, + length=0, + error=error, + invalid_action=False, + latencies=latencies, + ) + ) + continue + + score = 0.0 + points_for = 0 + points_against = 0 + terminated = False + truncated = False + failed_episode: PongEpisode | None = None + + for length in range(1, max_steps + 1): + started_at = time.perf_counter() + try: + ram = _extract_ram(info) + proposed_action = policy.act(ram) + except Exception as error: + failed_episode = _failure_episode( + seed=seed, + variant=variant, + failure_score=failure_score, + length=length, + error=error, + invalid_action=False, + latencies=latencies, + ) + break + latencies.append((time.perf_counter() - started_at) * 1000.0) + try: + action = _validate_action( + proposed_action, + int(environment.action_space.n), + ) + except Exception as error: + failed_episode = _failure_episode( + seed=seed, + variant=variant, + failure_score=failure_score, + length=length, + error=error, + invalid_action=True, + latencies=latencies, + ) + break + + _, reward, terminated, truncated, info = _step_env( + environment, + action, + ) + score += reward + if reward > 0.0: + points_for += int(round(reward)) + elif reward < 0.0: + points_against += int(round(-reward)) + if terminated or truncated: + break + + if failed_episode is not None: + episodes.append(failed_episode) + continue + if length >= max_steps and not terminated and not truncated: + truncated = True + episodes.append( + PongEpisode( + seed=seed, + variant=variant.name, + score=float(score), + length=length, + points_for=points_for, + points_against=points_against, + terminated=bool(terminated), + truncated=bool(truncated), + invalid_action=False, + policy_error=None, + mean_action_latency_ms=float(np.mean(latencies)) + if latencies + else 0.0, + ) + ) + finally: + close = getattr(environment, "close", None) + if close is not None: + close() + + return PongEvaluation( + policy_path=str(path), + max_steps=max_steps, + episodes=tuple(episodes), + ) diff --git a/src/genesisbench/vizdoom.py b/src/genesisbench/vizdoom.py new file mode 100644 index 0000000..5fec001 --- /dev/null +++ b/src/genesisbench/vizdoom.py @@ -0,0 +1,690 @@ +from __future__ import annotations + +import ast +import importlib.util +import json +import sys +import tempfile +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from types import MappingProxyType, ModuleType +from typing import Any, Iterable, Mapping + +import numpy as np + + +D1_ALLOWED_VARIABLES = ("HEALTH",) +D3_ALLOWED_VARIABLES = ( + "HEALTH", + "AMMO2", + "HITCOUNT", + "DAMAGECOUNT", + "KILLCOUNT", +) +VIZDOOM_ARTICLE_ENVPOOL_VERSION = "1.1.1" + +_FORBIDDEN_IMPORT_ROOTS = {"envpool", "vizdoom"} +_FORBIDDEN_POLICY_NAMES = { + "automap", + "cfg_path", + "env_id", + "game_state", + "label_buffer", + "labels", + "linedefs", + "map_id", + "object_positions", + "sectors", + "things", + "vertices", + "wad_path", +} +_FORBIDDEN_STRING_FRAGMENTS = ( + ".wad", + "d1_basic.cfg", + "d3_battle.cfg", + "/oracle", + "/verifier", +) + + +class PolicySourceViolation(ValueError): + """Raised when a policy tries to access privileged VizDoom state.""" + + +@dataclass(frozen=True) +class VizDoomEpisode: + seed: int + lane: int + return_: float + length: int + terminated: bool + truncated: bool + invalid_action: bool + policy_error: str | None + mean_action_latency_ms: float + final_variables: dict[str, float] + + +@dataclass(frozen=True) +class VizDoomEvaluation: + scenario: str + policy_path: str + envpool_version: str + batch_seed: int + max_steps: int + frame_skip: int + render_width: int + render_height: int + episodes: tuple[VizDoomEpisode, ...] + + @property + def mean_return(self) -> float: + return float(np.mean([episode.return_ for episode in self.episodes])) + + @property + def min_return(self) -> float: + return float(np.min([episode.return_ for episode in self.episodes])) + + @property + def max_return(self) -> float: + return float(np.max([episode.return_ for episode in self.episodes])) + + @property + def invalid_episode_rate(self) -> float: + return float( + np.mean( + [ + episode.invalid_action or episode.policy_error is not None + for episode in self.episodes + ] + ) + ) + + @property + def mean_action_latency_ms(self) -> float: + return float( + np.mean( + [episode.mean_action_latency_ms for episode in self.episodes] + ) + ) + + def to_dict(self) -> dict[str, Any]: + rendered_episodes = [] + for episode in self.episodes: + rendered_episode = asdict(episode) + rendered_episode["return"] = rendered_episode.pop("return_") + rendered_episodes.append(rendered_episode) + return { + "scenario": self.scenario, + "policy_path": self.policy_path, + "envpool_version": self.envpool_version, + "batch_seed": self.batch_seed, + "max_steps": self.max_steps, + "frame_skip": self.frame_skip, + "render_width": self.render_width, + "render_height": self.render_height, + "score": self.mean_return, + "mean_return": self.mean_return, + "min_return": self.min_return, + "max_return": self.max_return, + "invalid_episode_rate": self.invalid_episode_rate, + "mean_action_latency_ms": self.mean_action_latency_ms, + "episodes": rendered_episodes, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + + +def _policy_files(policy_path: Path) -> tuple[Path, ...]: + root = policy_path.parent + for candidate in root.rglob("*"): + if candidate.is_symlink(): + raise PolicySourceViolation( + f"Policy bundle may not contain symlinks: {candidate}" + ) + return tuple(sorted(root.rglob("*.py"))) + + +def _import_root(alias: ast.alias) -> str: + return alias.name.split(".", 1)[0].lower() + + +def audit_vizdoom_policy(policy_path: str | Path) -> None: + """Reject direct access to maps, labels, objects, or EnvPool internals.""" + + path = Path(policy_path).resolve() + if path.is_dir(): + path = path / "policy.py" + if not path.is_file(): + raise FileNotFoundError(path) + + for source_path in _policy_files(path): + try: + tree = ast.parse(source_path.read_text(), filename=str(source_path)) + except (OSError, SyntaxError) as error: + raise PolicySourceViolation( + f"Unable to audit {source_path.name}: {error}" + ) from error + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + roots = {_import_root(alias) for alias in node.names} + forbidden = roots & _FORBIDDEN_IMPORT_ROOTS + if forbidden: + raise PolicySourceViolation( + "Policy may not import simulator internals: " + + ", ".join(sorted(forbidden)) + ) + elif isinstance(node, ast.ImportFrom) and node.module: + root = node.module.split(".", 1)[0].lower() + if root in _FORBIDDEN_IMPORT_ROOTS: + raise PolicySourceViolation( + f"Policy may not import simulator internals: {root}" + ) + elif isinstance(node, ast.Name): + if node.id.lower() in _FORBIDDEN_POLICY_NAMES: + raise PolicySourceViolation( + f"Forbidden privileged-state identifier: {node.id}" + ) + elif isinstance(node, ast.Attribute): + if node.attr.lower() in _FORBIDDEN_POLICY_NAMES: + raise PolicySourceViolation( + f"Forbidden privileged-state attribute: {node.attr}" + ) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + value = node.value.lower() + if any( + fragment in value + for fragment in _FORBIDDEN_STRING_FRAGMENTS + ): + raise PolicySourceViolation( + "Policy source references a forbidden map or hidden path" + ) + + +def _load_policy_module(policy_path: Path) -> ModuleType: + module_name = f"genesisbench_vizdoom_submission_{abs(hash(policy_path))}" + spec = importlib.util.spec_from_file_location(module_name, policy_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to import policy from {policy_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(module_name, None) + raise + return module + + +def _instantiate_policy(module: ModuleType) -> Any: + if hasattr(module, "make_policy"): + return module.make_policy() + if hasattr(module, "Policy"): + return module.Policy() + raise AttributeError("Submission must define Policy or make_policy") + + +def _reset_policy(policy: Any) -> None: + reset = getattr(policy, "reset", None) + if reset is not None: + reset() + + +def _readonly_frame(frame: np.ndarray) -> np.ndarray: + public_frame = np.array(frame, dtype=np.uint8, copy=True, order="C") + public_frame.flags.writeable = False + return public_frame + + +def _to_hwc(frame: np.ndarray) -> np.ndarray: + array = np.asarray(frame) + if array.ndim != 3: + raise ValueError(f"Expected a three-dimensional frame, got {array.shape}") + if array.shape[0] in (1, 3, 4) and array.shape[-1] not in (1, 3, 4): + array = np.transpose(array, (1, 2, 0)) + if array.shape[-1] not in (1, 3, 4): + raise ValueError(f"Expected channel-last image data, got {array.shape}") + return array + + +def _row_variables( + info: Mapping[str, Any], + row: int, + allowed_variables: Iterable[str], +) -> dict[str, float]: + variables: dict[str, float] = {} + for name in allowed_variables: + if name not in info: + raise KeyError(f"EnvPool did not provide required variable {name}") + values = np.asarray(info[name]) + variables[name] = float(values[row]) + return variables + + +def _active_info( + info: Mapping[str, Any], + keep: np.ndarray, + row_count: int, +) -> dict[str, np.ndarray]: + return { + key: np.asarray(value)[keep] + for key, value in info.items() + if np.asarray(value).ndim > 0 + and len(np.asarray(value)) == row_count + } + + +def _validate_d1_action(action: Any) -> int: + array = np.asarray(action) + if array.shape != (): + raise ValueError(f"Expected a scalar D1 action, got {array.shape}") + if np.issubdtype(array.dtype, np.bool_): + raise ValueError("Boolean actions are not valid D1 actions") + value = float(array) + if not np.isfinite(value) or not value.is_integer(): + raise ValueError(f"Expected an integer D1 action, got {action!r}") + integer = int(value) + if integer < 0 or integer > 5: + raise ValueError(f"D1 action must be in [0, 5], got {integer}") + return integer + + +def _validate_d3_action(action: Any) -> np.ndarray: + array = np.asarray(action, dtype=np.float64) + if array.shape != (8,): + raise ValueError(f"Expected D3 action shape (8,), got {array.shape}") + if not np.all(np.isfinite(array)): + raise ValueError("D3 action contains NaN or infinite values") + if np.any(array[:7] < 0.0) or np.any(array[:7] > 1.0): + raise ValueError("D3 button channels must be in [0, 1]") + if abs(float(array[7])) > 12.0: + raise ValueError("D3 turn delta must be in [-12, 12]") + return array + + +def _make_d1_env( + *, + episodes: int, + seed: int, + max_steps: int, + frame_skip: int, + render_width: int, + render_height: int, +) -> tuple[Any, None]: + import envpool + + _require_article_envpool_version(envpool) + env = envpool.make_gymnasium( + "D1Basic-v1", + num_envs=episodes, + seed=seed, + use_combined_action=True, + stack_num=1, + frame_skip=frame_skip, + max_episode_steps=max_steps, + render_mode="rgb_array", + render_width=render_width, + render_height=render_height, + ) + return env, None + + +def _d3_config_text(*, width: int, height: int) -> str: + from envpool.vizdoom.registration import maps_path + + source = Path(maps_path) / "D3_battle.cfg" + config = source.read_text() + replacements = { + "screen_resolution = RES_160X120": ( + f"screen_resolution = RES_{width}X{height}" + ), + "screen_format = GRAY8": "screen_format = CRCGCB", + "render_weapon = true": "render_weapon = false", + "render_crosshair = true": "render_crosshair = false", + } + for original, replacement in replacements.items(): + if original not in config: + raise RuntimeError( + f"EnvPool D3 config is missing expected line: {original}" + ) + config = config.replace(original, replacement, 1) + + buttons = """available_buttons = + { + ATTACK + SPEED + MOVE_FORWARD + MOVE_BACKWARD + MOVE_RIGHT + MOVE_LEFT + TURN180 + TURN_LEFT_RIGHT_DELTA + } + +""" + start = config.index("available_buttons") + end = config.index("# Game variables") + return config[:start] + buttons + config[end:] + + +def _require_article_envpool_version(envpool: ModuleType) -> None: + version = getattr(envpool, "__version__", None) + if version != VIZDOOM_ARTICLE_ENVPOOL_VERSION: + raise RuntimeError( + "VizDoom article tasks require EnvPool " + f"{VIZDOOM_ARTICLE_ENVPOOL_VERSION}, found {version!r}" + ) + + +def _make_d3_env( + *, + episodes: int, + seed: int, + max_steps: int, + frame_skip: int, + render_width: int, + render_height: int, +) -> tuple[Any, tempfile.TemporaryDirectory[str]]: + import envpool + + _require_article_envpool_version(envpool) + temporary_directory = tempfile.TemporaryDirectory( + prefix="genesisbench-simulation-heuristics-vizdoom-d3-v1-" + ) + config_path = Path(temporary_directory.name) / "battle-screen-cv.cfg" + config_path.write_text( + _d3_config_text(width=render_width, height=render_height) + ) + try: + env = envpool.make_gymnasium( + "D3Battle-v1", + num_envs=episodes, + seed=seed, + cfg_path=str(config_path), + use_combined_action=False, + stack_num=1, + frame_skip=frame_skip, + max_episode_steps=max_steps, + img_width=render_width, + img_height=render_height, + reward_config={ + "DAMAGECOUNT": [1.0, 0.0], + "KILLCOUNT": [10.0, 0.0], + }, + selected_weapon_reward_config={}, + ) + except Exception: + temporary_directory.cleanup() + raise + return env, temporary_directory + + +def _failed_evaluation( + *, + scenario: str, + policy_path: Path, + seed: int, + episodes: int, + max_steps: int, + frame_skip: int, + render_width: int, + render_height: int, + failure_return: float, + error: Exception, +) -> VizDoomEvaluation: + message = f"{type(error).__name__}: {error}" + failed = tuple( + VizDoomEpisode( + seed=seed + lane, + lane=lane, + return_=failure_return, + length=0, + terminated=False, + truncated=False, + invalid_action=isinstance(error, ValueError), + policy_error=message, + mean_action_latency_ms=0.0, + final_variables={}, + ) + for lane in range(episodes) + ) + return VizDoomEvaluation( + scenario=scenario, + policy_path=str(policy_path), + envpool_version=VIZDOOM_ARTICLE_ENVPOOL_VERSION, + batch_seed=seed, + max_steps=max_steps, + frame_skip=frame_skip, + render_width=render_width, + render_height=render_height, + episodes=failed, + ) + + +def evaluate_vizdoom_policy( + policy_path: str | Path, + *, + scenario: str, + seed: int, + episodes: int = 10, + max_steps: int | None = None, + frame_skip: int | None = None, + render_width: int | None = None, + render_height: int | None = None, + failure_return: float = 0.0, +) -> VizDoomEvaluation: + """Evaluate a screen-only D1 or D3 policy in one EnvPool seed batch.""" + + if scenario not in {"d1", "d3"}: + raise ValueError(f"Unsupported VizDoom scenario: {scenario}") + if episodes <= 0: + raise ValueError("episodes must be positive") + + defaults = { + "d1": (2100, 1, 240, 180), + "d3": (1050, 2, 640, 480), + } + default_steps, default_skip, default_width, default_height = defaults[scenario] + configured_steps = default_steps if max_steps is None else max_steps + configured_skip = default_skip if frame_skip is None else frame_skip + configured_width = default_width if render_width is None else render_width + configured_height = ( + default_height if render_height is None else render_height + ) + + path = Path(policy_path).resolve() + if path.is_dir(): + path = path / "policy.py" + if not path.is_file(): + raise FileNotFoundError(path) + + try: + audit_vizdoom_policy(path) + module = _load_policy_module(path) + except Exception as error: + return _failed_evaluation( + scenario=scenario, + policy_path=path, + seed=seed, + episodes=episodes, + max_steps=configured_steps, + frame_skip=configured_skip, + render_width=configured_width, + render_height=configured_height, + failure_return=failure_return, + error=error, + ) + + make_env = _make_d1_env if scenario == "d1" else _make_d3_env + env, temporary_directory = make_env( + episodes=episodes, + seed=seed, + max_steps=configured_steps, + frame_skip=configured_skip, + render_width=configured_width, + render_height=configured_height, + ) + allowed_variables = ( + D1_ALLOWED_VARIABLES + if scenario == "d1" + else D3_ALLOWED_VARIABLES + ) + validate_action = ( + _validate_d1_action if scenario == "d1" else _validate_d3_action + ) + + returns = np.zeros(episodes, dtype=np.float64) + lengths = np.zeros(episodes, dtype=np.int64) + terminated_by_lane = np.zeros(episodes, dtype=np.bool_) + truncated_by_lane = np.zeros(episodes, dtype=np.bool_) + invalid_by_lane = np.zeros(episodes, dtype=np.bool_) + errors: list[str | None] = [None] * episodes + latencies: list[list[float]] = [[] for _ in range(episodes)] + final_variables: list[dict[str, float]] = [ + {} for _ in range(episodes) + ] + + try: + observations, info = env.reset() + active_ids = np.asarray(info["env_id"], dtype=np.int64) + active_info: dict[str, Any] = dict(info) + active_observations = np.asarray(observations) + policies: dict[int, Any] = {} + for lane in range(episodes): + try: + policy = _instantiate_policy(module) + _reset_policy(policy) + policies[lane] = policy + except Exception as error: + returns[lane] = failure_return + errors[lane] = f"{type(error).__name__}: {error}" + + rendered_frames = None + if scenario == "d1": + rendered_frames = env.render( + env_ids=np.arange(episodes, dtype=np.int64) + ) + + for _ in range(configured_steps): + step_ids: list[int] = [] + actions: list[Any] = [] + + for row, lane_value in enumerate(active_ids): + lane = int(lane_value) + if lane not in policies or errors[lane] is not None: + continue + variables = _row_variables( + active_info, + row, + allowed_variables, + ) + final_variables[lane] = variables + if scenario == "d1": + assert rendered_frames is not None + frame = rendered_frames[lane] + else: + frame = _to_hwc(active_observations[row]) + + started_at = time.perf_counter() + try: + public_variables = MappingProxyType(dict(variables)) + action = policies[lane].act( + _readonly_frame(frame), + public_variables, + ) + action = validate_action(action) + except Exception as error: + errors[lane] = f"{type(error).__name__}: {error}" + invalid_by_lane[lane] = isinstance(error, ValueError) + returns[lane] = failure_return + continue + finally: + latencies[lane].append( + (time.perf_counter() - started_at) * 1000.0 + ) + step_ids.append(lane) + actions.append(action) + + if not step_ids: + break + + action_array = np.asarray( + actions, + dtype=np.int64 if scenario == "d1" else np.float64, + ) + next_observations, rewards, terminated, truncated, info = env.step( + action_array, + np.asarray(step_ids, dtype=np.int64), + ) + done = np.logical_or(terminated, truncated) + current_ids = np.asarray(info["env_id"], dtype=np.int64) + + for row, lane_value in enumerate(current_ids): + lane = int(lane_value) + returns[lane] += float(np.asarray(rewards)[row]) + lengths[lane] += 1 + final_variables[lane] = _row_variables( + info, + row, + allowed_variables, + ) + if bool(done[row]): + terminated_by_lane[lane] = bool(terminated[row]) + truncated_by_lane[lane] = bool(truncated[row]) + + keep = ~done + active_ids = current_ids[keep] + active_info = _active_info(info, keep, len(done)) + if scenario == "d3": + active_observations = np.asarray(next_observations)[keep] + if len(active_ids) == 0: + break + if scenario == "d1": + rendered_frames = env.render( + env_ids=np.arange(episodes, dtype=np.int64) + ) + + unfinished = { + int(lane) + for lane in active_ids + if errors[int(lane)] is None + } + for lane in unfinished: + truncated_by_lane[lane] = True + finally: + env.close() + if temporary_directory is not None: + temporary_directory.cleanup() + + episode_results = tuple( + VizDoomEpisode( + seed=seed + lane, + lane=lane, + return_=float(returns[lane]), + length=int(lengths[lane]), + terminated=bool(terminated_by_lane[lane]), + truncated=bool(truncated_by_lane[lane]), + invalid_action=bool(invalid_by_lane[lane]), + policy_error=errors[lane], + mean_action_latency_ms=( + float(np.mean(latencies[lane])) if latencies[lane] else 0.0 + ), + final_variables=final_variables[lane], + ) + for lane in range(episodes) + ) + return VizDoomEvaluation( + scenario=scenario, + policy_path=str(path), + envpool_version=VIZDOOM_ARTICLE_ENVPOOL_VERSION, + batch_seed=seed, + max_steps=configured_steps, + frame_skip=configured_skip, + render_width=configured_width, + render_height=configured_height, + episodes=episode_results, + ) diff --git a/tasks/simulation_heuristics_ant_v1/README.md b/tasks/simulation_heuristics_ant_v1/README.md index 5785e70..29c1c09 100644 --- a/tasks/simulation_heuristics_ant_v1/README.md +++ b/tasks/simulation_heuristics_ant_v1/README.md @@ -1,8 +1,8 @@ # Simulation Heuristics Ant v1 — Reference Task -`simulation_heuristics_ant_v1` is the canonical, end-to-end example for GenesisBench contributors. -It demonstrates how to turn an open-ended robotics research loop into a -repeatable coding-agent benchmark: +`simulation_heuristics_ant_v1` is the canonical, end-to-end example for +GenesisBench contributors. It demonstrates how to turn an open-ended robotics +research loop into a repeatable coding-agent benchmark: ```text starter policy @@ -44,11 +44,46 @@ raw score = The normalized score maps: - the checked-in starter controller to `0`; -- the checked-in stronger reference controller to `100`. +- the checked-in article residual-MPC reference controller to `100`. The verifier evaluates trusted copies of both anchors on the same platform as the submitted policy, avoiding small MuJoCo macOS/Linux drift. Scores above -`100` are valid. +`100` are valid. It caches evaluations by `policy.py` fingerprint, so an oracle +submission byte-identical to the reference is not evaluated twice. + +## Article reference + +The trusted reference adapts the final Ant controller from *Learning Beyond +Gradients* at revision +`3555c2956c257d49a5015b782cbe485b14fd659e`. + +It combines a speed-adaptive CPG/PD gait with warm-started residual MPC: + +```text +96 candidate plans per external step +× 10-step horizon +× copied MuJoCo model +``` + +The article's EnvPool rerun reported seeds `0..4` with mean `6005.521`, +minimum `5776.805`, and maximum `6146.208`. + +Gymnasium and the supplied EnvPool XML parse to the same MuJoCo model. Their +reset random-number streams differ, so platform-specific results are recorded +separately in `evidence/source_provenance.json`. + +On the development host, the GenesisBench oracle reproduced the imported +source controller exactly under Gymnasium: + +```text +seeds 0..4 +mean 5895.932216 +min 5791.444245 +max 6131.400491 +``` + +All five episodes reached 1,000 steps. The adapted and source controllers also +selected byte-identical floating-point actions over a 50-step parity probe. ## Run locally @@ -62,6 +97,38 @@ uv run python tasks/simulation_heuristics_ant_v1/verifier/evaluate_hidden.py \ tasks/simulation_heuristics_ant_v1/starter_policy/policy.py ``` +The hidden-evaluator command runs the full suite and calibrates the MPC +reference locally, so it can take tens of minutes. Use the focused pytest suite +for ordinary development. + +Run a one-step MPC smoke: + +```bash +uv run python tasks/simulation_heuristics_ant_v1/evaluate.py \ + --policy tasks/simulation_heuristics_ant_v1/oracle/policy.py \ + --episodes 1 \ + --max-steps 1 \ + --seed 0 +``` + +Run the full five-seed article-reference reproduction: + +```bash +uv run python tasks/simulation_heuristics_ant_v1/evaluate.py \ + --policy tasks/simulation_heuristics_ant_v1/oracle/policy.py \ + --episodes 5 \ + --max-steps 1000 \ + --seed 0 +``` + +The exact planner is intentionally CPU-heavy. Use the gated regression test +when a full reproduction is required: + +```bash +GENESISBENCH_RUN_SLOW_ANT_MPC=1 \ + uv run pytest -q tests/test_simulation_heuristics_ant_v1.py +``` + Validate the native BenchFlow package: ```bash @@ -80,6 +147,13 @@ uv run bench eval run \ --context-root . ``` +The checked-in `verifier/config.toml` is the full publication-grade suite and +can take tens of minutes with the residual-MPC reference. CI temporarily swaps +in `verifier/config_smoke.toml` for the command above. The smoke suite still +exercises the Docker environment, copied-model planning, a dynamics variant, +anchor normalization, and reward emission, while keeping the GitHub Actions +gate bounded. + `--context-root .` lets BenchFlow stage the repo-root sources referenced by the task's `environment/Dockerfile` into its isolated build context. @@ -105,8 +179,10 @@ The custom OpenHands experiment workspace must not contain `verifier/`, | `task_context/policy_api.md` | Stable policy interface | | `verifier/evaluate_hidden.py` | Clean final score calculation | | `verifier/config.toml` | Reproducibility evaluation suite | +| `verifier/config_smoke.toml` | One-step CI integration suite | | `verifier/anchors.json` | Frozen normalization anchors | | `oracle/solve.sh` | BenchFlow oracle entrypoint | +| `evidence/source_provenance.json` | Source hashes, license, and reproduction metrics | ## Public versus private final suites diff --git a/tasks/simulation_heuristics_ant_v1/evidence/source_provenance.json b/tasks/simulation_heuristics_ant_v1/evidence/source_provenance.json new file mode 100644 index 0000000..751e72f --- /dev/null +++ b/tasks/simulation_heuristics_ant_v1/evidence/source_provenance.json @@ -0,0 +1,77 @@ +{ + "schema_version": "1.0", + "source": { + "repository": "https://github.com/Trinkle23897/learning-beyond-gradients", + "revision": "3555c2956c257d49a5015b782cbe485b14fd659e", + "files": [ + { + "path": "mujoco/ant/heuristic_ant.py", + "sha256": "8a6e200f91d9392e7f752469c3266673dabac1ea3ee7bb50636a2e2ce934514c", + "license": "Apache-2.0", + "copyright": "Copyright 2021 Garena Online Private Limited" + }, + { + "path": "mujoco/ant/ant_envpool.xml", + "sha256": "e2de8745656b2e7046718d4bd3264d18deb802739033d287d7bdcf20f925a0b5", + "license": "Apache-2.0", + "copyright": "Copyright 2026 Garena Online Private Limited" + } + ] + }, + "article_reproduction": { + "runtime": "EnvPool 1.1.1 Ant-v5", + "seeds": [0, 1, 2, 3, 4], + "max_steps": 1000, + "reported_mean_return": 6005.521, + "reported_min_return": 5776.805, + "reported_max_return": 6146.208 + }, + "genesisbench_adaptation": { + "runtime": "Gymnasium Ant-v5", + "observation_size": 27, + "action_size": 8, + "live_environment_exposed": false, + "copied_model_hook": "configure_simulator(model_xml_path, frame_skip)", + "gymnasium_xml_sha256": "cd5f83ef0ea35b0969e65d360c5bacd5b74ccaef6b27e4433b5168c605e3e2be", + "source_and_gymnasium_xml_parse_equal": true, + "source_action_parity": { + "seed": 0, + "checked_steps": 50, + "max_absolute_difference": 0.0 + } + }, + "local_source_policy_on_gymnasium": { + "date": "2026-07-13", + "seeds": [0, 1, 2, 3, 4], + "max_steps": 1000, + "returns": [ + 6131.400491081534, + 5817.390376060449, + 5791.444245416141, + 5866.045000176889, + 5873.380967941683 + ], + "mean_return": 5895.932216135339, + "min_return": 5791.444245416141, + "max_return": 6131.400491081534 + }, + "local_genesisbench_oracle_on_gymnasium": { + "date": "2026-07-13", + "seeds": [0, 1, 2, 3, 4], + "max_steps": 1000, + "returns": [ + 6131.400491081534, + 5817.390376060449, + 5791.444245416141, + 5866.045000176889, + 5873.380967941683 + ], + "mean_return": 5895.932216135338, + "min_return": 5791.444245416141, + "max_return": 6131.400491081534, + "invalid_episode_rate": 0.0, + "mean_action_latency_ms": 167.1076581293997, + "parallel_wall_seconds": 335.64, + "matches_local_source_policy": true + } +} diff --git a/tasks/simulation_heuristics_ant_v1/oracle/policy.py b/tasks/simulation_heuristics_ant_v1/oracle/policy.py index dbeded3..7ddaa2e 100644 --- a/tasks/simulation_heuristics_ant_v1/oracle/policy.py +++ b/tasks/simulation_heuristics_ant_v1/oracle/policy.py @@ -5,13 +5,26 @@ # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Warm-started speed-adaptive residual MPC reference for Ant-v5. -"""Frozen improved rhythmic reference used to calibrate Simulation Heuristics Ant v1.""" +Adapted for the GenesisBench policy API from ``mujoco/ant/heuristic_ant.py`` +at learning-beyond-gradients revision +``3555c2956c257d49a5015b782cbe485b14fd659e``. +""" from __future__ import annotations import math +from dataclasses import dataclass +import mujoco import numpy as np @@ -22,17 +35,192 @@ HEADING_AXIS = np.asarray([1.0, 1.0, -1.0, -1.0], dtype=np.float64) PITCH_AXIS = np.asarray([1.0, 1.0, -1.0, -1.0], dtype=np.float64) ROLL_AXIS = np.asarray([-1.0, 1.0, 1.0, -1.0], dtype=np.float64) +FOOT_BODY_IDS = np.asarray([13, 4, 7, 10], dtype=np.int64) +FOOT_OBS_ROWS = FOOT_BODY_IDS - 1 + + +@dataclass(frozen=True) +class ControllerConfig: + dphi: float = 0.660934259732249 + dphi_speed_gain: float = -0.02 + dphi_speed_target: float = 5.8 + dphi_min: float = 0.62 + dphi_max: float = 0.72 + hip_bias: float = 0.12217781430672398 + hip_amp: float = 0.5705418365199333 + ankle_bias: float = 0.36651046903486795 + ankle_amp: float = 0.26587749767314783 + kp: float = 0.8108143632989734 + kd: float = 0.0 + pitch_gain: float = -0.19444590796726124 + pitch_rate_gain: float = 0.04099276700871415 + roll_gain: float = -0.25536960225655303 + roll_rate_gain: float = 0.023293075237761272 + contact_hip_gain: float = 0.0 + contact_ankle_gain: float = 0.0 + contact_push_hip_amp: float = 0.0 + contact_push_ankle_amp: float = 0.0 + contact_push_phase: float = 2.356194490192345 + contact_push_width: float = 0.55 + stance_duty: float = 0.6355364206196007 + stance_duty_speed_gain: float = -0.01 + stance_duty_speed_target: float = 5.8 + stance_duty_min: float = 0.6 + stance_duty_max: float = 0.67 + hip_stance_scale: float = 1.0479076970107701 + hip_swing_scale: float = 1.0031777685985328 + ankle_stance_scale: float = 0.976603459922793 + ankle_swing_scale: float = 0.9374473230114526 + yaw_gain: float = -0.12067720879887742 + yaw_rate_gain: float = 0.04418873596679619 + hip_h2_amp: float = 0.10975404801587477 + hip_h2_phase: float = 2.0862256065597453 + ankle_h2_amp: float = -0.003434817287963554 + ankle_h2_phase: float = 1.2927488104774438 + hip_h3_amp: float = 0.04827596673280693 + hip_h3_phase: float = -0.49944083263433436 + ankle_h3_amp: float = -0.06968988354403895 + ankle_h3_phase: float = 1.5873441034476188 + mpc_horizon: int = 10 + mpc_candidates: int = 96 + mpc_sigma: float = 0.07614211639071694 + mpc_clip: float = 0.12016284361036686 + mpc_pose_cost: float = 23.348190567885954 + mpc_pitch_target: float = 0.0 + mpc_yaw_cost: float = 2.7292168081366723 + mpc_z_cost: float = 2.1215830559511737 + mpc_z_target: float = 0.4519975076600261 + mpc_forward_weight: float = 1.0 + mpc_ctrl_cost: float = 0.5 + mpc_terminal_vel_cost: float = 0.01 + mpc_plan_decay: float = 0.504186948858276 + mpc_seed: int = 12 + + +CONFIG = ControllerConfig() + + +def _warp_leg_phase(leg_phase: np.ndarray, stance_duty: float) -> np.ndarray: + clipped_duty = float(np.clip(stance_duty, 0.05, 0.95)) + if abs(clipped_duty - 0.5) < 1e-12: + return np.mod(leg_phase, 2.0 * math.pi) + + phase_unit = np.mod(leg_phase, 2.0 * math.pi) / (2.0 * math.pi) + stance_unit = 0.5 * phase_unit / clipped_duty + swing_unit = 0.5 + 0.5 * (phase_unit - clipped_duty) / (1.0 - clipped_duty) + warped_unit = np.where( + phase_unit < clipped_duty, + stance_unit, + swing_unit, + ) + return 2.0 * math.pi * warped_unit + + +def _adaptive_dphi(config: ControllerConfig, x_velocity: float) -> float: + dphi = config.dphi + config.dphi_speed_gain * ( + x_velocity - config.dphi_speed_target + ) + return float(np.clip(dphi, config.dphi_min, config.dphi_max)) + + +def _adaptive_stance_duty( + config: ControllerConfig, + x_velocity: float, +) -> float: + stance_duty = config.stance_duty + config.stance_duty_speed_gain * ( + x_velocity - config.stance_duty_speed_target + ) + return float( + np.clip( + stance_duty, + config.stance_duty_min, + config.stance_duty_max, + ) + ) + + +def _rhythmic_action( + config: ControllerConfig, + phase: float, + stance_duty: float, + q: np.ndarray, + dq: np.ndarray, + roll: float, + pitch: float, + yaw: float, + roll_rate: float, + pitch_rate: float, + yaw_rate: float, + foot_contacts: np.ndarray, +) -> np.ndarray: + leg_phase = _warp_leg_phase(phase + LEG_PHASE, stance_duty) + stance_mask = leg_phase < math.pi + hip_wave = config.hip_bias + np.where( + stance_mask, + config.hip_stance_scale, + config.hip_swing_scale, + ) * ( + config.hip_amp * np.sin(leg_phase) + + config.hip_h2_amp * np.sin(2.0 * leg_phase + config.hip_h2_phase) + + config.hip_h3_amp * np.sin(3.0 * leg_phase + config.hip_h3_phase) + ) + ankle_wave = config.ankle_bias + np.where( + stance_mask, + config.ankle_stance_scale, + config.ankle_swing_scale, + ) * ( + config.ankle_amp * np.cos(leg_phase) + + config.ankle_h2_amp * np.cos(2.0 * leg_phase + config.ankle_h2_phase) + + config.ankle_h3_amp * np.cos(3.0 * leg_phase + config.ankle_h3_phase) + ) + push_width = max(config.contact_push_width, 1e-3) + contact_push = ( + np.exp(-0.5 * np.square((leg_phase - config.contact_push_phase) / push_width)) + * stance_mask + * foot_contacts + ) + hip_wave = hip_wave + config.contact_push_hip_amp * contact_push + ankle_wave = ankle_wave + config.contact_push_ankle_amp * contact_push + balance_wave = PITCH_AXIS * ( + config.pitch_gain * pitch + config.pitch_rate_gain * pitch_rate + ) - ROLL_AXIS * (config.roll_gain * roll + config.roll_rate_gain * roll_rate) + + action = np.empty(8, dtype=np.float64) + action[0::2] = ( + config.kp + * ( + HIP_SIGN * hip_wave + + HEADING_AXIS + * ( + config.yaw_gain * yaw + + config.yaw_rate_gain * yaw_rate + + config.contact_hip_gain * foot_contacts + ) + - q[0::2] + ) + - config.kd * dq[0::2] + ) + action[1::2] = ( + config.kp + * ( + ANKLE_SIGN + * (ankle_wave + balance_wave + config.contact_ankle_gain * foot_contacts) + - q[1::2] + ) + - config.kd * dq[1::2] + ) + return np.clip(action, -1.0, 1.0) -def _rpy(quaternion: np.ndarray) -> tuple[float, float, float]: - w, x, y, z = quaternion / (np.linalg.norm(quaternion) + 1e-12) +def _euler(quaternion: np.ndarray) -> tuple[float, float, float]: + quaternion = np.asarray(quaternion, dtype=np.float64) + quaternion /= np.linalg.norm(quaternion) + 1e-12 + w, x, y, z = quaternion roll = math.atan2( 2.0 * (w * x + y * z), 1.0 - 2.0 * (x * x + y * y), ) - pitch = math.asin( - np.clip(2.0 * (w * y - z * x), -1.0, 1.0) - ) + pitch = math.asin(np.clip(2.0 * (w * y - z * x), -1.0, 1.0)) yaw = math.atan2( 2.0 * (w * z + x * y), 1.0 - 2.0 * (y * y + z * z), @@ -40,45 +228,228 @@ def _rpy(quaternion: np.ndarray) -> tuple[float, float, float]: return roll, pitch, yaw +def _observation_array(observation: np.ndarray) -> np.ndarray: + array = np.asarray(observation) + if array.ndim == 2: + array = array[0] + if array.ndim != 1 or array.shape[0] < 27: + raise ValueError(f"Unsupported Ant observation shape: {array.shape}") + return array + + +def _joint_state(observation: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + array = _observation_array(observation) + return array[5:13], array[19:27] + + +def _forward_velocity(observation: np.ndarray) -> float: + return float(_observation_array(observation)[13]) + + +def _torso_state( + observation: np.ndarray, +) -> tuple[float, float, float, float, float, float]: + array = _observation_array(observation) + quaternion = np.asarray(array[1:5], dtype=np.float64) + quaternion /= np.linalg.norm(quaternion) + 1e-12 + roll, pitch, yaw = _euler(quaternion) + return ( + roll, + pitch, + yaw, + float(array[16]), + float(array[17]), + float(array[18]), + ) + + +def _observation_foot_contacts(observation: np.ndarray) -> np.ndarray: + array = _observation_array(observation) + if array.shape[0] < 27 + 13 * 6: + return np.zeros(4, dtype=np.float64) + contact_forces = array[27 : 27 + 13 * 6].reshape(13, 6) + return np.clip(contact_forces[FOOT_OBS_ROWS, 5], 0.0, 1.0) + + class Policy: def __init__(self, seed: int = 0) -> None: + self._model: mujoco.MjModel | None = None + self._rollout_data: mujoco.MjData | None = None + self._frame_skip = 5 + self._residual_plan = np.zeros( + (CONFIG.mpc_horizon, 8), + dtype=np.float64, + ) self.reset(seed) + def configure_simulator( + self, + *, + model_xml_path: str, + frame_skip: int = 5, + ) -> None: + model = mujoco.MjModel.from_xml_path(model_xml_path) + if (model.nq, model.nv, model.nu) != (15, 14, 8): + raise ValueError( + "Expected Ant model dimensions (15, 14, 8), " + f"got {(model.nq, model.nv, model.nu)}" + ) + self._model = model + self._rollout_data = mujoco.MjData(model) + self._frame_skip = int(frame_skip) + def reset(self, seed: int = 0) -> None: + self.seed = int(seed) self.phase = 0.0 + self._rng = np.random.default_rng(CONFIG.mpc_seed) + self._residual_plan.fill(0.0) def act(self, observation: np.ndarray) -> np.ndarray: - observation = np.asarray(observation, dtype=np.float64) - q = observation[5:13][Q_INDEX] - _, _, yaw = _rpy(observation[1:5]) - yaw_rate = float(observation[18]) - leg_phase = self.phase + LEG_PHASE - - hip_wave = 0.12217781430672398 + ( - 0.3618286792202296 * np.sin(leg_phase) - + 0.10975404801587477 - * np.sin(2.0 * leg_phase + 2.0862256065597453) - + 0.04827596673280693 - * np.sin(3.0 * leg_phase - 0.4994408326343364) - ) - ankle_wave = 0.36651046903486795 + ( - 0.3226052803819716 * np.cos(leg_phase) - - 0.003434817287963554 - * np.cos(2.0 * leg_phase + 1.2927488104774438) - - 0.06968988354403895 - * np.cos(3.0 * leg_phase + 1.5873441034476188) + if self._model is None or self._rollout_data is None: + raise RuntimeError("configure_simulator must be called before act") + observation = _observation_array(observation) + self._set_model_state(observation) + q = self._rollout_data.qpos[7:15][Q_INDEX] + dq = self._rollout_data.qvel[6:14][Q_INDEX] + x_velocity = _forward_velocity(observation) + ( + roll, + pitch, + yaw, + roll_rate, + pitch_rate, + yaw_rate, + ) = _torso_state(observation) + foot_contacts = _observation_foot_contacts(observation) + stance_duty = _adaptive_stance_duty(CONFIG, x_velocity) + base_action = _rhythmic_action( + CONFIG, + self.phase, + stance_duty, + q, + dq, + roll, + pitch, + yaw, + roll_rate, + pitch_rate, + yaw_rate, + foot_contacts, ) - action = np.empty(8, dtype=np.float64) - action[0::2] = 0.5493824068786054 * ( - HIP_SIGN * hip_wave - + HEADING_AXIS - * (-0.12067720879887742 * yaw + 0.04418873596679619 * yaw_rate) - - q[0::2] + best_residuals = self._residual_plan.copy() + best_objective = self._rollout_objective( + observation, + best_residuals, ) - action[1::2] = 0.5493824068786054 * ( - ANKLE_SIGN * ankle_wave - q[1::2] + for _ in range(CONFIG.mpc_candidates - 1): + residuals = np.clip( + best_residuals + + self._rng.normal( + 0.0, + CONFIG.mpc_sigma, + size=(CONFIG.mpc_horizon, 8), + ), + -CONFIG.mpc_clip, + CONFIG.mpc_clip, + ) + residuals[1:] = 0.6 * residuals[1:] + 0.4 * residuals[:-1] + objective = self._rollout_objective(observation, residuals) + if objective > best_objective: + best_objective = objective + best_residuals = residuals + + self.phase += _adaptive_dphi(CONFIG, x_velocity) + self._residual_plan[:-1] = CONFIG.mpc_plan_decay * best_residuals[1:] + self._residual_plan[-1] = 0.0 + return np.clip( + base_action + best_residuals[0], + -1.0, + 1.0, ) - self.phase += 0.43096661190230784 - return np.clip(action, -1.0, 1.0) + def _set_model_state(self, observation: np.ndarray) -> None: + assert self._model is not None + assert self._rollout_data is not None + self._rollout_data.qpos[0:2] = 0.0 + self._rollout_data.qpos[2:] = observation[:13] + self._rollout_data.qvel[:] = observation[13:27] + mujoco.mj_forward(self._model, self._rollout_data) + + def _rollout_objective( + self, + observation: np.ndarray, + residuals: np.ndarray, + ) -> float: + assert self._model is not None + assert self._rollout_data is not None + self._set_model_state(observation) + objective = 0.0 + phase = self.phase + for horizon_index, residual in enumerate(residuals): + q = self._rollout_data.qpos[7:15][Q_INDEX] + dq = self._rollout_data.qvel[6:14][Q_INDEX] + x_velocity_before = float(self._rollout_data.qvel[0]) + roll, pitch, yaw = _euler(self._rollout_data.qpos[3:7]) + roll_rate = float(self._rollout_data.qvel[3]) + pitch_rate = float(self._rollout_data.qvel[4]) + yaw_rate = float(self._rollout_data.qvel[5]) + if horizon_index == 0: + foot_contacts = _observation_foot_contacts(observation) + else: + foot_contacts = np.clip( + self._rollout_data.cfrc_ext[FOOT_BODY_IDS, 5], + 0.0, + 1.0, + ) + stance_duty = _adaptive_stance_duty( + CONFIG, + x_velocity_before, + ) + action = np.clip( + _rhythmic_action( + CONFIG, + phase, + stance_duty, + q, + dq, + roll, + pitch, + yaw, + roll_rate, + pitch_rate, + yaw_rate, + foot_contacts, + ) + + residual, + -1.0, + 1.0, + ) + x_before = float(self._rollout_data.qpos[0]) + self._rollout_data.ctrl[:] = action + for _ in range(self._frame_skip): + mujoco.mj_step(self._model, self._rollout_data) + + dt = self._frame_skip * self._model.opt.timestep + x_velocity = (float(self._rollout_data.qpos[0]) - x_before) / dt + z_position = float(self._rollout_data.qpos[2]) + roll, pitch, yaw = _euler(self._rollout_data.qpos[3:7]) + objective += CONFIG.mpc_forward_weight * x_velocity + ( + 1.0 if 0.2 <= z_position <= 1.0 else -50.0 + ) + objective -= CONFIG.mpc_ctrl_cost * float(np.square(action).sum()) + objective -= CONFIG.mpc_pose_cost * ( + roll * roll + + (pitch - CONFIG.mpc_pitch_target) * (pitch - CONFIG.mpc_pitch_target) + ) + objective -= CONFIG.mpc_yaw_cost * yaw * yaw + objective -= CONFIG.mpc_z_cost * (z_position - CONFIG.mpc_z_target) ** 2 + if z_position < 0.23 or z_position > 0.95: + objective -= 100.0 + phase += _adaptive_dphi(CONFIG, x_velocity) + + joint_velocity = self._rollout_data.qvel[6:14] + objective -= CONFIG.mpc_terminal_vel_cost * float( + np.dot(joint_velocity, joint_velocity) + ) + return objective diff --git a/tasks/simulation_heuristics_ant_v1/task.md b/tasks/simulation_heuristics_ant_v1/task.md index f3f8b7f..3f286a0 100644 --- a/tasks/simulation_heuristics_ant_v1/task.md +++ b/tasks/simulation_heuristics_ant_v1/task.md @@ -11,13 +11,15 @@ task: - ant - mujoco - continuous-control + - model-predictive-control metadata: category: locomotion - difficulty: medium + difficulty: hard tags: - mujoco - continuous-control - policy-search + - model-predictive-control - robotics reference_task: true genesisbench: @@ -39,7 +41,7 @@ agent: user: agent network_mode: public verifier: - timeout_sec: 300 + timeout_sec: 4200 user: root network_mode: no-network environment: @@ -80,6 +82,15 @@ The environment has: may modify it, replace it, add training or search code, bundle learned weights, or implement model-predictive control. +The trusted reference adapts the final Ant controller from *Learning Beyond +Gradients*: a speed-adaptive, asymmetric rhythmic gait plus warm-started +residual MPC with 96 candidate plans and a 10-step horizon. The article's +EnvPool rerun over seeds `0..4` reported: + +```text +mean 6005.521, min 5776.805, max 6146.208 +``` + ## Development evaluation Run: @@ -97,6 +108,20 @@ python evaluate.py \ --max-steps 150 ``` +For a full five-seed reproduction: + +```bash +python evaluate.py \ + --policy path/to/policy.py \ + --episodes 5 \ + --max-steps 1000 \ + --seed 0 +``` + +The article MPC controller is CPU-intensive. Its copied-model rollouts do not +increase the external episode-step count, but they consume wall-clock time and +are included in action-latency metrics. + The public evaluator uses visible development seeds. The final evaluator uses unseen seeds and conservative unseen dynamics variants. A policy that memorizes the development episodes is unlikely to score well. @@ -129,6 +154,21 @@ The returned object must implement `reset` and `act`. Each action must be a finite NumPy-compatible array with shape `(8,)`. Values outside `[-1, 1]` are clipped. +For model-based planning, a policy may optionally implement: + +```python +def configure_simulator( + self, + *, + model_xml_path: str, + frame_skip: int, +) -> None: ... +``` + +The evaluator calls this once per episode with a path to a copied MuJoCo model. +It does not expose the live environment, reward, `info`, or hidden suite +configuration. + ## Rules 1. Work autonomously. Do not ask for user feedback. @@ -137,8 +177,9 @@ outside `[-1, 1]` are clipped. 4. Do not access `/oracle`, `/verifier`, or reconstruct hidden evaluation seeds or dynamics files. 5. Do not copy a completed Ant solution from the internet. -6. The final policy receives observations only. It does not receive reward, - environment `info`, simulator objects, or hidden variant parameters. +6. The final policy receives observations, its reset seed, and the optional + copied-model configuration hook only. It does not receive reward, + environment `info`, or a live simulator object. 7. The final policy must run offline with the packages already installed. 8. Before finishing, evaluate `final_policy/policy.py` and leave the best working version in place. diff --git a/tasks/simulation_heuristics_ant_v1/task_context/article_reference.md b/tasks/simulation_heuristics_ant_v1/task_context/article_reference.md new file mode 100644 index 0000000..2857f32 --- /dev/null +++ b/tasks/simulation_heuristics_ant_v1/task_context/article_reference.md @@ -0,0 +1,33 @@ +# Learning Beyond Gradients Ant reference + +The trusted reference adapts the final `mpc` configuration from +`mujoco/ant/heuristic_ant.py` at revision +`3555c2956c257d49a5015b782cbe485b14fd659e`. + +Its controller is composed of: + +- an asymmetric four-leg CPG with second- and third-order harmonics; +- PD tracking plus roll, pitch, yaw, and angular-rate feedback; +- phase increment and stance duty adapted to measured forward velocity; +- a 10-step copied-MuJoCo planning horizon; +- 96 sampled residual-action plans per external step; +- temporally smoothed residual noise; +- warm-start plan shifting with decay `0.504186948858276`; +- forward, control, posture, yaw, height, health, and terminal joint-velocity + terms in the planning objective. + +The article's EnvPool command used five episodes from seed `0` and reported: + +```text +mean 6005.521 +min 5776.805 +max 6146.208 +``` + +The policy's planning RNG is reset to seed `12` every episode. Environment +reset seeds remain controlled independently by GenesisBench. + +The source controller and XML are Apache-2.0 licensed by Garena Online Private +Limited. Machine-readable hashes, licensing, target metrics, and platform +reproduction results live in `evidence/source_provenance.json`, which is +excluded from the public agent workspace. diff --git a/tasks/simulation_heuristics_ant_v1/task_context/evaluation.md b/tasks/simulation_heuristics_ant_v1/task_context/evaluation.md index d72ca07..b097174 100644 --- a/tasks/simulation_heuristics_ant_v1/task_context/evaluation.md +++ b/tasks/simulation_heuristics_ant_v1/task_context/evaluation.md @@ -18,7 +18,27 @@ The public evaluator reports: - mean action latency; - per-episode metrics. -The final leaderboard score is the mean return over hidden nominal episodes -and hidden conservative dynamics variants. Invalid policies receive a fixed -failure return for the affected episode. +The checked-in hidden suite runs full 1,000-step episodes over unseen nominal +seeds and two conservative dynamics variants. Its raw score is: +```text +0.70 * hidden nominal mean return ++ 0.30 * hidden dynamics-robustness mean return +``` + +The score is normalized locally so the weak public CPG/PD starter maps to `0` +and the article residual-MPC reference maps to `100`. Invalid policies receive +a fixed failure return for the affected episode. + +## Article reference + +The *Learning Beyond Gradients* EnvPool rerun reported seeds `0..4` with mean +`6005.521`, minimum `5776.805`, and maximum `6146.208`. + +GenesisBench runs Gymnasium rather than EnvPool. The supplied article XML and +Gymnasium's Ant XML parse to the same MuJoCo model, but reset RNG behavior +differs between runtimes. Exact source-policy checks on this platform are +recorded under `evidence/source_provenance.json`. + +Internal MPC model transitions are not external environment steps. They do +count against verifier wall-clock limits and action-latency metrics. diff --git a/tasks/simulation_heuristics_ant_v1/task_context/policy_api.md b/tasks/simulation_heuristics_ant_v1/task_context/policy_api.md index 6efcac0..3f3d240 100644 --- a/tasks/simulation_heuristics_ant_v1/task_context/policy_api.md +++ b/tasks/simulation_heuristics_ant_v1/task_context/policy_api.md @@ -30,3 +30,35 @@ to the eight Ant hinge actuators and must have shape `(8,)`. The policy may keep recurrent state and may load files stored below `final_policy/`. +## Optional simulator configuration + +Model-based policies may define: + +```python +def configure_simulator( + self, + *, + model_xml_path: str, + frame_skip: int, +) -> None: + ... +``` + +Call order is: + +```text +reset the environment +instantiate policy +configure_simulator(...) # when implemented +reset(seed=...) +act(observation) repeated for the episode +``` + +`model_xml_path` is a private per-episode XML copy matching the current +episode, including a conservative hidden dynamics variant when applicable. +Policies may load that model into their own `MjModel`/`MjData` for planning. + +The policy never receives the live `gymnasium.Env`, its mutable `MjData`, +reward, `info`, or hidden suite configuration. The copied model therefore +enables MPC without allowing the policy to step or mutate the scored +environment. diff --git a/tasks/simulation_heuristics_ant_v1/verifier/anchor_policies/reference_policy.py b/tasks/simulation_heuristics_ant_v1/verifier/anchor_policies/reference_policy.py index dbeded3..7ddaa2e 100644 --- a/tasks/simulation_heuristics_ant_v1/verifier/anchor_policies/reference_policy.py +++ b/tasks/simulation_heuristics_ant_v1/verifier/anchor_policies/reference_policy.py @@ -5,13 +5,26 @@ # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Warm-started speed-adaptive residual MPC reference for Ant-v5. -"""Frozen improved rhythmic reference used to calibrate Simulation Heuristics Ant v1.""" +Adapted for the GenesisBench policy API from ``mujoco/ant/heuristic_ant.py`` +at learning-beyond-gradients revision +``3555c2956c257d49a5015b782cbe485b14fd659e``. +""" from __future__ import annotations import math +from dataclasses import dataclass +import mujoco import numpy as np @@ -22,17 +35,192 @@ HEADING_AXIS = np.asarray([1.0, 1.0, -1.0, -1.0], dtype=np.float64) PITCH_AXIS = np.asarray([1.0, 1.0, -1.0, -1.0], dtype=np.float64) ROLL_AXIS = np.asarray([-1.0, 1.0, 1.0, -1.0], dtype=np.float64) +FOOT_BODY_IDS = np.asarray([13, 4, 7, 10], dtype=np.int64) +FOOT_OBS_ROWS = FOOT_BODY_IDS - 1 + + +@dataclass(frozen=True) +class ControllerConfig: + dphi: float = 0.660934259732249 + dphi_speed_gain: float = -0.02 + dphi_speed_target: float = 5.8 + dphi_min: float = 0.62 + dphi_max: float = 0.72 + hip_bias: float = 0.12217781430672398 + hip_amp: float = 0.5705418365199333 + ankle_bias: float = 0.36651046903486795 + ankle_amp: float = 0.26587749767314783 + kp: float = 0.8108143632989734 + kd: float = 0.0 + pitch_gain: float = -0.19444590796726124 + pitch_rate_gain: float = 0.04099276700871415 + roll_gain: float = -0.25536960225655303 + roll_rate_gain: float = 0.023293075237761272 + contact_hip_gain: float = 0.0 + contact_ankle_gain: float = 0.0 + contact_push_hip_amp: float = 0.0 + contact_push_ankle_amp: float = 0.0 + contact_push_phase: float = 2.356194490192345 + contact_push_width: float = 0.55 + stance_duty: float = 0.6355364206196007 + stance_duty_speed_gain: float = -0.01 + stance_duty_speed_target: float = 5.8 + stance_duty_min: float = 0.6 + stance_duty_max: float = 0.67 + hip_stance_scale: float = 1.0479076970107701 + hip_swing_scale: float = 1.0031777685985328 + ankle_stance_scale: float = 0.976603459922793 + ankle_swing_scale: float = 0.9374473230114526 + yaw_gain: float = -0.12067720879887742 + yaw_rate_gain: float = 0.04418873596679619 + hip_h2_amp: float = 0.10975404801587477 + hip_h2_phase: float = 2.0862256065597453 + ankle_h2_amp: float = -0.003434817287963554 + ankle_h2_phase: float = 1.2927488104774438 + hip_h3_amp: float = 0.04827596673280693 + hip_h3_phase: float = -0.49944083263433436 + ankle_h3_amp: float = -0.06968988354403895 + ankle_h3_phase: float = 1.5873441034476188 + mpc_horizon: int = 10 + mpc_candidates: int = 96 + mpc_sigma: float = 0.07614211639071694 + mpc_clip: float = 0.12016284361036686 + mpc_pose_cost: float = 23.348190567885954 + mpc_pitch_target: float = 0.0 + mpc_yaw_cost: float = 2.7292168081366723 + mpc_z_cost: float = 2.1215830559511737 + mpc_z_target: float = 0.4519975076600261 + mpc_forward_weight: float = 1.0 + mpc_ctrl_cost: float = 0.5 + mpc_terminal_vel_cost: float = 0.01 + mpc_plan_decay: float = 0.504186948858276 + mpc_seed: int = 12 + + +CONFIG = ControllerConfig() + + +def _warp_leg_phase(leg_phase: np.ndarray, stance_duty: float) -> np.ndarray: + clipped_duty = float(np.clip(stance_duty, 0.05, 0.95)) + if abs(clipped_duty - 0.5) < 1e-12: + return np.mod(leg_phase, 2.0 * math.pi) + + phase_unit = np.mod(leg_phase, 2.0 * math.pi) / (2.0 * math.pi) + stance_unit = 0.5 * phase_unit / clipped_duty + swing_unit = 0.5 + 0.5 * (phase_unit - clipped_duty) / (1.0 - clipped_duty) + warped_unit = np.where( + phase_unit < clipped_duty, + stance_unit, + swing_unit, + ) + return 2.0 * math.pi * warped_unit + + +def _adaptive_dphi(config: ControllerConfig, x_velocity: float) -> float: + dphi = config.dphi + config.dphi_speed_gain * ( + x_velocity - config.dphi_speed_target + ) + return float(np.clip(dphi, config.dphi_min, config.dphi_max)) + + +def _adaptive_stance_duty( + config: ControllerConfig, + x_velocity: float, +) -> float: + stance_duty = config.stance_duty + config.stance_duty_speed_gain * ( + x_velocity - config.stance_duty_speed_target + ) + return float( + np.clip( + stance_duty, + config.stance_duty_min, + config.stance_duty_max, + ) + ) + + +def _rhythmic_action( + config: ControllerConfig, + phase: float, + stance_duty: float, + q: np.ndarray, + dq: np.ndarray, + roll: float, + pitch: float, + yaw: float, + roll_rate: float, + pitch_rate: float, + yaw_rate: float, + foot_contacts: np.ndarray, +) -> np.ndarray: + leg_phase = _warp_leg_phase(phase + LEG_PHASE, stance_duty) + stance_mask = leg_phase < math.pi + hip_wave = config.hip_bias + np.where( + stance_mask, + config.hip_stance_scale, + config.hip_swing_scale, + ) * ( + config.hip_amp * np.sin(leg_phase) + + config.hip_h2_amp * np.sin(2.0 * leg_phase + config.hip_h2_phase) + + config.hip_h3_amp * np.sin(3.0 * leg_phase + config.hip_h3_phase) + ) + ankle_wave = config.ankle_bias + np.where( + stance_mask, + config.ankle_stance_scale, + config.ankle_swing_scale, + ) * ( + config.ankle_amp * np.cos(leg_phase) + + config.ankle_h2_amp * np.cos(2.0 * leg_phase + config.ankle_h2_phase) + + config.ankle_h3_amp * np.cos(3.0 * leg_phase + config.ankle_h3_phase) + ) + push_width = max(config.contact_push_width, 1e-3) + contact_push = ( + np.exp(-0.5 * np.square((leg_phase - config.contact_push_phase) / push_width)) + * stance_mask + * foot_contacts + ) + hip_wave = hip_wave + config.contact_push_hip_amp * contact_push + ankle_wave = ankle_wave + config.contact_push_ankle_amp * contact_push + balance_wave = PITCH_AXIS * ( + config.pitch_gain * pitch + config.pitch_rate_gain * pitch_rate + ) - ROLL_AXIS * (config.roll_gain * roll + config.roll_rate_gain * roll_rate) + + action = np.empty(8, dtype=np.float64) + action[0::2] = ( + config.kp + * ( + HIP_SIGN * hip_wave + + HEADING_AXIS + * ( + config.yaw_gain * yaw + + config.yaw_rate_gain * yaw_rate + + config.contact_hip_gain * foot_contacts + ) + - q[0::2] + ) + - config.kd * dq[0::2] + ) + action[1::2] = ( + config.kp + * ( + ANKLE_SIGN + * (ankle_wave + balance_wave + config.contact_ankle_gain * foot_contacts) + - q[1::2] + ) + - config.kd * dq[1::2] + ) + return np.clip(action, -1.0, 1.0) -def _rpy(quaternion: np.ndarray) -> tuple[float, float, float]: - w, x, y, z = quaternion / (np.linalg.norm(quaternion) + 1e-12) +def _euler(quaternion: np.ndarray) -> tuple[float, float, float]: + quaternion = np.asarray(quaternion, dtype=np.float64) + quaternion /= np.linalg.norm(quaternion) + 1e-12 + w, x, y, z = quaternion roll = math.atan2( 2.0 * (w * x + y * z), 1.0 - 2.0 * (x * x + y * y), ) - pitch = math.asin( - np.clip(2.0 * (w * y - z * x), -1.0, 1.0) - ) + pitch = math.asin(np.clip(2.0 * (w * y - z * x), -1.0, 1.0)) yaw = math.atan2( 2.0 * (w * z + x * y), 1.0 - 2.0 * (y * y + z * z), @@ -40,45 +228,228 @@ def _rpy(quaternion: np.ndarray) -> tuple[float, float, float]: return roll, pitch, yaw +def _observation_array(observation: np.ndarray) -> np.ndarray: + array = np.asarray(observation) + if array.ndim == 2: + array = array[0] + if array.ndim != 1 or array.shape[0] < 27: + raise ValueError(f"Unsupported Ant observation shape: {array.shape}") + return array + + +def _joint_state(observation: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + array = _observation_array(observation) + return array[5:13], array[19:27] + + +def _forward_velocity(observation: np.ndarray) -> float: + return float(_observation_array(observation)[13]) + + +def _torso_state( + observation: np.ndarray, +) -> tuple[float, float, float, float, float, float]: + array = _observation_array(observation) + quaternion = np.asarray(array[1:5], dtype=np.float64) + quaternion /= np.linalg.norm(quaternion) + 1e-12 + roll, pitch, yaw = _euler(quaternion) + return ( + roll, + pitch, + yaw, + float(array[16]), + float(array[17]), + float(array[18]), + ) + + +def _observation_foot_contacts(observation: np.ndarray) -> np.ndarray: + array = _observation_array(observation) + if array.shape[0] < 27 + 13 * 6: + return np.zeros(4, dtype=np.float64) + contact_forces = array[27 : 27 + 13 * 6].reshape(13, 6) + return np.clip(contact_forces[FOOT_OBS_ROWS, 5], 0.0, 1.0) + + class Policy: def __init__(self, seed: int = 0) -> None: + self._model: mujoco.MjModel | None = None + self._rollout_data: mujoco.MjData | None = None + self._frame_skip = 5 + self._residual_plan = np.zeros( + (CONFIG.mpc_horizon, 8), + dtype=np.float64, + ) self.reset(seed) + def configure_simulator( + self, + *, + model_xml_path: str, + frame_skip: int = 5, + ) -> None: + model = mujoco.MjModel.from_xml_path(model_xml_path) + if (model.nq, model.nv, model.nu) != (15, 14, 8): + raise ValueError( + "Expected Ant model dimensions (15, 14, 8), " + f"got {(model.nq, model.nv, model.nu)}" + ) + self._model = model + self._rollout_data = mujoco.MjData(model) + self._frame_skip = int(frame_skip) + def reset(self, seed: int = 0) -> None: + self.seed = int(seed) self.phase = 0.0 + self._rng = np.random.default_rng(CONFIG.mpc_seed) + self._residual_plan.fill(0.0) def act(self, observation: np.ndarray) -> np.ndarray: - observation = np.asarray(observation, dtype=np.float64) - q = observation[5:13][Q_INDEX] - _, _, yaw = _rpy(observation[1:5]) - yaw_rate = float(observation[18]) - leg_phase = self.phase + LEG_PHASE - - hip_wave = 0.12217781430672398 + ( - 0.3618286792202296 * np.sin(leg_phase) - + 0.10975404801587477 - * np.sin(2.0 * leg_phase + 2.0862256065597453) - + 0.04827596673280693 - * np.sin(3.0 * leg_phase - 0.4994408326343364) - ) - ankle_wave = 0.36651046903486795 + ( - 0.3226052803819716 * np.cos(leg_phase) - - 0.003434817287963554 - * np.cos(2.0 * leg_phase + 1.2927488104774438) - - 0.06968988354403895 - * np.cos(3.0 * leg_phase + 1.5873441034476188) + if self._model is None or self._rollout_data is None: + raise RuntimeError("configure_simulator must be called before act") + observation = _observation_array(observation) + self._set_model_state(observation) + q = self._rollout_data.qpos[7:15][Q_INDEX] + dq = self._rollout_data.qvel[6:14][Q_INDEX] + x_velocity = _forward_velocity(observation) + ( + roll, + pitch, + yaw, + roll_rate, + pitch_rate, + yaw_rate, + ) = _torso_state(observation) + foot_contacts = _observation_foot_contacts(observation) + stance_duty = _adaptive_stance_duty(CONFIG, x_velocity) + base_action = _rhythmic_action( + CONFIG, + self.phase, + stance_duty, + q, + dq, + roll, + pitch, + yaw, + roll_rate, + pitch_rate, + yaw_rate, + foot_contacts, ) - action = np.empty(8, dtype=np.float64) - action[0::2] = 0.5493824068786054 * ( - HIP_SIGN * hip_wave - + HEADING_AXIS - * (-0.12067720879887742 * yaw + 0.04418873596679619 * yaw_rate) - - q[0::2] + best_residuals = self._residual_plan.copy() + best_objective = self._rollout_objective( + observation, + best_residuals, ) - action[1::2] = 0.5493824068786054 * ( - ANKLE_SIGN * ankle_wave - q[1::2] + for _ in range(CONFIG.mpc_candidates - 1): + residuals = np.clip( + best_residuals + + self._rng.normal( + 0.0, + CONFIG.mpc_sigma, + size=(CONFIG.mpc_horizon, 8), + ), + -CONFIG.mpc_clip, + CONFIG.mpc_clip, + ) + residuals[1:] = 0.6 * residuals[1:] + 0.4 * residuals[:-1] + objective = self._rollout_objective(observation, residuals) + if objective > best_objective: + best_objective = objective + best_residuals = residuals + + self.phase += _adaptive_dphi(CONFIG, x_velocity) + self._residual_plan[:-1] = CONFIG.mpc_plan_decay * best_residuals[1:] + self._residual_plan[-1] = 0.0 + return np.clip( + base_action + best_residuals[0], + -1.0, + 1.0, ) - self.phase += 0.43096661190230784 - return np.clip(action, -1.0, 1.0) + def _set_model_state(self, observation: np.ndarray) -> None: + assert self._model is not None + assert self._rollout_data is not None + self._rollout_data.qpos[0:2] = 0.0 + self._rollout_data.qpos[2:] = observation[:13] + self._rollout_data.qvel[:] = observation[13:27] + mujoco.mj_forward(self._model, self._rollout_data) + + def _rollout_objective( + self, + observation: np.ndarray, + residuals: np.ndarray, + ) -> float: + assert self._model is not None + assert self._rollout_data is not None + self._set_model_state(observation) + objective = 0.0 + phase = self.phase + for horizon_index, residual in enumerate(residuals): + q = self._rollout_data.qpos[7:15][Q_INDEX] + dq = self._rollout_data.qvel[6:14][Q_INDEX] + x_velocity_before = float(self._rollout_data.qvel[0]) + roll, pitch, yaw = _euler(self._rollout_data.qpos[3:7]) + roll_rate = float(self._rollout_data.qvel[3]) + pitch_rate = float(self._rollout_data.qvel[4]) + yaw_rate = float(self._rollout_data.qvel[5]) + if horizon_index == 0: + foot_contacts = _observation_foot_contacts(observation) + else: + foot_contacts = np.clip( + self._rollout_data.cfrc_ext[FOOT_BODY_IDS, 5], + 0.0, + 1.0, + ) + stance_duty = _adaptive_stance_duty( + CONFIG, + x_velocity_before, + ) + action = np.clip( + _rhythmic_action( + CONFIG, + phase, + stance_duty, + q, + dq, + roll, + pitch, + yaw, + roll_rate, + pitch_rate, + yaw_rate, + foot_contacts, + ) + + residual, + -1.0, + 1.0, + ) + x_before = float(self._rollout_data.qpos[0]) + self._rollout_data.ctrl[:] = action + for _ in range(self._frame_skip): + mujoco.mj_step(self._model, self._rollout_data) + + dt = self._frame_skip * self._model.opt.timestep + x_velocity = (float(self._rollout_data.qpos[0]) - x_before) / dt + z_position = float(self._rollout_data.qpos[2]) + roll, pitch, yaw = _euler(self._rollout_data.qpos[3:7]) + objective += CONFIG.mpc_forward_weight * x_velocity + ( + 1.0 if 0.2 <= z_position <= 1.0 else -50.0 + ) + objective -= CONFIG.mpc_ctrl_cost * float(np.square(action).sum()) + objective -= CONFIG.mpc_pose_cost * ( + roll * roll + + (pitch - CONFIG.mpc_pitch_target) * (pitch - CONFIG.mpc_pitch_target) + ) + objective -= CONFIG.mpc_yaw_cost * yaw * yaw + objective -= CONFIG.mpc_z_cost * (z_position - CONFIG.mpc_z_target) ** 2 + if z_position < 0.23 or z_position > 0.95: + objective -= 100.0 + phase += _adaptive_dphi(CONFIG, x_velocity) + + joint_velocity = self._rollout_data.qvel[6:14] + objective -= CONFIG.mpc_terminal_vel_cost * float( + np.dot(joint_velocity, joint_velocity) + ) + return objective diff --git a/tasks/simulation_heuristics_ant_v1/verifier/anchors.json b/tasks/simulation_heuristics_ant_v1/verifier/anchors.json index 297bdbd..3372d2f 100644 --- a/tasks/simulation_heuristics_ant_v1/verifier/anchors.json +++ b/tasks/simulation_heuristics_ant_v1/verifier/anchors.json @@ -2,7 +2,7 @@ "environment": "Gymnasium Ant-v5 with 27 observations and no contact cost", "max_steps": 1000, "reference_policy": { - "description": "Frozen higher-harmonic rhythmic policy", + "description": "Article warm-started speed-adaptive residual MPC policy", "path": "anchor_policies/reference_policy.py" }, "starter_policy": { diff --git a/tasks/simulation_heuristics_ant_v1/verifier/config_smoke.toml b/tasks/simulation_heuristics_ant_v1/verifier/config_smoke.toml new file mode 100644 index 0000000..7788bb8 --- /dev/null +++ b/tasks/simulation_heuristics_ant_v1/verifier/config_smoke.toml @@ -0,0 +1,22 @@ +version = "1.0" + +[evaluation] +max_steps = 1 +failure_return = -1000.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.7 +seeds = [101] + +[[evaluation.suites]] +name = "hidden_robustness" +weight = 0.3 +seeds = [503] + +[[evaluation.variants]] +name = "light_low_friction" +density_scale = 0.9 +friction_scale = 0.85 +damping_scale = 0.95 +actuator_scale = 1.0 diff --git a/tasks/simulation_heuristics_ant_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_ant_v1/verifier/evaluate_hidden.py index 022dcda..10b4592 100644 --- a/tasks/simulation_heuristics_ant_v1/verifier/evaluate_hidden.py +++ b/tasks/simulation_heuristics_ant_v1/verifier/evaluate_hidden.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse +import hashlib import json import sys import tomllib @@ -10,9 +11,7 @@ VERIFIER_DIR = Path(__file__).resolve().parent RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] -RUNTIME_CANDIDATES.extend( - ancestor / "src" for ancestor in VERIFIER_DIR.parents -) +RUNTIME_CANDIDATES.extend(ancestor / "src" for ancestor in VERIFIER_DIR.parents) for candidate in RUNTIME_CANDIDATES: if (candidate / "genesisbench").is_dir(): sys.path.insert(0, str(candidate)) @@ -25,6 +24,9 @@ ) +EvaluationBundle = tuple[AntEvaluation, AntEvaluation, float] + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Run hidden Ant evaluation.") parser.add_argument("policy", type=Path) @@ -44,11 +46,20 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def _policy_path(path: Path) -> Path: + resolved = path.resolve() + return resolved / "policy.py" if resolved.is_dir() else resolved + + +def _policy_fingerprint(path: Path) -> str: + return hashlib.sha256(_policy_path(path).read_bytes()).hexdigest() + + def _evaluate_raw( policy: Path, *, evaluation: dict, -) -> tuple[AntEvaluation, AntEvaluation, float]: +) -> EvaluationBundle: suites = {suite["name"]: suite for suite in evaluation["suites"]} nominal = evaluate_ant_policy( policy, @@ -57,9 +68,7 @@ def _evaluate_raw( variants=(DynamicsVariant(),), failure_return=evaluation["failure_return"], ) - variants = tuple( - DynamicsVariant(**variant) for variant in evaluation["variants"] - ) + variants = tuple(DynamicsVariant(**variant) for variant in evaluation["variants"]) robustness = evaluate_ant_policy( policy, seeds=suites["hidden_robustness"]["seeds"], @@ -74,12 +83,28 @@ def _evaluate_raw( return nominal, robustness, score +def _evaluate_cached( + policy: Path, + *, + evaluation: dict, + cache: dict[str, EvaluationBundle], +) -> EvaluationBundle: + fingerprint = _policy_fingerprint(policy) + if fingerprint not in cache: + cache[fingerprint] = _evaluate_raw( + policy, + evaluation=evaluation, + ) + return cache[fingerprint] + + def _anchor_score( anchors: dict, name: str, *, anchors_path: Path, evaluation: dict, + cache: dict[str, EvaluationBundle], ) -> float: anchor = anchors[name] score = anchor.get("score") @@ -89,9 +114,10 @@ def _anchor_score( if not isinstance(relative_path, str) or not relative_path: raise ValueError(f"{name} must declare score or path") policy = anchors_path.parent / relative_path - _, _, calibrated_score = _evaluate_raw( + _, _, calibrated_score = _evaluate_cached( policy, evaluation=evaluation, + cache=cache, ) return calibrated_score @@ -102,21 +128,25 @@ def main() -> None: anchors = json.loads(args.anchors.read_text()) evaluation = config["evaluation"] suites = {suite["name"]: suite for suite in evaluation["suites"]} - nominal, robustness, score = _evaluate_raw( + cache: dict[str, EvaluationBundle] = {} + nominal, robustness, score = _evaluate_cached( args.policy, evaluation=evaluation, + cache=cache, ) starter_score = _anchor_score( anchors, "starter_policy", anchors_path=args.anchors, evaluation=evaluation, + cache=cache, ) reference_score = _anchor_score( anchors, "reference_policy", anchors_path=args.anchors, evaluation=evaluation, + cache=cache, ) if reference_score == starter_score: raise ValueError("starter and reference anchors must have different scores") @@ -132,6 +162,7 @@ def main() -> None: "reference_score": reference_score, "hidden_nominal_weight": suites["hidden_nominal"]["weight"], "hidden_robustness_weight": suites["hidden_robustness"]["weight"], + "unique_policy_evaluations": len(cache), "hidden_nominal": nominal.to_dict(), "hidden_robustness": robustness.to_dict(), } diff --git a/tasks/simulation_heuristics_ant_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_ant_v1/verifier/rubrics/verifier.md index 15da717..292b37f 100644 --- a/tasks/simulation_heuristics_ant_v1/verifier/rubrics/verifier.md +++ b/tasks/simulation_heuristics_ant_v1/verifier/rubrics/verifier.md @@ -1,4 +1,4 @@ # Simulation Heuristics Ant v1 Verifier Rubric - `task_success`: normalized hidden-suite Ant return, where the starter policy - maps to `0` and the frozen stronger reference maps to `1`. + maps to `0` and the article residual-MPC reference maps to `1`. diff --git a/tasks/simulation_heuristics_ant_v1/verifier/test.sh b/tasks/simulation_heuristics_ant_v1/verifier/test.sh index 0d01f3d..4d050d2 100755 --- a/tasks/simulation_heuristics_ant_v1/verifier/test.sh +++ b/tasks/simulation_heuristics_ant_v1/verifier/test.sh @@ -10,11 +10,38 @@ if [ ! -f "$policy" ]; then exit 0 fi -python /verifier/evaluate_hidden.py \ +set +e +timeout --signal=TERM --kill-after=30s 3900s \ + python /verifier/evaluate_hidden.py \ "$policy" \ --config /verifier/config.toml \ --anchors /verifier/anchors.json \ --output /logs/verifier/genesis-score.json +evaluation_status=$? +set -e + +if [ "$evaluation_status" -eq 124 ] || [ "$evaluation_status" -eq 137 ]; then + python - <<'PY' +import json +from pathlib import Path + +Path("/logs/verifier/genesis-score.json").write_text( + json.dumps( + { + "score": None, + "normalized_score": 0.0, + "verifier_timeout": True, + "timeout_seconds": 3900, + }, + indent=2, + sort_keys=True, + ) + + "\n" +) +PY +elif [ "$evaluation_status" -ne 0 ]; then + exit "$evaluation_status" +fi python - <<'PY' import json @@ -27,4 +54,3 @@ Path("/logs/verifier/reward.json").write_text( json.dumps({"reward": reward}, indent=2) + "\n" ) PY - diff --git a/tasks/simulation_heuristics_ant_v1/verifier/verifier.md b/tasks/simulation_heuristics_ant_v1/verifier/verifier.md index ecee89a..6ff2349 100644 --- a/tasks/simulation_heuristics_ant_v1/verifier/verifier.md +++ b/tasks/simulation_heuristics_ant_v1/verifier/verifier.md @@ -30,3 +30,12 @@ dynamics variants. BenchFlow's canonical reward is the normalized GenesisBench score clamped to `[0, 1]`; the full raw and normalized metrics remain available in `genesis-score.json`. +The evaluator has an internal 3,900-second fail-closed deadline. A candidate +whose planning cost prevents hidden evaluation from finishing receives +normalized score `0` with `verifier_timeout=true`, rather than triggering an +expensive whole-task retry. + +Normalization evaluates local copies of the weak starter and article +residual-MPC reference. Policies with identical `policy.py` content share one +cached evaluation per verifier invocation, preventing the oracle from paying +the MPC reference cost twice. diff --git a/tasks/simulation_heuristics_atari57_v1/README.md b/tasks/simulation_heuristics_atari57_v1/README.md new file mode 100644 index 0000000..a8a74de --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/README.md @@ -0,0 +1,113 @@ +# Simulation Heuristics Atari57 v1 + +This package turns the article's Atari57 scaling experiment into one aggregate +GenesisBench task. The unit of submission is a manifest resolving 342 policy +slots, not a directory of duplicated standalone tasks. + +## Contract at a glance + +```text +57 games +x 2 observation modes +x 3 independent searches += 342 search trajectories + +342 x 20,000,000 frames += 6,840,000,000-frame full-search target +``` + +The final evaluator runs each repeat-specific policy on its corresponding +evaluation seed, computes per-episode human-normalized score (HNS), averages +the three independent policies within each game/mode, selects the better mode +per game, and takes the median over games. + +## Local commands + +Validate the starter without EnvPool: + +```bash +uv run python tasks/simulation_heuristics_atari57_v1/evaluate.py \ + --artifact tasks/simulation_heuristics_atari57_v1/starter_artifact \ + --validate-only +``` + +Run the deterministic verifier smoke suite: + +```bash +uv run python \ + tasks/simulation_heuristics_atari57_v1/verifier/evaluate_hidden.py \ + tasks/simulation_heuristics_atari57_v1/starter_artifact \ + --config \ + tasks/simulation_heuristics_atari57_v1/verifier/config_smoke.toml +``` + +Run the real public subset inside an environment containing exactly +`envpool==1.1.1`: + +```bash +python tasks/simulation_heuristics_atari57_v1/evaluate.py \ + --artifact tasks/simulation_heuristics_atari57_v1/starter_artifact +``` + +Validate the native package: + +```bash +uv run bench tasks check \ + tasks/simulation_heuristics_atari57_v1 \ + --level publication-grade +``` + +Run the contract oracle through BenchFlow: + +```bash +uv run bench eval run \ + --tasks-dir tasks/simulation_heuristics_atari57_v1 \ + --agent oracle \ + --sandbox docker \ + --context-root . +``` + +## Full runtime versus deterministic tests + +The checked-in `verifier/config.toml` requires all 342 completed search records +before starting the complete 57-game × 2-mode × 3-repeat EnvPool evaluation. +Incomplete artifacts return zero immediately. `config_smoke.toml` disables +that eligibility gate only for deterministic contract tests. + +The bundled seeded-random oracle/reference artifact is only a runnable software +contract artifact. Its ledger remains at zero rather than pretending that the +342 article-scale searches were run, so the full verifier disqualifies it with +reward zero. It is not the `100` anchor and must not be cited as a reproduction +of the article's measured HNS. + +## Source-faithful reference metrics + +The task context preserves the supplied article measurements: + +- native-observation median HNS `0.31874552826138824` at `988,645` steps; +- RAM median HNS `0.25770816471064345` at `988,645` steps; +- native-observation median HNS `0.8079186493157826` at `9,746,987` steps; +- RAM median HNS `0.5914131823634771` at `9,746,987` steps; +- final best-input-mean median HNS `0.8283015254994576`, used as the numeric + normalized-score `100` anchor; +- final best-single-run median HNS `1.1813031161473089`. + +The score anchor is intentionally numeric; the remaining values are context, +not hard-coded evaluator outputs. + +## Package map + +| Path | Purpose | +| --- | --- | +| `task.md` | BenchFlow-native task and agent contract | +| `starter_artifact/` | Honest zero-search, 342-slot starter | +| `evaluate.py` | Representative public EnvPool evaluator | +| `task_context/` | Protocol, artifact, policy, scoring, and source metrics | +| `verifier/config.toml` | Full aggregate reproduction suite | +| `verifier/config_smoke.toml` | Deterministic lightweight contract suite | +| `verifier/evaluate_hidden.py` | Hidden aggregate HNS scorer | +| `verifier/anchors.json` | Numeric HNS `0` and article `100` anchors | +| `oracle/reference_artifact/` | Runnable non-target contract artifact | + +Production deployments may inject private evaluation seeds, but the published +numeric HNS anchors and complete-ledger eligibility gate remain explicit. diff --git a/tasks/simulation_heuristics_atari57_v1/environment/Dockerfile b/tasks/simulation_heuristics_atari57_v1/environment/Dockerfile new file mode 100644 index 0000000..16fb533 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/environment/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.11-slim + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends \ + bash \ + ca-certificates \ + libgl1 \ + libglib2.0-0 \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir \ + "envpool==1.1.1" \ + "gymnasium[mujoco]>=1.2.2,<2" \ + "numpy>=1.26,<3" \ + "pyyaml>=6,<7" + +RUN useradd --create-home --uid 1000 agent \ + && mkdir -p /app /opt/genesisbench + +COPY src/genesisbench /opt/genesisbench/genesisbench +COPY tasks/simulation_heuristics_atari57_v1/evaluate.py /app/evaluate.py +COPY tasks/simulation_heuristics_atari57_v1/starter_artifact /app/starter_artifact +COPY tasks/simulation_heuristics_atari57_v1/task_context /app/task_context + +RUN cp -R /app/starter_artifact /app/final_artifact \ + && chown -R root:root /opt/genesisbench \ + && chmod -R a-w /opt/genesisbench \ + && chown -R agent:agent /app + +ENV PYTHONPATH="/opt/genesisbench" + +WORKDIR /app + +RUN mkdir -p /logs/verifier /logs/agent /logs/artifacts diff --git a/tasks/simulation_heuristics_atari57_v1/evaluate.py b/tasks/simulation_heuristics_atari57_v1/evaluate.py new file mode 100755 index 0000000..d6a7ca8 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/evaluate.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +from dataclasses import asdict +import json +import sys +from pathlib import Path + + +TASK_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = ( + TASK_DIR / "_runtime", + TASK_DIR.parent / "src", + TASK_DIR.parent.parent / "src", +) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.atari57 import ( # noqa: E402 + OBSERVATION_MODES, + evaluate_atari57_artifact, + load_atari57_artifact, + load_hns_references, +) + + +DEFAULT_GAMES = ( + "Breakout-v5", + "Freeway-v5", + "MontezumaRevenge-v5", + "Pong-v5", + "Seaquest-v5", + "Skiing-v5", +) + + +def _csv_strings(value: str) -> tuple[str, ...]: + return tuple(item.strip() for item in value.split(",") if item.strip()) + + +def _csv_ints(value: str) -> tuple[int, ...]: + return tuple(int(item) for item in _csv_strings(value)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate an aggregate Atari57 policy artifact." + ) + parser.add_argument( + "--artifact", + type=Path, + default=Path("final_artifact"), + ) + parser.add_argument( + "--games", + default=",".join(DEFAULT_GAMES), + help="Comma-separated Atari57 environment ids.", + ) + parser.add_argument( + "--obs-modes", + default=",".join(OBSERVATION_MODES), + help="Comma-separated observation modes.", + ) + parser.add_argument( + "--seeds", + default="101,202,303", + help=( + "Three comma-separated seeds mapped in order to repeat-specific " + "policy slots 0, 1, and 2." + ), + ) + parser.add_argument("--max-steps", type=int, default=5000) + parser.add_argument( + "--hns-table", + type=Path, + default=TASK_DIR / "task_context" / "atari57_games.csv", + ) + parser.add_argument( + "--validate-only", + action="store_true", + help="Validate the 342-slot manifest and interaction ledger only.", + ) + parser.add_argument("--json-output-file", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + artifact = load_atari57_artifact(args.artifact) + budget = asdict(artifact.interaction_budget) + if args.validate_only: + payload = { + "valid": True, + "artifact": str(artifact.root), + "policy_slots": len(artifact.policies), + "interaction_budget": budget, + "full_article_reproduction": ( + budget["completed_trajectories"] == budget["planned_trajectories"] + ), + } + else: + references = load_hns_references(args.hns_table) + result = evaluate_atari57_artifact( + artifact, + games=_csv_strings(args.games), + obs_modes=_csv_strings(args.obs_modes), + seeds=_csv_ints(args.seeds), + max_steps=args.max_steps, + hns_references=references, + ) + payload = { + **result.to_dict(), + "suite": "public_representative_subset", + "interaction_budget": budget, + "full_article_reproduction": False, + } + + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if args.json_output_file is not None: + args.json_output_file.parent.mkdir(parents=True, exist_ok=True) + args.json_output_file.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/interaction_ledger.json b/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/interaction_ledger.json new file mode 100644 index 0000000..a6035e1 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/interaction_ledger.json @@ -0,0 +1,4 @@ +{ + "schema_version": "1.0", + "records": [] +} diff --git a/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/manifest.json b/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/manifest.json new file mode 100644 index 0000000..9c8d7e1 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/manifest.json @@ -0,0 +1,37 @@ +{ + "schema_version": "1.0", + "protocol": { + "suite": "Atari57", + "envpool_version": "1.1.1", + "observation_modes": [ + "ram", + "native_obs" + ], + "search_repeats": 3, + "frame_budget_per_search": 20000000, + "expected_search_trajectories": 342, + "expected_policy_slots": 342, + "expected_total_frame_target": 6840000000, + "atari_settings": { + "img_height": 210, + "img_width": 160, + "stack_num": 1, + "gray_scale": false, + "frame_skip": 1, + "noop_max": 1, + "use_fire_reset": true, + "episodic_life": false, + "reward_clip": false, + "repeat_action_probability": 0.0, + "full_action_space": false + } + }, + "policy_defaults": { + "module": "policy.py", + "config": { + "strategy": "seeded_random_actions" + } + }, + "policies": [], + "interaction_ledger": "interaction_ledger.json" +} diff --git a/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/policy.py b/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/policy.py new file mode 100644 index 0000000..cdd5280 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/oracle/reference_artifact/policy.py @@ -0,0 +1,30 @@ +"""Runnable seeded-random contract artifact, not the article score anchor.""" + +from __future__ import annotations + +import numpy as np + + +class Policy: + def __init__( + self, + *, + action_count: int, + env_id: str = "", + repeat_index: int = 0, + seed: int = 0, + **_: object, + ) -> None: + self.action_count = action_count + self.offset = ( + sum(ord(character) for character in env_id) + 1_000_003 * repeat_index + ) + self.reset(seed=seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.rng = np.random.default_rng(seed + self.offset) + + def act(self, observation, info=None) -> int: + del observation, info + return int(self.rng.integers(self.action_count)) diff --git a/tasks/simulation_heuristics_atari57_v1/oracle/solve.sh b/tasks/simulation_heuristics_atari57_v1/oracle/solve.sh new file mode 100755 index 0000000..20f85d7 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/oracle/solve.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +rm -rf /app/final_artifact +cp -R /oracle/reference_artifact /app/final_artifact diff --git a/tasks/simulation_heuristics_atari57_v1/starter_artifact/interaction_ledger.json b/tasks/simulation_heuristics_atari57_v1/starter_artifact/interaction_ledger.json new file mode 100644 index 0000000..a6035e1 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/starter_artifact/interaction_ledger.json @@ -0,0 +1,4 @@ +{ + "schema_version": "1.0", + "records": [] +} diff --git a/tasks/simulation_heuristics_atari57_v1/starter_artifact/manifest.json b/tasks/simulation_heuristics_atari57_v1/starter_artifact/manifest.json new file mode 100644 index 0000000..514b85a --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/starter_artifact/manifest.json @@ -0,0 +1,37 @@ +{ + "schema_version": "1.0", + "protocol": { + "suite": "Atari57", + "envpool_version": "1.1.1", + "observation_modes": [ + "ram", + "native_obs" + ], + "search_repeats": 3, + "frame_budget_per_search": 20000000, + "expected_search_trajectories": 342, + "expected_policy_slots": 342, + "expected_total_frame_target": 6840000000, + "atari_settings": { + "img_height": 210, + "img_width": 160, + "stack_num": 1, + "gray_scale": false, + "frame_skip": 1, + "noop_max": 1, + "use_fire_reset": true, + "episodic_life": false, + "reward_clip": false, + "repeat_action_probability": 0.0, + "full_action_space": false + } + }, + "policy_defaults": { + "module": "policy.py", + "config": { + "strategy": "noop" + } + }, + "policies": [], + "interaction_ledger": "interaction_ledger.json" +} diff --git a/tasks/simulation_heuristics_atari57_v1/starter_artifact/policy.py b/tasks/simulation_heuristics_atari57_v1/starter_artifact/policy.py new file mode 100644 index 0000000..39fb6e7 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/starter_artifact/policy.py @@ -0,0 +1,22 @@ +"""Runnable zero-interaction baseline for every Atari57 game and input mode.""" + +from __future__ import annotations + + +class Policy: + def __init__( + self, + *, + action_count: int, + seed: int = 0, + **_: object, + ) -> None: + self.action_count = action_count + self.reset(seed=seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + + def act(self, observation, info=None) -> int: + del observation, info + return 0 diff --git a/tasks/simulation_heuristics_atari57_v1/task.md b/tasks/simulation_heuristics_atari57_v1/task.md new file mode 100644 index 0000000..06a536e --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/task.md @@ -0,0 +1,241 @@ +--- +schema_version: "1.3" +task: + name: genesisbench/simulation_heuristics_atari57_v1 + description: Improve one aggregate 342-policy Atari57 submission produced by 342 independently accounted heuristic-search trajectories. + authors: + - name: GenesisBench Contributors + keywords: + - simulation + - heuristics + - atari57 + - envpool + - batch-experiment +metadata: + category: atari + difficulty: hard + tags: + - atari57 + - programmatic-policy + - aggregate-evaluation + - sample-efficiency + - batch-search + reference_task: false + genesisbench: + starter: + path: starter_artifact + submission: + directory: final_artifact + entrypoint: manifest.json + development: + games: + - Breakout-v5 + - Freeway-v5 + - MontezumaRevenge-v5 + - Pong-v5 + - Seaquest-v5 + - Skiing-v5 + observation_modes: + - ram + - native_obs + repeats: 3 + seeds: [101, 202, 303] + max_steps: 5000 + verifier: + reproduction_config: verifier/config.toml + anchors: verifier/anchors.json + supports_private_config: true +agent: + timeout_sec: 604800 + user: agent + network_mode: no-network +verifier: + timeout_sec: 172800 + user: root + network_mode: no-network +environment: + build_timeout_sec: 1800 + cpus: 14 + memory_mb: 8192 + storage_mb: 102400 + network_mode: public + allow_internet: true + workdir: /app +benchflow: + document_version: "0.6" +--- +# Scale heuristic search across Atari57 + +Improve a single aggregate submission containing one programmatic policy slot +for every Atari57 game, observation mode, and independent repeat. This is the +batch/scaling experiment. Do not split it into standalone tasks. + +## Article protocol + +The experiment matrix is: + +```text +57 games x 2 observation modes x 3 repeats = 342 search trajectories +``` + +Each search trajectory has a target of `20,000,000` Atari frames. Because +`frame_skip=1`, one environment step is one counted frame. The complete target +across the matrix is: + +```text +342 x 20,000,000 = 6,840,000,000 counted environment steps +``` + +Every environment call used for probing, debugging, tuning, comparison, or +final validation must be included in the corresponding ledger record. Never +invent completed searches or interaction counts. A partial submission remains +evaluable with the public development tool, but the official verifier assigns +it zero rather than treating it as an article-faithful reproduction. + +The three repeats must be independent search trajectories. Use `repeat_index` +as a seed offset or other decorrelation mechanism rather than replaying the +same search three times. + +## Fixed Atari environment + +All search and evaluation environments use EnvPool `1.1.1` and these exact +settings: + +```python +envpool.make_gym( + ENV_ID, + num_envs=1, + batch_size=1, + seed=seed, + img_height=210, + img_width=160, + stack_num=1, + gray_scale=False, + frame_skip=1, + noop_max=1, + use_fire_reset=True, + episodic_life=False, + reward_clip=False, + repeat_action_probability=0.0, + full_action_space=False, +) +``` + +In particular, `frame_skip=1`, `reward_clip=false`, and sticky action is +disabled with `repeat_action_probability=0.0`. + +## Observation modes + +- `native_obs`: the policy may use only the observation returned by + `reset`/`step` plus its own internal state. The evaluator passes no `info`. +- `ram`: the policy may additionally use the public `info["ram"]` array. The + evaluator removes every other `info` field before calling the policy. + +Do not read EnvPool implementation files, tests, ROM internals, emulator state, +or hidden evaluator data. Do not use `render()` as an extra policy input. Do +not train a neural network or copy completed Atari policies from the internet. + +## Starting artifact + +`starter_artifact/` is a valid aggregate artifact. Its no-op policy is shared +by all 342 game/mode/repeat slots, and its sparse interaction ledger honestly +reports zero completed searches. + +The manifest supports a shared `policy_defaults` entry plus per-slot overrides. +You may share code between games or use a different module for every search. +The resolved artifact must still cover every tuple in: + +```text +Atari57 games x {ram, native_obs} x {0, 1, 2} +``` + +## Public development evaluation + +First validate the aggregate artifact without requiring EnvPool: + +```bash +python evaluate.py --artifact final_artifact --validate-only +``` + +Run the representative public subset: + +```bash +python evaluate.py --artifact final_artifact +``` + +The public subset evaluates all three repeat-specific policy slots on public +seeds for dense reward, sparse reward, negative-return, visual tracking, and +resource-management games. It is iteration feedback only; it is not evidence +that the complete 57-game protocol was reproduced. + +## Required final artifact + +Write the aggregate submission to: + +```text +final_artifact/ + manifest.json + interaction_ledger.json + policy.py # optional shared default + policies/ # optional per-game/mode/repeat modules + searches/ # evidence for completed search trajectories +``` + +`manifest.json` must retain the fixed protocol block. `policy_defaults` defines +the fallback module/configuration. Entries in `policies` override a specific +`env_id`, `obs_mode`, and `repeat_index`. + +Each selected module must define `Policy` or `make_policy`. A policy receives +`env_id`, `obs_mode`, `repeat_index`, `action_count`, `seed`, and its manifest +`config` when those arguments are accepted. Its action API is: + +```python +class Policy: + def reset(self, seed: int = 0) -> None: ... + def act(self, observation, info=None) -> int: ... +``` + +Return exactly one integer in `[0, action_count)`. + +`interaction_ledger.json` contains sparse records keyed by +`env_id / obs_mode / repeat_index`. Omitted entries count as `not_run`. +Use `status="complete"` only after counting at least `20,000,000` environment +steps for that search trajectory and retaining an `evidence_path` directory +with `policy.py`, `trials.jsonl`, `summary.csv`, `sample_efficiency.png`, and +`README.md`. Each record's best policy remains a separate repeat-specific +manifest slot. Do not collapse the three independent searches into one policy. + +## Final scoring + +For each evaluated episode: + +```text +HNS = (episode_return - random_score) / (human_score - random_score) +``` + +The aggregate follows the article's strict best-input-mean convention: + +1. mean HNS over the three repeats for each game and observation mode; +2. for each game, select the higher of the two mode means; +3. take the median across all 57 games. + +The verifier also reports mean HNS and the looser best-single-run median, but +the primary score is the strict median above. Natural HNS `0.0` maps to +normalized score `0`; the article's reported best-input-mean median HNS +`0.8283015254994576` maps to `100`. Values above `100` are valid. + +For context, the source article reported a final best-input-mean median HNS of +`0.8283015254994576` and a best-single-run median HNS of +`1.1813031161473089`. The first is intentionally the numeric normalization +reference. Neither value is a claim made by the starter or oracle artifacts. + +## Finish criteria + +Before exiting: + +1. run `--validate-only`; +2. run as much public evaluation as the available runtime permits; +3. leave the best working aggregate artifact in `final_artifact/`; +4. keep every interaction count honest; +5. document incomplete searches as incomplete rather than claiming a full + 6.84B-frame reproduction. diff --git a/tasks/simulation_heuristics_atari57_v1/task_context/article_protocol.md b/tasks/simulation_heuristics_atari57_v1/task_context/article_protocol.md new file mode 100644 index 0000000..016a08e --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/task_context/article_protocol.md @@ -0,0 +1,45 @@ +# Source article protocol + +The supplied source artifacts define one unattended heuristic-search run for +each: + +```text +57 Atari games x {ram, native_obs} x 3 repeats +``` + +That is 342 coding-agent search trajectories. + +Each trajectory: + +- uses EnvPool `1.1.1`; +- targets 20,000,000 counted frames; +- counts every probe, debug rollout, and trial; +- keeps `frame_skip=1`, `reward_clip=False`, and sticky action probability + `0.0`; +- forbids neural-network training and environment-source/hidden-state access; +- writes a best policy and experiment history; +- simplifies a new best policy and rechecks it before retaining it. + +The article reported these checkpoints from its aggregate curve: + +| Mode | Counted steps | Median HNS | +| --- | ---: | ---: | +| `native_obs` | 988,645 | 0.31874552826138824 | +| `ram` | 988,645 | 0.25770816471064345 | +| `native_obs` | 9,746,987 | 0.8079186493157826 | +| `ram` | 9,746,987 | 0.5914131823634771 | + +Its final cross-game summaries were: + +| Method | Mean HNS | Median HNS | Games with HNS >= 1 | +| --- | ---: | ---: | ---: | +| OpenAI Baselines PPO2 | 4.54578947368421 | 0.8 | 28 | +| CleanRL EnvPool PPO | 6.841228070175438 | 0.98 | 28 | +| Codex best input mean | 2.3427762956086453 | 0.8283015254994576 | 26 | +| Codex best single run | 3.8967649230310704 | 1.1813031161473089 | 36 | + +“Best single run” is intentionally reported as a looser diagnostic. The +GenesisBench primary metric uses best-input mean. + +These are historical source measurements. They are not embedded as evaluator +answers, and the deterministic smoke tests do not claim to reproduce them. diff --git a/tasks/simulation_heuristics_atari57_v1/task_context/article_reference_metrics.json b/tasks/simulation_heuristics_atari57_v1/task_context/article_reference_metrics.json new file mode 100644 index 0000000..f9b7ef3 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/task_context/article_reference_metrics.json @@ -0,0 +1,55 @@ +{ + "aggregate_curve_checkpoints": [ + { + "median_hns": 0.31874552826138824, + "obs_mode": "native_obs", + "steps": 988645 + }, + { + "median_hns": 0.25770816471064345, + "obs_mode": "ram", + "steps": 988645 + }, + { + "median_hns": 0.8079186493157826, + "obs_mode": "native_obs", + "steps": 9746987 + }, + { + "median_hns": 0.5914131823634771, + "obs_mode": "ram", + "steps": 9746987 + } + ], + "batch": { + "frame_budget_per_search": 20000000, + "games": 57, + "observation_modes": 2, + "repeats": 3, + "search_trajectories": 342, + "total_frame_target": 6840000000 + }, + "final_hns": { + "CleanRL EnvPool PPO": { + "games_ge_1": 28, + "mean": 6.841228070175438, + "median": 0.98 + }, + "Codex best input mean": { + "games_ge_1": 26, + "mean": 2.3427762956086453, + "median": 0.8283015254994576 + }, + "Codex best single run": { + "games_ge_1": 36, + "mean": 3.8967649230310704, + "median": 1.1813031161473089 + }, + "OpenAI Baselines PPO2": { + "games_ge_1": 28, + "mean": 4.54578947368421, + "median": 0.8 + } + }, + "source": "learning-beyond-gradients Atari57 artifacts" +} diff --git a/tasks/simulation_heuristics_atari57_v1/task_context/artifact_contract.md b/tasks/simulation_heuristics_atari57_v1/task_context/artifact_contract.md new file mode 100644 index 0000000..3c46c77 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/task_context/artifact_contract.md @@ -0,0 +1,73 @@ +# Aggregate artifact contract + +## Layout + +```text +final_artifact/ + manifest.json + interaction_ledger.json + policy.py + policies/ +``` + +Only the manifest and ledger are mandatory filenames. Policy modules may use +any safe relative path below the artifact root. + +## Manifest + +The fixed `protocol` block identifies EnvPool `1.1.1`, the two observation +modes, three search repeats, the 20M-frame per-search target, and the exact +Atari wrapper settings. + +`policy_defaults` contains a default `module` and `config`. `policies` is a +sparse override array: + +```json +{ + "env_id": "Breakout-v5", + "obs_mode": "native_obs", + "repeat_index": 1, + "module": "policies/breakout_native_repeat_1.py", + "config": {"threshold": 17} +} +``` + +The evaluator expands defaults plus overrides into exactly 342 +`env_id / obs_mode / repeat_index` slots and rejects unknown games, modes, +repeat indices, duplicate overrides, unsafe paths, or missing modules. Shared +defaults are allowed, but repeat identity is never discarded. + +## Interaction ledger + +The ledger is sparse. Each explicit record has: + +```json +{ + "env_id": "Breakout-v5", + "obs_mode": "native_obs", + "repeat_index": 0, + "cumulative_env_steps": 20001234, + "cumulative_episodes": 731, + "status": "complete" +} +``` + +Allowed statuses are `not_run`, `running`, `complete`, and `failed`. Omitted +matrix entries are treated as `not_run`. `complete` requires at least +20,000,000 counted environment steps and an `evidence_path` containing the +article outputs: + +```text +policy.py +trials.jsonl +summary.csv +sample_efficiency.png +README.md +``` + +The ledger is an auditable claim, not a mechanism that can prove no step was +omitted. + +The checked-in starter and reference keep an empty ledger. That is deliberate: +they validate the software path without claiming article-scale search. The full +verifier disqualifies both before EnvPool execution. diff --git a/tasks/simulation_heuristics_atari57_v1/task_context/atari57_games.csv b/tasks/simulation_heuristics_atari57_v1/task_context/atari57_games.csv new file mode 100644 index 0000000..a596309 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/task_context/atari57_games.csv @@ -0,0 +1,58 @@ +env_id,known_best_score,random_score,human_score +Alien-v5,741812.63,227.8,7127.7 +Amidar-v5,31331.37,5.8,1719.5 +Assault-v5,143972.03,222.4,742.0 +Asterix-v5,999403.53,210.0,8503.3 +Asteroids-v5,6785558.64,719.1,47388.7 +Atlantis-v5,1674767.2,12850.0,29028.1 +BankHeist-v5,38536.66,14.2,753.1 +BattleZone-v5,992600.31,2360.0,37187.5 +BeamRider-v5,4549993.53,363.9,16926.5 +Berzerk-v5,85932.6,123.7,2630.4 +Bowling-v5,260.13,23.1,160.7 +Boxing-v5,100.0,0.1,12.1 +Breakout-v5,864.0,1.7,30.5 +Centipede-v5,1159049.27,2090.9,12017.0 +ChopperCommand-v5,999900.0,811.0,7387.8 +CrazyClimber-v5,729482.83,10780.5,35829.4 +Defender-v5,839642.95,2874.5,18688.9 +DemonAttack-v5,143964.26,152.1,1971.0 +DoubleDunk-v5,24.0,-18.6,-16.4 +Enduro-v5,2382.44,0.0,860.5 +FishingDerby-v5,91.16,-91.7,-38.7 +Freeway-v5,34.0,0.0,29.6 +Frostbite-v5,631378.53,65.2,4334.7 +Gopher-v5,130345.58,257.6,2412.5 +Gravitar-v5,21068.03,173.0,3351.4 +Hero-v5,114736.26,1027.0,30826.4 +IceHockey-v5,86.89,-11.2,0.9 +Jamesbond-v5,158142.36,29.0,302.8 +Kangaroo-v5,37392.82,52.0,3035.0 +Krull-v5,291043.06,1598.0,2665.5 +KungFuMaster-v5,267766.63,258.5,22736.3 +MontezumaRevenge-v5,19093.74,0.0,4753.3 +MsPacman-v5,243401.1,307.3,6951.6 +NameThisGame-v5,157177.85,2292.3,8049.0 +Phoenix-v5,990638.12,761.4,7242.6 +Pitfall-v5,18756.01,-229.4,6463.7 +Pong-v5,21.0,-20.7,14.6 +PrivateEye-v5,100314.44,24.9,69571.3 +Qbert-v5,777071.3,163.9,13455.0 +Riverraid-v5,323417.18,1338.5,17118.0 +RoadRunner-v5,613659.42,11.5,7845.0 +Robotank-v5,144.0,2.2,11.9 +Seaquest-v5,999999.0,68.4,42054.7 +Skiing-v5,-3851.44,-17098.1,-4336.9 +Solaris-v5,67306.29,1236.3,12326.7 +SpaceInvaders-v5,74335.3,148.0,1668.7 +StarGunner-v5,998600.28,664.0,10250.0 +Surround-v5,10.0,-10.0,6.5 +Tennis-v5,24.0,-23.8,-8.3 +TimePilot-v5,476763.9,3568.0,5229.2 +Tutankham-v5,2354.91,11.4,167.6 +UpNDown-v5,715545.61,533.4,11693.2 +Venture-v5,2623.71,0.0,1187.5 +VideoPinball-v5,999697.05,0.0,17667.9 +WizardOfWor-v5,197126.0,563.5,4756.5 +YarsRevenge-v5,999807.02,3092.9,54576.9 +Zaxxon-v5,725853.9,32.5,9173.3 diff --git a/tasks/simulation_heuristics_atari57_v1/task_context/evaluation.md b/tasks/simulation_heuristics_atari57_v1/task_context/evaluation.md new file mode 100644 index 0000000..476b661 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/task_context/evaluation.md @@ -0,0 +1,43 @@ +# Evaluation and aggregation + +For game `g`, raw episode return `R`, random anchor `r_g`, and human anchor +`h_g`: + +```text +HNS(g, R) = (R - r_g) / (h_g - r_g) +``` + +`atari57_games.csv` contains the 57 random/human anchors inferred by the source +artifact and the known-best score supplied to each article search. + +The primary aggregate is: + +1. evaluate repeat-specific policy `0`, `1`, and `2` on their corresponding + evaluation seeds; +2. compute mean HNS over those three policies for each game/mode; +3. select the higher mode mean for each game; +4. take the median over 57 games. + +This is the article's **best input mean** convention. The verifier also reports: + +- mean of the 57 selected mode means; +- median and mean after taking the best single evaluation run per game; +- per-game and per-mode returns/HNS; +- invalid-policy rate and counted evaluation steps; +- the separate 342-search interaction-ledger summary. + +The public evaluator uses six representative games and three public seeds, +covering 36 repeat-specific slots. It is for debugging only. The checked-in +hidden reproduction config uses all 57 games, both modes, and three seeds: 342 +final evaluation trajectories. + +The normalized GenesisBench score uses natural numeric HNS anchors: + +```text +0.0 HNS = 0 +0.8283015254994576 HNS = 100 +``` + +The second value is the article's reported Codex best-input-mean median HNS. +The seeded-random oracle artifact is not this target. Normalization does not +replace HNS; raw HNS metrics remain in the details JSON. diff --git a/tasks/simulation_heuristics_atari57_v1/task_context/policy_api.md b/tasks/simulation_heuristics_atari57_v1/task_context/policy_api.md new file mode 100644 index 0000000..0c5ff9c --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/task_context/policy_api.md @@ -0,0 +1,37 @@ +# Aggregate policy API + +The artifact manifest resolves one policy module for each of the 342 +`env_id / obs_mode / repeat_index` slots. Multiple slots may share a module, +but the evaluator instantiates the resolved module separately for every repeat. + +A module defines `Policy` or `make_policy`. The evaluator supplies any accepted +arguments from: + +```python +env_id: str +obs_mode: str +repeat_index: int +action_count: int +seed: int +config: dict +``` + +The resulting object implements: + +```python +class Policy: + def reset(self, seed: int = 0) -> None: + ... + + def act(self, observation, info=None) -> int: + ... +``` + +The action must be one integer in `[0, action_count)`. + +For `native_obs`, `info` is always `None`. For `ram`, `info` is a dictionary +containing only `ram` when EnvPool exposes it. Batch dimension one is removed +from both observations and RAM before the policy call. + +Policies may keep recurrent state and load files below `final_artifact/`. +They must run offline in the task image. diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/anchors.json b/tasks/simulation_heuristics_atari57_v1/verifier/anchors.json new file mode 100644 index 0000000..e9a4cbe --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/anchors.json @@ -0,0 +1,10 @@ +{ + "reference_policy": { + "description": "Article-reported Codex best-input-mean median HNS", + "score": 0.8283015254994576 + }, + "starter_policy": { + "description": "Natural random-baseline HNS origin", + "score": 0.0 + } +} diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/config.toml b/tasks/simulation_heuristics_atari57_v1/verifier/config.toml new file mode 100644 index 0000000..2a00fe9 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/config.toml @@ -0,0 +1,10 @@ +version = "1.0" + +[evaluation] +backend = "envpool" +games = "atari57" +obs_modes = "both" +seeds = [10001, 20001, 30001] +max_steps = 108000 +hns_table = "hns_normalization.csv" +require_complete_search_ledger = true diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/config_smoke.toml b/tasks/simulation_heuristics_atari57_v1/verifier/config_smoke.toml new file mode 100644 index 0000000..254b81a --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/config_smoke.toml @@ -0,0 +1,10 @@ +version = "1.0" + +[evaluation] +backend = "deterministic" +games = ["Breakout-v5", "Pong-v5", "Skiing-v5"] +obs_modes = ["ram", "native_obs"] +seeds = [101, 202, 303] +max_steps = 4 +hns_table = "hns_normalization.csv" +require_complete_search_ledger = false diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_atari57_v1/verifier/evaluate_hidden.py new file mode 100755 index 0000000..74390dd --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/evaluate_hidden.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +from dataclasses import asdict +import json +import sys +import tomllib +from pathlib import Path +from typing import Any + + +VERIFIER_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] +RUNTIME_CANDIDATES.extend(ancestor / "src" for ancestor in VERIFIER_DIR.parents) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.atari57 import ( # noqa: E402 + ATARI57_GAMES, + OBSERVATION_MODES, + Atari57Artifact, + Atari57ArtifactError, + deterministic_test_env_factory, + evaluate_atari57_artifact, + load_atari57_artifact, + load_hns_references, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run hidden aggregate Atari57 evaluation." + ) + parser.add_argument("artifact", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument( + "--config", + type=Path, + default=VERIFIER_DIR / "config.toml", + help="Evaluation suite config; production can inject a private file.", + ) + parser.add_argument( + "--anchors", + type=Path, + default=VERIFIER_DIR / "anchors.json", + help="Normalization anchors matching the selected suite.", + ) + return parser.parse_args() + + +def _sequence( + value: object, + *, + all_values: tuple[str, ...], + shorthand: str, + field: str, +) -> tuple[str, ...]: + if value == shorthand: + return all_values + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{field} must be {shorthand!r} or an array") + return tuple(value) + + +def _evaluate_raw( + artifact: Atari57Artifact, + *, + config_path: Path, + evaluation: dict[str, Any], +) -> Any: + backend = evaluation.get("backend", "envpool") + if backend == "envpool": + env_factory = None + elif backend == "deterministic": + env_factory = deterministic_test_env_factory + else: + raise ValueError(f"Unsupported evaluation backend: {backend!r}") + + games = _sequence( + evaluation.get("games", "atari57"), + all_values=ATARI57_GAMES, + shorthand="atari57", + field="evaluation.games", + ) + obs_modes = _sequence( + evaluation.get("obs_modes", "both"), + all_values=OBSERVATION_MODES, + shorthand="both", + field="evaluation.obs_modes", + ) + seeds = evaluation.get("seeds") + if not isinstance(seeds, list) or not all(isinstance(seed, int) for seed in seeds): + raise ValueError("evaluation.seeds must be an integer array") + if len(seeds) != 3: + raise ValueError( + "Hidden Atari57 evaluation requires exactly three repeat seeds" + ) + hns_path = config_path.parent / evaluation["hns_table"] + references = load_hns_references(hns_path) + result = evaluate_atari57_artifact( + artifact, + games=games, + obs_modes=obs_modes, + seeds=seeds, + max_steps=int(evaluation["max_steps"]), + hns_references=references, + env_factory=env_factory, + ) + return result + + +def _anchor_score(anchors: dict[str, Any], name: str) -> float: + anchor = anchors[name] + score = anchor.get("score") + if not isinstance(score, int | float): + raise ValueError(f"{name} must declare a numeric score") + return float(score) + + +def _emit(payload: dict[str, Any], output: Path | None) -> None: + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if output is not None: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered + "\n") + + +def _zero_payload( + *, + starter_score: float, + reference_score: float, + reason: str, + artifact: Atari57Artifact | None, +) -> dict[str, Any]: + return { + "score": 0.0, + "normalized_score": 0.0, + "starter_score": starter_score, + "reference_score": reference_score, + "evaluation": None, + "interaction_budget": ( + asdict(artifact.interaction_budget) if artifact is not None else None + ), + "protocol_complete": False, + "disqualification_reason": reason, + } + + +def _is_protocol_complete(artifact: Atari57Artifact) -> bool: + budget = artifact.interaction_budget + return budget.completed_trajectories == budget.planned_trajectories == 342 + + +def main() -> None: + args = parse_args() + config = tomllib.loads(args.config.read_text()) + anchors = json.loads(args.anchors.read_text()) + evaluation = config["evaluation"] + starter_score = _anchor_score(anchors, "starter_policy") + reference_score = _anchor_score(anchors, "reference_policy") + if reference_score == starter_score: + raise ValueError("starter and reference anchors must have different scores") + + try: + artifact = load_atari57_artifact(args.artifact) + except (Atari57ArtifactError, FileNotFoundError) as error: + _emit( + _zero_payload( + starter_score=starter_score, + reference_score=reference_score, + reason=f"Invalid aggregate artifact: {error}", + artifact=None, + ), + args.output, + ) + return + + protocol_complete = _is_protocol_complete(artifact) + if ( + evaluation.get("require_complete_search_ledger", False) + and not protocol_complete + ): + completed = artifact.interaction_budget.completed_trajectories + _emit( + _zero_payload( + starter_score=starter_score, + reference_score=reference_score, + reason=( + "Official suite requires 342 completed search-ledger " + f"records; found {completed}." + ), + artifact=artifact, + ), + args.output, + ) + return + + result = _evaluate_raw( + artifact, + config_path=args.config, + evaluation=evaluation, + ) + normalized_score = round( + 100.0 * (result.score - starter_score) / (reference_score - starter_score), + 9, + ) + payload = { + "score": result.score, + "normalized_score": normalized_score, + "starter_score": starter_score, + "reference_score": reference_score, + "evaluation": result.to_dict(), + "interaction_budget": asdict(artifact.interaction_budget), + "protocol_complete": protocol_complete, + "disqualification_reason": None, + } + _emit(payload, args.output) + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/hns_normalization.csv b/tasks/simulation_heuristics_atari57_v1/verifier/hns_normalization.csv new file mode 100644 index 0000000..a596309 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/hns_normalization.csv @@ -0,0 +1,58 @@ +env_id,known_best_score,random_score,human_score +Alien-v5,741812.63,227.8,7127.7 +Amidar-v5,31331.37,5.8,1719.5 +Assault-v5,143972.03,222.4,742.0 +Asterix-v5,999403.53,210.0,8503.3 +Asteroids-v5,6785558.64,719.1,47388.7 +Atlantis-v5,1674767.2,12850.0,29028.1 +BankHeist-v5,38536.66,14.2,753.1 +BattleZone-v5,992600.31,2360.0,37187.5 +BeamRider-v5,4549993.53,363.9,16926.5 +Berzerk-v5,85932.6,123.7,2630.4 +Bowling-v5,260.13,23.1,160.7 +Boxing-v5,100.0,0.1,12.1 +Breakout-v5,864.0,1.7,30.5 +Centipede-v5,1159049.27,2090.9,12017.0 +ChopperCommand-v5,999900.0,811.0,7387.8 +CrazyClimber-v5,729482.83,10780.5,35829.4 +Defender-v5,839642.95,2874.5,18688.9 +DemonAttack-v5,143964.26,152.1,1971.0 +DoubleDunk-v5,24.0,-18.6,-16.4 +Enduro-v5,2382.44,0.0,860.5 +FishingDerby-v5,91.16,-91.7,-38.7 +Freeway-v5,34.0,0.0,29.6 +Frostbite-v5,631378.53,65.2,4334.7 +Gopher-v5,130345.58,257.6,2412.5 +Gravitar-v5,21068.03,173.0,3351.4 +Hero-v5,114736.26,1027.0,30826.4 +IceHockey-v5,86.89,-11.2,0.9 +Jamesbond-v5,158142.36,29.0,302.8 +Kangaroo-v5,37392.82,52.0,3035.0 +Krull-v5,291043.06,1598.0,2665.5 +KungFuMaster-v5,267766.63,258.5,22736.3 +MontezumaRevenge-v5,19093.74,0.0,4753.3 +MsPacman-v5,243401.1,307.3,6951.6 +NameThisGame-v5,157177.85,2292.3,8049.0 +Phoenix-v5,990638.12,761.4,7242.6 +Pitfall-v5,18756.01,-229.4,6463.7 +Pong-v5,21.0,-20.7,14.6 +PrivateEye-v5,100314.44,24.9,69571.3 +Qbert-v5,777071.3,163.9,13455.0 +Riverraid-v5,323417.18,1338.5,17118.0 +RoadRunner-v5,613659.42,11.5,7845.0 +Robotank-v5,144.0,2.2,11.9 +Seaquest-v5,999999.0,68.4,42054.7 +Skiing-v5,-3851.44,-17098.1,-4336.9 +Solaris-v5,67306.29,1236.3,12326.7 +SpaceInvaders-v5,74335.3,148.0,1668.7 +StarGunner-v5,998600.28,664.0,10250.0 +Surround-v5,10.0,-10.0,6.5 +Tennis-v5,24.0,-23.8,-8.3 +TimePilot-v5,476763.9,3568.0,5229.2 +Tutankham-v5,2354.91,11.4,167.6 +UpNDown-v5,715545.61,533.4,11693.2 +Venture-v5,2623.71,0.0,1187.5 +VideoPinball-v5,999697.05,0.0,17667.9 +WizardOfWor-v5,197126.0,563.5,4756.5 +YarsRevenge-v5,999807.02,3092.9,54576.9 +Zaxxon-v5,725853.9,32.5,9173.3 diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_atari57_v1/verifier/rubrics/verifier.md new file mode 100644 index 0000000..7364fa3 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/rubrics/verifier.md @@ -0,0 +1,26 @@ +# Simulation Heuristics Atari57 v1 verifier rubric + +## `task_success` + +The scored quantity is the normalized hidden-suite primary aggregate: + +1. mean HNS over three evaluation repeats for each game/mode; +2. better mode mean for each game; +3. median over all 57 games; +4. numeric normalization where HNS `0.0` maps to `0` and article-reference + HNS `0.8283015254994576` maps to `1`. + +The canonical reward is clamped to `[0, 1]`; the details JSON may contain +negative normalized scores or values above `100`. + +## Publication eligibility + +Artifact validity is mandatory. Interaction accounting is reported separately. +The checked-in full suite sets `require_complete_search_ledger=true`, making +342 completed records with at least 20M counted frames and the required +evidence files an eligibility gate. Incomplete artifacts return reward and +normalized score zero without starting EnvPool. The deterministic smoke config +disables this gate only for contract testing. + +The looser best-single-run HNS is diagnostic only and never replaces the +primary best-input-mean metric. diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/rubrics/verifier.toml b/tasks/simulation_heuristics_atari57_v1/verifier/rubrics/verifier.toml new file mode 100644 index 0000000..cd17b86 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/rubrics/verifier.toml @@ -0,0 +1,9 @@ +version = "0.1" + +[[criteria]] +id = "task_success" +description = "Normalized hidden Atari57 best-input-mean median HNS." +weight = 1.0 + +[scoring] +method = "weighted_mean" diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/test.sh b/tasks/simulation_heuristics_atari57_v1/verifier/test.sh new file mode 100755 index 0000000..6c820b2 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/test.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier +artifact=/app/final_artifact + +python /verifier/evaluate_hidden.py \ + "$artifact" \ + --config /verifier/config.toml \ + --anchors /verifier/anchors.json \ + --output /logs/verifier/genesis-score.json + +python - <<'PY' +import json +from pathlib import Path + +score = json.loads(Path("/logs/verifier/genesis-score.json").read_text()) +reward = max(0.0, min(1.0, float(score["normalized_score"]) / 100.0)) +Path("/logs/verifier/reward.txt").write_text(f"{reward}\n") +Path("/logs/verifier/reward.json").write_text( + json.dumps({"reward": reward}, indent=2) + "\n" +) +PY diff --git a/tasks/simulation_heuristics_atari57_v1/verifier/verifier.md b/tasks/simulation_heuristics_atari57_v1/verifier/verifier.md new file mode 100644 index 0000000..b415fa3 --- /dev/null +++ b/tasks/simulation_heuristics_atari57_v1/verifier/verifier.md @@ -0,0 +1,35 @@ +--- +document_version: "0.3" +verifier: + name: simulation_heuristics_atari57_v1 + default_strategy: deterministic + strategies: + deterministic: + type: script + command: ./test.sh + rubric: + combine: weighted_mean + dimensions: + task_success: + weight: 1.0 + source: deterministic + outputs: + reward_text: /logs/verifier/reward.txt + reward_json: /logs/verifier/reward.json + details_json: /logs/verifier/genesis-score.json + aggregate_policy: + method: weighted_mean + metrics: + task_success: 1.0 +--- + +## Verifier intent + +Resolve the aggregate artifact into 342 repeat-specific policy slots. If all +342 search records are complete, run each policy on its corresponding hidden +seed, compute article-style HNS aggregation, and normalize the strict median +against numeric HNS anchors. Incomplete artifacts receive zero before EnvPool +starts. + +The full details file retains raw HNS, best-single-run diagnostics, per-game +metrics, evaluation-step counts, and the 342-search interaction ledger. diff --git a/tasks/simulation_heuristics_breakout_ram_v1/README.md b/tasks/simulation_heuristics_breakout_ram_v1/README.md new file mode 100644 index 0000000..e368a3a --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/README.md @@ -0,0 +1,73 @@ +# Simulation Heuristics Breakout RAM v1 + +This native GenesisBench task packages the article's RAM-observation Breakout +experiment as an independently evaluated policy-improvement benchmark. + +## Contract + +The agent receives `task.md`, `starter_policy/`, `evaluate.py`, and +`task_context/`. It must leave: + +```text +final_policy/ + policy.py +``` + +The policy receives 128 RAM bytes only. The hidden verifier is not copied into +the agent workspace. + +## Score + +```text +raw score = + 0.50 * hidden nominal mean + + 0.50 * hidden shifted-start mean +``` + +Normalization maps the checked-in starter to `0` and the frozen 864-point +reference to `100`. Both trusted anchors are evaluated on the same runtime as +the submission. Scores above `100` remain representable. + +## Local verification + +Use a Linux environment with `envpool==1.1.1`, then run: + +```bash +python tasks/simulation_heuristics_breakout_ram_v1/evaluate.py \ + --policy tasks/simulation_heuristics_breakout_ram_v1/starter_policy/policy.py + +python tasks/simulation_heuristics_breakout_ram_v1/verifier/evaluate_hidden.py \ + tasks/simulation_heuristics_breakout_ram_v1/oracle/policy.py +``` + +Validate the native package: + +```bash +uv run bench tasks check \ + tasks/simulation_heuristics_breakout_ram_v1 \ + --level publication-grade +``` + +Run the trusted oracle through BenchFlow: + +```bash +uv run bench eval run \ + --tasks-dir tasks/simulation_heuristics_breakout_ram_v1 \ + --agent oracle \ + --sandbox docker \ + --context-root . +``` + +## Public and private suites + +The checked-in verifier config is a reproducibility suite. A hosted leaderboard +should inject a private config and matching private anchors: + +```bash +python verifier/evaluate_hidden.py final_policy/policy.py \ + --config /private/final-suite.toml \ + --anchors /private/final-anchors.json +``` + +See `task_context/article_progression.md` for the `387 -> 507 -> 839 -> 864` +mechanisms and `task_context/provenance.md` for revision/licensing notes. diff --git a/tasks/simulation_heuristics_breakout_ram_v1/environment/Dockerfile b/tasks/simulation_heuristics_breakout_ram_v1/environment/Dockerfile new file mode 100644 index 0000000..d463298 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/environment/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12-slim + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends bash ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir \ + "envpool==1.1.1" \ + "numpy>=1.26,<3" \ + "pyyaml>=6,<7" + +RUN useradd --create-home --uid 1000 agent \ + && mkdir -p /app /opt/genesisbench + +COPY src/genesisbench /opt/genesisbench/genesisbench +COPY tasks/simulation_heuristics_breakout_ram_v1/evaluate.py /app/evaluate.py +COPY tasks/simulation_heuristics_breakout_ram_v1/starter_policy /app/starter_policy +COPY tasks/simulation_heuristics_breakout_ram_v1/task_context /app/task_context + +RUN cp -R /app/starter_policy /app/final_policy \ + && chown -R root:root /opt/genesisbench \ + && chmod -R a-w /opt/genesisbench \ + && chown -R agent:agent /app + +ENV PYTHONPATH="/opt/genesisbench" + +WORKDIR /app + +RUN mkdir -p /logs/verifier /logs/agent /logs/artifacts diff --git a/tasks/simulation_heuristics_breakout_ram_v1/evaluate.py b/tasks/simulation_heuristics_breakout_ram_v1/evaluate.py new file mode 100644 index 0000000..bee3441 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/evaluate.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +TASK_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = ( + TASK_DIR / "_runtime", + TASK_DIR.parent / "src", + TASK_DIR.parent.parent / "src", +) +for candidate in RUNTIME_CANDIDATES: + if candidate.is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.breakout import evaluate_breakout_policy # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate a RAM-only Breakout policy." + ) + parser.add_argument( + "--policy", + type=Path, + default=Path("final_policy/policy.py"), + ) + parser.add_argument("--episodes", type=int, default=1) + parser.add_argument("--max-steps", type=int, default=27000) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--json-output-file", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + result = evaluate_breakout_policy( + args.policy, + observation_mode="ram", + seeds=range(args.seed, args.seed + args.episodes), + max_steps=args.max_steps, + ) + rendered = result.to_json() + print(rendered) + if args.json_output_file is not None: + args.json_output_file.parent.mkdir(parents=True, exist_ok=True) + args.json_output_file.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_breakout_ram_v1/oracle/policy.py b/tasks/simulation_heuristics_breakout_ram_v1/oracle/policy.py new file mode 100644 index 0000000..f8b88a0 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/oracle/policy.py @@ -0,0 +1,257 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen observation-only RAM reference for Breakout.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +FIRE = 1 +RIGHT = 2 +LEFT = 3 + +FIELD_LEFT = 8.0 +FIELD_RIGHT = 151.0 +PADDLE_MIN_X = 15.5 +PADDLE_MAX_X = 152.5 +PADDLE_Y = 189.5 +HOME_X = 106.5 +DEADBAND = 3.0 +CHASE_LEAD_STEPS = 6.0 +LAUNCH_OFFSET = 24.0 +FAST_LOW_BALL_LEAD_STEPS = 3.0 +FAST_BALL_MIN_VY = 3.0 +MAX_VELOCITY_JUMP = 24.0 +STUCK_TRIGGER_STEPS = 1024 +STUCK_SWITCH_STEPS = 256 +STUCK_OFFSET = 12.0 +STUCK_RELEASE_HORIZON = 8.0 +BRICK_BALANCE_DEADZONE = 0.01 +LATE_GAME_PADDLE_LAG = 2.0 +LATE_GAME_LAG_BALL_Y = 170.0 + + +def _clip(value: float, lower: float, upper: float) -> float: + return min(max(value, lower), upper) + + +def _reflect(value: float, lower: float, upper: float) -> float: + span = upper - lower + shifted = (value - lower) % (2.0 * span) + if shifted <= span: + return lower + shifted + return upper - (shifted - span) + + +def _bit_count(values: np.ndarray) -> int: + return int(sum(int(value).bit_count() for value in values.tolist())) + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.prev_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + self.last_action = NOOP + self.launch_sign = 1.0 + self.had_visible_ball = False + self.ball_missing = True + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + self.previous_score: int | None = None + self.score = 0 + + def act(self, observation: np.ndarray) -> int: + ram = np.asarray(observation, dtype=np.uint8) + if ram.shape != (128,): + raise ValueError(f"Expected RAM shape (128,), got {ram.shape}") + + brick_bytes = ram[:36] + self.score = self._decode_score(ram) + self._update_progress(self.score) + + paddle_x = 1.005232 * float(ram[72]) - 39.797062 + ball = None + if int(ram[101]) != 0: + ball = ( + 0.999043 * float(ram[99]) - 48.370898, + 0.993263 * float(ram[101]) + 11.227841, + ) + self._update_ball(ball) + + if ball is None: + action = self._serve(paddle_x) + else: + target_x = self._target_x(ball, self._brick_balance(brick_bytes)) + control_x = self._control_paddle_x(paddle_x, ball) + error = target_x - control_x + if error > DEADBAND: + action = RIGHT + elif error < -DEADBAND: + action = LEFT + else: + action = NOOP + + self.last_action = action + return action + + def _update_progress(self, score: int) -> None: + if self.previous_score is not None and score != self.previous_score: + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + elif self.previous_score is not None: + self.steps_since_progress += 1 + if ( + self.steps_since_progress >= STUCK_TRIGGER_STEPS + and ( + self.steps_since_progress - STUCK_TRIGGER_STEPS + ) + % STUCK_SWITCH_STEPS + == 0 + ): + self.stuck_offset_index += 1 + + self.previous_score = score + + def _update_ball(self, ball: tuple[float, float] | None) -> None: + if ball is None: + if self.had_visible_ball and not self.ball_missing: + self.launch_sign *= -1.0 + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + self.ball_missing = True + self.prev_ball = None + self.velocity = None + return + + self.had_visible_ball = True + self.ball_missing = False + if self.prev_ball is not None: + dx = ball[0] - self.prev_ball[0] + dy = ball[1] - self.prev_ball[1] + if ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ): + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * self.velocity[0] + 0.5 * dx, + 0.5 * self.velocity[1] + 0.5 * dy, + ) + else: + self.velocity = None + self.prev_ball = ball + + def _serve(self, paddle_x: float) -> int: + target_x = HOME_X + self.launch_sign * LAUNCH_OFFSET + error = target_x - paddle_x + if error > DEADBAND: + return RIGHT + if error < -DEADBAND: + return LEFT + return FIRE + + def _target_x(self, ball: tuple[float, float], balance: float) -> float: + ball_x, ball_y = ball + if self.velocity is None: + return _clip(ball_x, PADDLE_MIN_X, PADDLE_MAX_X) + + vx, vy = self.velocity + if vy > 0.1 and ball_y <= PADDLE_Y: + steps_to_paddle = max((PADDLE_Y - ball_y) / vy, 0.0) + intercept = _reflect( + ball_x + vx * steps_to_paddle, + FIELD_LEFT, + FIELD_RIGHT, + ) + target_x = intercept + self._stuck_offset( + steps_to_paddle, + balance, + ) + elif vy >= FAST_BALL_MIN_VY: + target_x = ball_x + FAST_LOW_BALL_LEAD_STEPS * vx + else: + target_x = ball_x + CHASE_LEAD_STEPS * vx + return _clip(target_x, PADDLE_MIN_X, PADDLE_MAX_X) + + def _control_paddle_x( + self, + paddle_x: float, + ball: tuple[float, float], + ) -> float: + if ( + self.score < 432 + or self.velocity is None + or self.velocity[1] <= 0.1 + or ball[1] < LATE_GAME_LAG_BALL_Y + ): + return paddle_x + if self.last_action == RIGHT: + return _clip( + paddle_x + LATE_GAME_PADDLE_LAG, + PADDLE_MIN_X, + PADDLE_MAX_X, + ) + if self.last_action == LEFT: + return _clip( + paddle_x - LATE_GAME_PADDLE_LAG, + PADDLE_MIN_X, + PADDLE_MAX_X, + ) + return paddle_x + + def _stuck_offset(self, steps_to_paddle: float, balance: float) -> float: + if self.steps_since_progress < STUCK_TRIGGER_STEPS: + return 0.0 + + phase = self.stuck_offset_index % 4 + direction = 1.0 if phase in (0, 2) else -1.0 + magnitude = STUCK_OFFSET if phase in (0, 1) else 0.5 * STUCK_OFFSET + if self.score >= 432: + if balance > BRICK_BALANCE_DEADZONE: + direction = 1.0 + elif balance < -BRICK_BALANCE_DEADZONE: + direction = -1.0 + release = _clip( + steps_to_paddle / STUCK_RELEASE_HORIZON, + 0.0, + 1.0, + ) + magnitude *= release + return direction * magnitude + + @staticmethod + def _brick_balance(brick_bytes: np.ndarray) -> float: + return ( + _bit_count(brick_bytes[:18]) / 132.0 + - _bit_count(brick_bytes[18:36]) / 108.0 + ) + + @staticmethod + def _decode_score(ram: np.ndarray) -> int: + low_bcd = int(ram[77]) + return ( + 100 * int(ram[76]) + + 10 * (low_bcd >> 4) + + (low_bcd & 0x0F) + ) diff --git a/tasks/simulation_heuristics_breakout_ram_v1/oracle/solve.sh b/tasks/simulation_heuristics_breakout_ram_v1/oracle/solve.sh new file mode 100755 index 0000000..1af9fd8 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/oracle/solve.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /app/final_policy +cp /oracle/policy.py /app/final_policy/policy.py diff --git a/tasks/simulation_heuristics_breakout_ram_v1/starter_policy/policy.py b/tasks/simulation_heuristics_breakout_ram_v1/starter_policy/policy.py new file mode 100644 index 0000000..44821b6 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/starter_policy/policy.py @@ -0,0 +1,134 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Article-faithful 387-point RAM starter for Breakout.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +FIRE = 1 +RIGHT = 2 +LEFT = 3 + +FIELD_LEFT = 8.0 +FIELD_RIGHT = 151.0 +PADDLE_MIN_X = 15.5 +PADDLE_MAX_X = 152.5 +PADDLE_Y = 189.5 +HOME_X = 106.5 +DEADBAND = 3.0 +CHASE_LEAD_STEPS = 6.0 +LAUNCH_OFFSET = 24.0 +MAX_VELOCITY_JUMP = 24.0 + + +def _clip(value: float, lower: float, upper: float) -> float: + return min(max(value, lower), upper) + + +def _reflect(value: float, lower: float, upper: float) -> float: + span = upper - lower + shifted = (value - lower) % (2.0 * span) + if shifted <= span: + return lower + shifted + return upper - (shifted - span) + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.prev_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + self.launch_sign = 1.0 + self.had_visible_ball = False + self.ball_missing = True + + def act(self, observation: np.ndarray) -> int: + ram = np.asarray(observation, dtype=np.uint8) + if ram.shape != (128,): + raise ValueError(f"Expected RAM shape (128,), got {ram.shape}") + + paddle_x = 1.005232 * float(ram[72]) - 39.797062 + ball = None + if int(ram[101]) != 0: + ball = ( + 0.999043 * float(ram[99]) - 48.370898, + 0.993263 * float(ram[101]) + 11.227841, + ) + self._update_ball(ball) + + if ball is None: + target_x = HOME_X + self.launch_sign * LAUNCH_OFFSET + else: + target_x = self._target_x(ball) + + error = target_x - paddle_x + if error > DEADBAND: + return RIGHT + if error < -DEADBAND: + return LEFT + return FIRE if ball is None else NOOP + + def _update_ball(self, ball: tuple[float, float] | None) -> None: + if ball is None: + if self.had_visible_ball and not self.ball_missing: + self.launch_sign *= -1.0 + self.ball_missing = True + self.prev_ball = None + self.velocity = None + return + + self.had_visible_ball = True + self.ball_missing = False + if self.prev_ball is not None: + dx = ball[0] - self.prev_ball[0] + dy = ball[1] - self.prev_ball[1] + if ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ): + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * self.velocity[0] + 0.5 * dx, + 0.5 * self.velocity[1] + 0.5 * dy, + ) + else: + self.velocity = None + self.prev_ball = ball + + def _target_x(self, ball: tuple[float, float]) -> float: + ball_x, ball_y = ball + if self.velocity is None: + return _clip(ball_x, PADDLE_MIN_X, PADDLE_MAX_X) + + vx, vy = self.velocity + if vy > 0.1 and ball_y <= PADDLE_Y: + steps_to_paddle = max((PADDLE_Y - ball_y) / vy, 0.0) + target_x = _reflect( + ball_x + vx * steps_to_paddle, + FIELD_LEFT, + FIELD_RIGHT, + ) + else: + target_x = ball_x + CHASE_LEAD_STEPS * vx + return _clip(target_x, PADDLE_MIN_X, PADDLE_MAX_X) diff --git a/tasks/simulation_heuristics_breakout_ram_v1/task.md b/tasks/simulation_heuristics_breakout_ram_v1/task.md new file mode 100644 index 0000000..fbb6c13 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/task.md @@ -0,0 +1,141 @@ +--- +schema_version: "1.3" +task: + name: genesisbench/simulation_heuristics_breakout_ram_v1 + description: Improve a RAM-only programmatic Atari Breakout policy toward the 864-point article result. + authors: + - name: GenesisBench Contributors + keywords: + - simulation + - heuristics + - breakout + - atari + - ram +metadata: + category: game-control + difficulty: medium + tags: + - atari + - discrete-control + - policy-search + - ram-observation + reference_task: false + genesisbench: + starter: + path: starter_policy + submission: + directory: final_policy + entrypoint: policy.py + development: + episodes: 1 + max_steps: 27000 + seeds: [0] + observation_mode: ram + verifier: + reproduction_config: verifier/config.toml + anchors: verifier/anchors.json + supports_private_config: true +agent: + timeout_sec: 1800 + user: agent + network_mode: public +verifier: + timeout_sec: 300 + user: root + network_mode: no-network +environment: + build_timeout_sec: 1200 + cpus: 4 + memory_mb: 4096 + storage_mb: 10240 + network_mode: public + allow_internet: true + workdir: /app +benchflow: + document_version: "0.6" +--- +# Improve the RAM-only Breakout policy + +You are given the article's 387-point programmatic controller for EnvPool +`Breakout-v5`. Improve it through autonomous research, coding, and +experimentation. + +## Objective + +Maximize the native Breakout game score. The article experiment progressed +through `387 -> 507 -> 839 -> 864`; `864` is the target reproduced by the +packaged reference policy. + +## Observation and action contract + +The policy receives only a NumPy-compatible `uint8` RAM vector with shape +`(128,)`. It does not receive pixels, reward, `info`, lives, the ALE object, or +hidden evaluator configuration. + +Return one integer action: + +```text +0 = NOOP +1 = FIRE +2 = RIGHT +3 = LEFT +``` + +The evaluator runs one ALE frame per policy action with no frame skipping. + +## Starting point + +`starter_policy/policy.py` reproduces the article's `387` node: it predicts +reflected paddle interceptions but has no stuck-loop perturbation, fast-low-ball +special case, or late-game offset release. + +## Development evaluation + +Run: + +```bash +python evaluate.py --policy starter_policy/policy.py +``` + +Run the full article-scale horizon with: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --episodes 1 \ + --max-steps 108000 +``` + +The public evaluator uses a nominal visible setup. The final evaluator uses +unseen seeds and shifted initial paddle configurations. A step-indexed replay +of one public trajectory is not a robust solution. + +## Required submission + +Store your best policy at: + +```text +final_policy/policy.py +``` + +It must define either: + +```python +class Policy: + def __init__(self, seed: int = 0): ... + def reset(self, seed: int = 0) -> None: ... + def act(self, observation) -> int: ... +``` + +or `make_policy(seed=0)` returning an object with `reset` and `act`. + +## Rules + +1. Work autonomously. Do not ask for user feedback. +2. Do not modify `evaluate.py` or trusted runtime files. +3. Do not access `/oracle`, `/verifier`, or hidden configuration. +4. Do not copy a completed Breakout solution from the internet. +5. The final policy receives the 128 RAM bytes only. +6. The final policy must run offline with the installed packages. +7. Before finishing, evaluate `final_policy/policy.py` and leave the best + working version in place. diff --git a/tasks/simulation_heuristics_breakout_ram_v1/task_context/article_progression.md b/tasks/simulation_heuristics_breakout_ram_v1/task_context/article_progression.md new file mode 100644 index 0000000..cc0c84c --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/task_context/article_progression.md @@ -0,0 +1,15 @@ +# Article progression + +The packaged policies preserve the RAM experiment's reported control path: + +| Node | Score | Mechanism | +| --- | ---: | --- | +| Starter | 387 | Reflected interception with no trajectory perturbation | +| Intermediate | 507 | Resettable stuck-loop offset cycling | +| Intermediate | 839 | Fast-low-ball lead of three frames | +| Reference | 864 | Post-432 offset release plus two-pixel paddle-lag compensation | + +The checked-in starter is the `387` node and the trusted reference is the +`864` node. Both are observation-only adaptations: reward progress and lives +needed by the controller are inferred from the 128 RAM bytes rather than +passed as side channels. diff --git a/tasks/simulation_heuristics_breakout_ram_v1/task_context/evaluation.md b/tasks/simulation_heuristics_breakout_ram_v1/task_context/evaluation.md new file mode 100644 index 0000000..9bd4bd7 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/task_context/evaluation.md @@ -0,0 +1,24 @@ +# Evaluation + +The environment is EnvPool `1.1.1` `Breakout-v5` with the article's settings: + +- RAM observations; +- the reduced four-action set; +- `frameskip=1`; +- `noop_max=1`; +- `use_fire_reset=True`; +- `episodic_life=False`; +- `repeat_action_probability=0`; +- `reward_clip=False`; +- a full final horizon of 108,000 policy actions. + +The public evaluator reports mean/minimum/maximum return, 864-point completion +rate, invalid-policy rate, action latency, and per-episode details. + +The checked-in final suite combines a nominal start with shifted initial paddle +positions. Those zero-reward prefixes change the trajectory before policy +control begins and make a memorized action tape brittle while preserving the +same observation and reward contract. + +Official hosted evaluation may inject a private config with different seeds +and prefixes. Invalid actions receive the configured failure return. diff --git a/tasks/simulation_heuristics_breakout_ram_v1/task_context/policy_api.md b/tasks/simulation_heuristics_breakout_ram_v1/task_context/policy_api.md new file mode 100644 index 0000000..7431fb4 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/task_context/policy_api.md @@ -0,0 +1,28 @@ +# Policy API + +The evaluator imports `final_policy/policy.py` in a clean Python process. + +Define `Policy` or `make_policy`. The policy is instantiated once per episode +and reset before the first scored action: + +```python +class Policy: + def __init__(self, seed: int = 0): + ... + + def reset(self, seed: int = 0) -> None: + ... + + def act(self, observation): + ... +``` + +`observation` is exactly one EnvPool `Breakout-v5` 128-byte `uint8` RAM vector, +extracted from the environment's per-step `info["ram"]` batch. + +The only valid outputs are scalar integer actions `0`, `1`, `2`, and `3`, +corresponding to `NOOP`, `FIRE`, `RIGHT`, and `LEFT`. The policy may keep +recurrent state and load files stored below `final_policy/`. + +No reward, pixels, `info`, lives counter, emulator object, or hidden variant +metadata is passed to the policy. diff --git a/tasks/simulation_heuristics_breakout_ram_v1/task_context/provenance.md b/tasks/simulation_heuristics_breakout_ram_v1/task_context/provenance.md new file mode 100644 index 0000000..ca990bf --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/task_context/provenance.md @@ -0,0 +1,18 @@ +# Provenance and licensing + +This task reimplements the RAM Breakout experiment described in **Learning +Beyond Gradients**. + +- Upstream project: `Trinkle23897/learning-beyond-gradients` +- Reviewed revision: `3555c2956c257d49a5015b782cbe485b14fd659e` +- Reviewed artifact: `atari/breakout/heuristic_breakout.py` +- Reported progression: `387 -> 507 -> 839 -> 864` + +The starter and reference policies adapt the published geometric controller +structure and retain its `Copyright 2021 Garena Online Private Limited` and +Apache License 2.0 notice. GenesisBench includes the Apache 2.0 text at +`LICENSES/Apache-2.0.txt`. + +No Atari ROM file is copied into this repository. The task image pins +`envpool==1.1.1`, the runtime used by the article, and relies on the Atari +assets distributed with that EnvPool wheel. diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchor_policies/reference_policy.py b/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchor_policies/reference_policy.py new file mode 100644 index 0000000..f8b88a0 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchor_policies/reference_policy.py @@ -0,0 +1,257 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen observation-only RAM reference for Breakout.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +FIRE = 1 +RIGHT = 2 +LEFT = 3 + +FIELD_LEFT = 8.0 +FIELD_RIGHT = 151.0 +PADDLE_MIN_X = 15.5 +PADDLE_MAX_X = 152.5 +PADDLE_Y = 189.5 +HOME_X = 106.5 +DEADBAND = 3.0 +CHASE_LEAD_STEPS = 6.0 +LAUNCH_OFFSET = 24.0 +FAST_LOW_BALL_LEAD_STEPS = 3.0 +FAST_BALL_MIN_VY = 3.0 +MAX_VELOCITY_JUMP = 24.0 +STUCK_TRIGGER_STEPS = 1024 +STUCK_SWITCH_STEPS = 256 +STUCK_OFFSET = 12.0 +STUCK_RELEASE_HORIZON = 8.0 +BRICK_BALANCE_DEADZONE = 0.01 +LATE_GAME_PADDLE_LAG = 2.0 +LATE_GAME_LAG_BALL_Y = 170.0 + + +def _clip(value: float, lower: float, upper: float) -> float: + return min(max(value, lower), upper) + + +def _reflect(value: float, lower: float, upper: float) -> float: + span = upper - lower + shifted = (value - lower) % (2.0 * span) + if shifted <= span: + return lower + shifted + return upper - (shifted - span) + + +def _bit_count(values: np.ndarray) -> int: + return int(sum(int(value).bit_count() for value in values.tolist())) + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.prev_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + self.last_action = NOOP + self.launch_sign = 1.0 + self.had_visible_ball = False + self.ball_missing = True + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + self.previous_score: int | None = None + self.score = 0 + + def act(self, observation: np.ndarray) -> int: + ram = np.asarray(observation, dtype=np.uint8) + if ram.shape != (128,): + raise ValueError(f"Expected RAM shape (128,), got {ram.shape}") + + brick_bytes = ram[:36] + self.score = self._decode_score(ram) + self._update_progress(self.score) + + paddle_x = 1.005232 * float(ram[72]) - 39.797062 + ball = None + if int(ram[101]) != 0: + ball = ( + 0.999043 * float(ram[99]) - 48.370898, + 0.993263 * float(ram[101]) + 11.227841, + ) + self._update_ball(ball) + + if ball is None: + action = self._serve(paddle_x) + else: + target_x = self._target_x(ball, self._brick_balance(brick_bytes)) + control_x = self._control_paddle_x(paddle_x, ball) + error = target_x - control_x + if error > DEADBAND: + action = RIGHT + elif error < -DEADBAND: + action = LEFT + else: + action = NOOP + + self.last_action = action + return action + + def _update_progress(self, score: int) -> None: + if self.previous_score is not None and score != self.previous_score: + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + elif self.previous_score is not None: + self.steps_since_progress += 1 + if ( + self.steps_since_progress >= STUCK_TRIGGER_STEPS + and ( + self.steps_since_progress - STUCK_TRIGGER_STEPS + ) + % STUCK_SWITCH_STEPS + == 0 + ): + self.stuck_offset_index += 1 + + self.previous_score = score + + def _update_ball(self, ball: tuple[float, float] | None) -> None: + if ball is None: + if self.had_visible_ball and not self.ball_missing: + self.launch_sign *= -1.0 + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + self.ball_missing = True + self.prev_ball = None + self.velocity = None + return + + self.had_visible_ball = True + self.ball_missing = False + if self.prev_ball is not None: + dx = ball[0] - self.prev_ball[0] + dy = ball[1] - self.prev_ball[1] + if ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ): + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * self.velocity[0] + 0.5 * dx, + 0.5 * self.velocity[1] + 0.5 * dy, + ) + else: + self.velocity = None + self.prev_ball = ball + + def _serve(self, paddle_x: float) -> int: + target_x = HOME_X + self.launch_sign * LAUNCH_OFFSET + error = target_x - paddle_x + if error > DEADBAND: + return RIGHT + if error < -DEADBAND: + return LEFT + return FIRE + + def _target_x(self, ball: tuple[float, float], balance: float) -> float: + ball_x, ball_y = ball + if self.velocity is None: + return _clip(ball_x, PADDLE_MIN_X, PADDLE_MAX_X) + + vx, vy = self.velocity + if vy > 0.1 and ball_y <= PADDLE_Y: + steps_to_paddle = max((PADDLE_Y - ball_y) / vy, 0.0) + intercept = _reflect( + ball_x + vx * steps_to_paddle, + FIELD_LEFT, + FIELD_RIGHT, + ) + target_x = intercept + self._stuck_offset( + steps_to_paddle, + balance, + ) + elif vy >= FAST_BALL_MIN_VY: + target_x = ball_x + FAST_LOW_BALL_LEAD_STEPS * vx + else: + target_x = ball_x + CHASE_LEAD_STEPS * vx + return _clip(target_x, PADDLE_MIN_X, PADDLE_MAX_X) + + def _control_paddle_x( + self, + paddle_x: float, + ball: tuple[float, float], + ) -> float: + if ( + self.score < 432 + or self.velocity is None + or self.velocity[1] <= 0.1 + or ball[1] < LATE_GAME_LAG_BALL_Y + ): + return paddle_x + if self.last_action == RIGHT: + return _clip( + paddle_x + LATE_GAME_PADDLE_LAG, + PADDLE_MIN_X, + PADDLE_MAX_X, + ) + if self.last_action == LEFT: + return _clip( + paddle_x - LATE_GAME_PADDLE_LAG, + PADDLE_MIN_X, + PADDLE_MAX_X, + ) + return paddle_x + + def _stuck_offset(self, steps_to_paddle: float, balance: float) -> float: + if self.steps_since_progress < STUCK_TRIGGER_STEPS: + return 0.0 + + phase = self.stuck_offset_index % 4 + direction = 1.0 if phase in (0, 2) else -1.0 + magnitude = STUCK_OFFSET if phase in (0, 1) else 0.5 * STUCK_OFFSET + if self.score >= 432: + if balance > BRICK_BALANCE_DEADZONE: + direction = 1.0 + elif balance < -BRICK_BALANCE_DEADZONE: + direction = -1.0 + release = _clip( + steps_to_paddle / STUCK_RELEASE_HORIZON, + 0.0, + 1.0, + ) + magnitude *= release + return direction * magnitude + + @staticmethod + def _brick_balance(brick_bytes: np.ndarray) -> float: + return ( + _bit_count(brick_bytes[:18]) / 132.0 + - _bit_count(brick_bytes[18:36]) / 108.0 + ) + + @staticmethod + def _decode_score(ram: np.ndarray) -> int: + low_bcd = int(ram[77]) + return ( + 100 * int(ram[76]) + + 10 * (low_bcd >> 4) + + (low_bcd & 0x0F) + ) diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchor_policies/starter_policy.py b/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchor_policies/starter_policy.py new file mode 100644 index 0000000..44821b6 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchor_policies/starter_policy.py @@ -0,0 +1,134 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Article-faithful 387-point RAM starter for Breakout.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +FIRE = 1 +RIGHT = 2 +LEFT = 3 + +FIELD_LEFT = 8.0 +FIELD_RIGHT = 151.0 +PADDLE_MIN_X = 15.5 +PADDLE_MAX_X = 152.5 +PADDLE_Y = 189.5 +HOME_X = 106.5 +DEADBAND = 3.0 +CHASE_LEAD_STEPS = 6.0 +LAUNCH_OFFSET = 24.0 +MAX_VELOCITY_JUMP = 24.0 + + +def _clip(value: float, lower: float, upper: float) -> float: + return min(max(value, lower), upper) + + +def _reflect(value: float, lower: float, upper: float) -> float: + span = upper - lower + shifted = (value - lower) % (2.0 * span) + if shifted <= span: + return lower + shifted + return upper - (shifted - span) + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.prev_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + self.launch_sign = 1.0 + self.had_visible_ball = False + self.ball_missing = True + + def act(self, observation: np.ndarray) -> int: + ram = np.asarray(observation, dtype=np.uint8) + if ram.shape != (128,): + raise ValueError(f"Expected RAM shape (128,), got {ram.shape}") + + paddle_x = 1.005232 * float(ram[72]) - 39.797062 + ball = None + if int(ram[101]) != 0: + ball = ( + 0.999043 * float(ram[99]) - 48.370898, + 0.993263 * float(ram[101]) + 11.227841, + ) + self._update_ball(ball) + + if ball is None: + target_x = HOME_X + self.launch_sign * LAUNCH_OFFSET + else: + target_x = self._target_x(ball) + + error = target_x - paddle_x + if error > DEADBAND: + return RIGHT + if error < -DEADBAND: + return LEFT + return FIRE if ball is None else NOOP + + def _update_ball(self, ball: tuple[float, float] | None) -> None: + if ball is None: + if self.had_visible_ball and not self.ball_missing: + self.launch_sign *= -1.0 + self.ball_missing = True + self.prev_ball = None + self.velocity = None + return + + self.had_visible_ball = True + self.ball_missing = False + if self.prev_ball is not None: + dx = ball[0] - self.prev_ball[0] + dy = ball[1] - self.prev_ball[1] + if ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ): + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * self.velocity[0] + 0.5 * dx, + 0.5 * self.velocity[1] + 0.5 * dy, + ) + else: + self.velocity = None + self.prev_ball = ball + + def _target_x(self, ball: tuple[float, float]) -> float: + ball_x, ball_y = ball + if self.velocity is None: + return _clip(ball_x, PADDLE_MIN_X, PADDLE_MAX_X) + + vx, vy = self.velocity + if vy > 0.1 and ball_y <= PADDLE_Y: + steps_to_paddle = max((PADDLE_Y - ball_y) / vy, 0.0) + target_x = _reflect( + ball_x + vx * steps_to_paddle, + FIELD_LEFT, + FIELD_RIGHT, + ) + else: + target_x = ball_x + CHASE_LEAD_STEPS * vx + return _clip(target_x, PADDLE_MIN_X, PADDLE_MAX_X) diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchors.json b/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchors.json new file mode 100644 index 0000000..ac051e2 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/anchors.json @@ -0,0 +1,13 @@ +{ + "environment": "EnvPool 1.1.1 Breakout-v5, RAM-only policy input, frame_skip 1", + "max_steps": 108000, + "target_score": 864, + "reference_policy": { + "description": "Frozen observation-only geometric controller reaching 864", + "path": "anchor_policies/reference_policy.py" + }, + "starter_policy": { + "description": "Article-faithful 387-point RAM controller", + "path": "anchor_policies/starter_policy.py" + } +} diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/config.toml b/tasks/simulation_heuristics_breakout_ram_v1/verifier/config.toml new file mode 100644 index 0000000..befcafa --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/config.toml @@ -0,0 +1,34 @@ +version = "1.0" + +[evaluation] +observation_mode = "ram" +max_steps = 108000 +failure_return = -1.0 +target_score = 864.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.5 +seeds = [101] +variants = ["nominal"] + +[[evaluation.suites]] +name = "hidden_shifted_start" +weight = 0.5 +seeds = [211] +variants = ["left_6", "right_6"] + +[[evaluation.variants]] +name = "nominal" +repeat_action_probability = 0.0 +initial_actions = [] + +[[evaluation.variants]] +name = "left_6" +repeat_action_probability = 0.0 +initial_actions = [3, 3, 3, 3, 3, 3] + +[[evaluation.variants]] +name = "right_6" +repeat_action_probability = 0.0 +initial_actions = [2, 2, 2, 2, 2, 2] diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_breakout_ram_v1/verifier/evaluate_hidden.py new file mode 100644 index 0000000..fa7b10f --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/evaluate_hidden.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +import tomllib +from pathlib import Path + + +VERIFIER_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] +RUNTIME_CANDIDATES.extend(ancestor / "src" for ancestor in VERIFIER_DIR.parents) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.breakout import ( # noqa: E402 + BreakoutEvaluation, + BreakoutVariant, + evaluate_breakout_policy, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run hidden RAM-only Breakout evaluation." + ) + parser.add_argument("policy", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument( + "--config", + type=Path, + default=Path(__file__).parent / "config.toml", + help="Evaluation config; production can inject a private file.", + ) + parser.add_argument( + "--anchors", + type=Path, + default=Path(__file__).parent / "anchors.json", + help="Normalization anchors matching the selected suite.", + ) + return parser.parse_args() + + +def _variants(evaluation: dict) -> dict[str, BreakoutVariant]: + return { + variant["name"]: BreakoutVariant( + name=variant["name"], + repeat_action_probability=float( + variant.get("repeat_action_probability", 0.0) + ), + initial_actions=tuple(int(action) for action in variant["initial_actions"]), + ) + for variant in evaluation["variants"] + } + + +def _evaluate_raw( + policy: Path, + *, + evaluation: dict, +) -> tuple[dict[str, BreakoutEvaluation], float]: + variants = _variants(evaluation) + results: dict[str, BreakoutEvaluation] = {} + score = 0.0 + for suite in evaluation["suites"]: + suite_variants = tuple(variants[name] for name in suite["variants"]) + result = evaluate_breakout_policy( + policy, + observation_mode=evaluation["observation_mode"], + seeds=suite["seeds"], + max_steps=evaluation["max_steps"], + variants=suite_variants, + failure_return=evaluation["failure_return"], + ) + results[suite["name"]] = result + score += float(suite["weight"]) * result.mean_return + return results, score + + +def _anchor_score( + anchors: dict, + name: str, + *, + anchors_path: Path, + evaluation: dict, +) -> float: + anchor = anchors[name] + score = anchor.get("score") + if isinstance(score, int | float): + return float(score) + relative_path = anchor.get("path") + if not isinstance(relative_path, str) or not relative_path: + raise ValueError(f"{name} must declare score or path") + _, calibrated_score = _evaluate_raw( + anchors_path.parent / relative_path, + evaluation=evaluation, + ) + return calibrated_score + + +def main() -> None: + args = parse_args() + config = tomllib.loads(args.config.read_text()) + anchors = json.loads(args.anchors.read_text()) + evaluation = config["evaluation"] + suite_results, score = _evaluate_raw(args.policy, evaluation=evaluation) + starter_score = _anchor_score( + anchors, + "starter_policy", + anchors_path=args.anchors, + evaluation=evaluation, + ) + reference_score = _anchor_score( + anchors, + "reference_policy", + anchors_path=args.anchors, + evaluation=evaluation, + ) + if reference_score == starter_score: + raise ValueError("starter and reference anchors must have different scores") + normalized_score = round( + 100.0 * (score - starter_score) / (reference_score - starter_score), + 9, + ) + payload = { + "score": score, + "normalized_score": normalized_score, + "starter_score": starter_score, + "reference_score": reference_score, + "target_score": evaluation["target_score"], + "target_gap": score - float(evaluation["target_score"]), + "suites": { + name: result.to_dict() for name, result in suite_results.items() + }, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_breakout_ram_v1/verifier/rubrics/verifier.md new file mode 100644 index 0000000..24ead67 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/rubrics/verifier.md @@ -0,0 +1,8 @@ +# Simulation Heuristics Breakout RAM v1 Verifier Rubric + +- `task_success`: weighted hidden-suite Breakout return normalized so the + public starter maps to `0` and the frozen 864-point RAM reference maps to + `1`. +- Invalid or out-of-range actions fail the affected episode. +- The canonical reward is clamped to `[0, 1]`; detailed JSON retains raw and + unclamped normalized scores. diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/rubrics/verifier.toml b/tasks/simulation_heuristics_breakout_ram_v1/verifier/rubrics/verifier.toml new file mode 100644 index 0000000..86222d0 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/rubrics/verifier.toml @@ -0,0 +1,9 @@ +version = "0.1" + +[[criteria]] +id = "task_success" +description = "Normalized hidden-suite RAM-only Breakout return." +weight = 1.0 + +[scoring] +method = "weighted_mean" diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/test.sh b/tasks/simulation_heuristics_breakout_ram_v1/verifier/test.sh new file mode 100755 index 0000000..bd98ee1 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/test.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier +policy=/app/final_policy/policy.py + +if [ ! -f "$policy" ]; then + printf '0.0\n' > /logs/verifier/reward.txt + printf '{"reward": 0.0}\n' > /logs/verifier/reward.json + exit 0 +fi + +python /verifier/evaluate_hidden.py \ + "$policy" \ + --config /verifier/config.toml \ + --anchors /verifier/anchors.json \ + --output /logs/verifier/genesis-score.json + +python - <<'PY' +import json +from pathlib import Path + +score = json.loads(Path("/logs/verifier/genesis-score.json").read_text()) +reward = max(0.0, min(1.0, float(score["normalized_score"]) / 100.0)) +Path("/logs/verifier/reward.txt").write_text(f"{reward}\n") +Path("/logs/verifier/reward.json").write_text( + json.dumps({"reward": reward}, indent=2) + "\n" +) +PY diff --git a/tasks/simulation_heuristics_breakout_ram_v1/verifier/verifier.md b/tasks/simulation_heuristics_breakout_ram_v1/verifier/verifier.md new file mode 100644 index 0000000..ad9f9d1 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_ram_v1/verifier/verifier.md @@ -0,0 +1,40 @@ +--- +document_version: "0.3" +verifier: + name: simulation_heuristics_breakout_ram_v1 + default_strategy: deterministic + strategies: + deterministic: + type: script + command: ./test.sh + rubric: + combine: weighted_mean + dimensions: + task_success: + weight: 1.0 + source: deterministic + outputs: + reward_text: /logs/verifier/reward.txt + reward_json: /logs/verifier/reward.json + details_json: /logs/verifier/genesis-score.json + aggregate_policy: + method: weighted_mean + metrics: + task_success: 1.0 +--- + +## Verifier intent + +Run the submitted RAM-only policy on a full-horizon nominal episode and +shifted-start trajectories. The shifts move the paddle before the policy is +instantiated, changing action timing without exposing hidden state or changing +the 128-byte observation contract. + +The raw score is a weighted native Breakout return. The normalized score maps +the trusted starter to `0` and the trusted 864-point reference to `100`. +BenchFlow's reward clamps normalized score divided by 100 to `[0, 1]`; full raw, +anchor, suite, completion, invalid-action, and latency metrics remain in +`genesis-score.json`. + +The checked-in suite is reproducible. Hosted evaluation should inject private +seeds, prefixes, and matching anchors. diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/README.md b/tasks/simulation_heuristics_breakout_rgb_v1/README.md new file mode 100644 index 0000000..bc61079 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/README.md @@ -0,0 +1,74 @@ +# Simulation Heuristics Breakout RGB v1 + +This native GenesisBench task packages the article's RAM-to-RGB Breakout +transfer as a strict pixel-only policy-improvement benchmark. + +## Contract + +The agent receives `task.md`, `starter_policy/`, `evaluate.py`, and +`task_context/`. It must leave: + +```text +final_policy/ + policy.py +``` + +The policy receives raw RGB pixels only. The evaluator does not pass RAM, +reward, `info`, or object labels, and the hidden verifier is not copied into +the agent workspace. + +## Score + +```text +raw score = + 0.50 * hidden nominal return + + 0.50 * hidden shifted-start return +``` + +Normalization maps the checked-in starter to `0` and the frozen pixel-only +864-point reference to `100`. Both trusted anchors are evaluated on the same +runtime as the submission. Scores above `100` remain representable. + +## Local verification + +Use a Linux environment with `envpool==1.1.1`, then run: + +```bash +python tasks/simulation_heuristics_breakout_rgb_v1/evaluate.py \ + --policy tasks/simulation_heuristics_breakout_rgb_v1/starter_policy/policy.py + +python tasks/simulation_heuristics_breakout_rgb_v1/verifier/evaluate_hidden.py \ + tasks/simulation_heuristics_breakout_rgb_v1/oracle/policy.py +``` + +Validate the native package: + +```bash +uv run bench tasks check \ + tasks/simulation_heuristics_breakout_rgb_v1 \ + --level publication-grade +``` + +Run the trusted oracle through BenchFlow: + +```bash +uv run bench eval run \ + --tasks-dir tasks/simulation_heuristics_breakout_rgb_v1 \ + --agent oracle \ + --sandbox docker \ + --context-root . +``` + +## Public and private suites + +The checked-in verifier config is a reproducibility suite. A hosted leaderboard +should inject a private config and matching private anchors: + +```bash +python verifier/evaluate_hidden.py final_policy/policy.py \ + --config /private/final-suite.toml \ + --anchors /private/final-anchors.json +``` + +See `task_context/article_progression.md` for the `310 -> 428 -> 864` transfer +mechanisms and `task_context/provenance.md` for revision/licensing notes. diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/environment/Dockerfile b/tasks/simulation_heuristics_breakout_rgb_v1/environment/Dockerfile new file mode 100644 index 0000000..edfb433 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/environment/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12-slim + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends bash ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir \ + "envpool==1.1.1" \ + "numpy>=1.26,<3" \ + "pyyaml>=6,<7" + +RUN useradd --create-home --uid 1000 agent \ + && mkdir -p /app /opt/genesisbench + +COPY src/genesisbench /opt/genesisbench/genesisbench +COPY tasks/simulation_heuristics_breakout_rgb_v1/evaluate.py /app/evaluate.py +COPY tasks/simulation_heuristics_breakout_rgb_v1/starter_policy /app/starter_policy +COPY tasks/simulation_heuristics_breakout_rgb_v1/task_context /app/task_context + +RUN cp -R /app/starter_policy /app/final_policy \ + && chown -R root:root /opt/genesisbench \ + && chmod -R a-w /opt/genesisbench \ + && chown -R agent:agent /app + +ENV PYTHONPATH="/opt/genesisbench" + +WORKDIR /app + +RUN mkdir -p /logs/verifier /logs/agent /logs/artifacts diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/evaluate.py b/tasks/simulation_heuristics_breakout_rgb_v1/evaluate.py new file mode 100644 index 0000000..2c65187 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/evaluate.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +TASK_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = ( + TASK_DIR / "_runtime", + TASK_DIR.parent / "src", + TASK_DIR.parent.parent / "src", +) +for candidate in RUNTIME_CANDIDATES: + if candidate.is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.breakout import evaluate_breakout_policy # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate a pixel-only Breakout policy." + ) + parser.add_argument( + "--policy", + type=Path, + default=Path("final_policy/policy.py"), + ) + parser.add_argument("--episodes", type=int, default=1) + parser.add_argument("--max-steps", type=int, default=30000) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--json-output-file", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + result = evaluate_breakout_policy( + args.policy, + observation_mode="rgb", + seeds=range(args.seed, args.seed + args.episodes), + max_steps=args.max_steps, + ) + rendered = result.to_json() + print(rendered) + if args.json_output_file is not None: + args.json_output_file.parent.mkdir(parents=True, exist_ok=True) + args.json_output_file.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/oracle/policy.py b/tasks/simulation_heuristics_breakout_rgb_v1/oracle/policy.py new file mode 100644 index 0000000..6ae7733 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/oracle/policy.py @@ -0,0 +1,310 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen pixel-only reference for the Breakout RGB transfer experiment.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +FIRE = 1 +RIGHT = 2 +LEFT = 3 + +RED = np.asarray([200, 72, 72], dtype=np.uint8) +BLACK = np.asarray([0, 0, 0], dtype=np.uint8) +WALL = np.asarray([142, 142, 142], dtype=np.uint8) + +FIELD_LEFT = 8.0 +FIELD_RIGHT = 151.0 +PADDLE_MIN_X = 15.5 +PADDLE_MAX_X = 152.5 +PADDLE_Y = 189.5 +HOME_X = 106.5 +DEADBAND = 3.0 +CHASE_LEAD_STEPS = 8.0 +LAUNCH_OFFSET = 24.0 +FAST_LOW_BALL_LEAD_STEPS = 3.0 +FAST_BALL_MIN_VY = 3.0 +MAX_VELOCITY_JUMP = 24.0 +STUCK_TRIGGER_STEPS = 1024 +STUCK_SWITCH_STEPS = 256 +STUCK_OFFSET = 12.0 +STUCK_RELEASE_HORIZON = 8.0 +BRICK_BALANCE_DEADZONE = 0.01 +PADDLE_LAG = 2.0 +LAG_BALL_Y = 170.0 +MAX_MISSING_BALL_FRAMES = 8 + + +def _clip(value: float, lower: float, upper: float) -> float: + return min(max(value, lower), upper) + + +def _reflect(value: float, lower: float, upper: float) -> float: + span = upper - lower + shifted = (value - lower) % (2.0 * span) + if shifted <= span: + return lower + shifted + return upper - (shifted - span) + + +def _paddle_center(mask: np.ndarray, x_offset: int) -> float | None: + best_length = 0 + best_center = None + for row in mask: + padded = np.pad(row.astype(np.int8), (1, 1)) + edges = np.diff(padded) + starts = np.flatnonzero(edges == 1) + ends = np.flatnonzero(edges == -1) + for start, end in zip(starts, ends, strict=True): + length = int(end - start) + if length > best_length: + best_length = length + best_center = x_offset + 0.5 * (start + end - 1) + return float(best_center) if best_length >= 8 else None + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.prev_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + self.last_action = NOOP + self.launch_sign = 1.0 + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + self.missing_ball_frames = 0 + self.previous_bricks: np.ndarray | None = None + + def act(self, observation: np.ndarray) -> int: + pixels = np.asarray(observation, dtype=np.uint8) + if pixels.shape != (3, 210, 160): + raise ValueError( + f"Expected RGB shape (3, 210, 160), got {pixels.shape}" + ) + frame = np.moveaxis(pixels, 0, -1) + + ball, paddle_x, brick_balance, bricks = self._detect(frame) + self._update_progress(bricks) + self._update_ball(ball) + + if ball is None: + action = self._serve(paddle_x) + elif paddle_x is None: + action = NOOP + else: + target_x = self._target_x(ball, brick_balance) + control_x = self._control_paddle_x(paddle_x, ball) + error = target_x - control_x + if error > DEADBAND: + action = RIGHT + elif error < -DEADBAND: + action = LEFT + else: + action = NOOP + + self.last_action = action + return action + + def _detect( + self, + frame: np.ndarray, + ) -> tuple[ + tuple[float, float] | None, + float | None, + float, + np.ndarray, + ]: + red = np.all(frame == RED, axis=-1) + paddle_x = _paddle_center(red[184:196, 8:152], 8) + + active = np.logical_and( + ~np.all(frame == BLACK, axis=-1), + ~np.all(frame == WALL, axis=-1), + ) + active[:30, :] = False + active[196:, :] = False + active[:, :8] = False + active[:, 152:] = False + if paddle_x is not None: + x0 = max(8, int(round(paddle_x)) - 12) + x1 = min(152, int(round(paddle_x)) + 13) + active[184:196, x0:x1] = False + + brick_active = active[57:93, 8:152] + cells = brick_active.reshape(6, 6, 18, 8).sum(axis=(1, 3)) > 24 + occupied_pixels = np.repeat( + np.repeat(cells, 6, axis=0), + 8, + axis=1, + ) + ball_mask = active.copy() + ball_mask[57:93, 8:152] &= ~occupied_pixels + ys, xs = np.nonzero(ball_mask) + ball = None + if ( + 2 <= xs.size <= 8 + and int(xs.max() - xs.min() + 1) <= 4 + and int(ys.max() - ys.min() + 1) <= 6 + ): + neighbors = np.zeros_like(ball_mask) + neighbors[1:, :] |= ball_mask[:-1, :] + neighbors[:-1, :] |= ball_mask[1:, :] + neighbors[:, 1:] |= ball_mask[:, :-1] + neighbors[:, :-1] |= ball_mask[:, 1:] + touches_other_object = np.any( + np.logical_and( + neighbors, + np.logical_and(active, ~ball_mask), + ) + ) + if not touches_other_object: + ball = (float(xs.mean()), float(ys.mean())) + + split = brick_active.shape[1] // 2 + left = float(np.count_nonzero(brick_active[:, :split])) + right = float(np.count_nonzero(brick_active[:, split:])) + brick_balance = (left - right) / max(left + right, 1.0) + return ball, paddle_x, brick_balance, cells + + def _update_progress(self, bricks: np.ndarray) -> None: + if self.previous_bricks is None: + self.previous_bricks = bricks.copy() + return + if not np.array_equal(bricks, self.previous_bricks): + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + else: + self.steps_since_progress += 1 + if ( + self.steps_since_progress >= STUCK_TRIGGER_STEPS + and ( + self.steps_since_progress - STUCK_TRIGGER_STEPS + ) + % STUCK_SWITCH_STEPS + == 0 + ): + self.stuck_offset_index += 1 + self.previous_bricks = bricks.copy() + + def _update_ball(self, ball: tuple[float, float] | None) -> None: + if ball is None: + self.missing_ball_frames += 1 + if self.missing_ball_frames > MAX_MISSING_BALL_FRAMES: + self.prev_ball = None + self.velocity = None + return + + self.missing_ball_frames = 0 + if self.prev_ball is not None: + dx = ball[0] - self.prev_ball[0] + dy = ball[1] - self.prev_ball[1] + if ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ): + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * self.velocity[0] + 0.5 * dx, + 0.5 * self.velocity[1] + 0.5 * dy, + ) + else: + self.velocity = None + self.prev_ball = ball + + def _serve(self, paddle_x: float | None) -> int: + if paddle_x is None: + return FIRE + target_x = HOME_X + self.launch_sign * LAUNCH_OFFSET + error = target_x - paddle_x + if error > DEADBAND: + return RIGHT + if error < -DEADBAND: + return LEFT + return FIRE + + def _target_x(self, ball: tuple[float, float], balance: float) -> float: + ball_x, ball_y = ball + if self.velocity is None: + return _clip(ball_x, PADDLE_MIN_X, PADDLE_MAX_X) + + vx, vy = self.velocity + if vy > 0.1 and ball_y <= PADDLE_Y: + steps_to_paddle = max((PADDLE_Y - ball_y) / vy, 0.0) + intercept = _reflect( + ball_x + vx * steps_to_paddle, + FIELD_LEFT, + FIELD_RIGHT, + ) + target_x = intercept + self._stuck_offset( + steps_to_paddle, + balance, + ) + elif vy >= FAST_BALL_MIN_VY: + target_x = ball_x + FAST_LOW_BALL_LEAD_STEPS * vx + else: + target_x = ball_x + CHASE_LEAD_STEPS * vx + return _clip(target_x, PADDLE_MIN_X, PADDLE_MAX_X) + + def _control_paddle_x( + self, + paddle_x: float, + ball: tuple[float, float], + ) -> float: + if ( + self.velocity is None + or self.velocity[1] <= 0.1 + or ball[1] < LAG_BALL_Y + ): + return paddle_x + if self.last_action == RIGHT: + return _clip( + paddle_x + PADDLE_LAG, + PADDLE_MIN_X, + PADDLE_MAX_X, + ) + if self.last_action == LEFT: + return _clip( + paddle_x - PADDLE_LAG, + PADDLE_MIN_X, + PADDLE_MAX_X, + ) + return paddle_x + + def _stuck_offset(self, steps_to_paddle: float, balance: float) -> float: + if self.steps_since_progress < STUCK_TRIGGER_STEPS: + return 0.0 + + phase = self.stuck_offset_index % 4 + direction = 1.0 if phase in (0, 2) else -1.0 + magnitude = STUCK_OFFSET if phase in (0, 1) else 0.5 * STUCK_OFFSET + if balance > BRICK_BALANCE_DEADZONE: + direction = 1.0 + elif balance < -BRICK_BALANCE_DEADZONE: + direction = -1.0 + release = _clip( + steps_to_paddle / STUCK_RELEASE_HORIZON, + 0.0, + 1.0, + ) + return direction * magnitude * release diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/oracle/solve.sh b/tasks/simulation_heuristics_breakout_rgb_v1/oracle/solve.sh new file mode 100755 index 0000000..1af9fd8 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/oracle/solve.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /app/final_policy +cp /oracle/policy.py /app/final_policy/policy.py diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/starter_policy/policy.py b/tasks/simulation_heuristics_breakout_rgb_v1/starter_policy/policy.py new file mode 100644 index 0000000..dbeba26 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/starter_policy/policy.py @@ -0,0 +1,253 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Article-faithful 310-point pixel-only starter for Breakout.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +FIRE = 1 +RIGHT = 2 +LEFT = 3 + +RED = np.asarray([200, 72, 72], dtype=np.uint8) +BLACK = np.asarray([0, 0, 0], dtype=np.uint8) +WALL = np.asarray([142, 142, 142], dtype=np.uint8) + +FIELD_LEFT = 8.0 +FIELD_RIGHT = 151.0 +PADDLE_MIN_X = 15.5 +PADDLE_MAX_X = 152.5 +PADDLE_Y = 189.5 +HOME_X = 106.5 +DEADBAND = 3.0 +CHASE_LEAD_STEPS = 6.0 +LAUNCH_OFFSET = 24.0 +FAST_LOW_BALL_LEAD_STEPS = 3.0 +FAST_BALL_MIN_VY = 3.0 +MAX_VELOCITY_JUMP = 24.0 +STUCK_TRIGGER_STEPS = 1024 +STUCK_SWITCH_STEPS = 256 +STUCK_OFFSET = 12.0 +MAX_MISSING_BALL_FRAMES = 8 + + +def _clip(value: float, lower: float, upper: float) -> float: + return min(max(value, lower), upper) + + +def _reflect(value: float, lower: float, upper: float) -> float: + span = upper - lower + shifted = (value - lower) % (2.0 * span) + if shifted <= span: + return lower + shifted + return upper - (shifted - span) + + +def _paddle_center(mask: np.ndarray, x_offset: int) -> float | None: + best_length = 0 + best_center = None + for row in mask: + padded = np.pad(row.astype(np.int8), (1, 1)) + edges = np.diff(padded) + starts = np.flatnonzero(edges == 1) + ends = np.flatnonzero(edges == -1) + for start, end in zip(starts, ends, strict=True): + length = int(end - start) + if length > best_length: + best_length = length + best_center = x_offset + 0.5 * (start + end - 1) + return float(best_center) if best_length >= 8 else None + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.prev_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + self.last_action = NOOP + self.launch_sign = 1.0 + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + self.missing_ball_frames = 0 + self.previous_bricks: np.ndarray | None = None + + def act(self, observation: np.ndarray) -> int: + pixels = np.asarray(observation, dtype=np.uint8) + if pixels.shape != (3, 210, 160): + raise ValueError( + f"Expected RGB shape (3, 210, 160), got {pixels.shape}" + ) + frame = np.moveaxis(pixels, 0, -1) + + ball, paddle_x, bricks = self._detect(frame) + self._update_progress(bricks) + self._update_ball(ball) + + if ball is None: + action = self._serve(paddle_x) + elif paddle_x is None: + action = NOOP + else: + error = self._target_x(ball) - paddle_x + if error > DEADBAND: + action = RIGHT + elif error < -DEADBAND: + action = LEFT + else: + action = NOOP + self.last_action = action + return action + + def _detect( + self, + frame: np.ndarray, + ) -> tuple[tuple[float, float] | None, float | None, np.ndarray]: + red = np.all(frame == RED, axis=-1) + paddle_x = _paddle_center(red[184:196, 8:152], 8) + active = np.logical_and( + ~np.all(frame == BLACK, axis=-1), + ~np.all(frame == WALL, axis=-1), + ) + active[:30, :] = False + active[196:, :] = False + active[:, :8] = False + active[:, 152:] = False + if paddle_x is not None: + x0 = max(8, int(round(paddle_x)) - 12) + x1 = min(152, int(round(paddle_x)) + 13) + active[184:196, x0:x1] = False + + brick_active = active[57:93, 8:152] + cells = brick_active.reshape(6, 6, 18, 8).sum(axis=(1, 3)) > 24 + occupied_pixels = np.repeat( + np.repeat(cells, 6, axis=0), + 8, + axis=1, + ) + ball_mask = active.copy() + ball_mask[57:93, 8:152] &= ~occupied_pixels + ys, xs = np.nonzero(ball_mask) + ball = None + if ( + 2 <= xs.size <= 8 + and int(xs.max() - xs.min() + 1) <= 4 + and int(ys.max() - ys.min() + 1) <= 6 + ): + neighbors = np.zeros_like(ball_mask) + neighbors[1:, :] |= ball_mask[:-1, :] + neighbors[:-1, :] |= ball_mask[1:, :] + neighbors[:, 1:] |= ball_mask[:, :-1] + neighbors[:, :-1] |= ball_mask[:, 1:] + touches_other_object = np.any( + np.logical_and( + neighbors, + np.logical_and(active, ~ball_mask), + ) + ) + if not touches_other_object: + ball = (float(xs.mean()), float(ys.mean())) + return ball, paddle_x, cells + + def _update_progress(self, bricks: np.ndarray) -> None: + if self.previous_bricks is None: + self.previous_bricks = bricks.copy() + return + if not np.array_equal(bricks, self.previous_bricks): + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + else: + self.steps_since_progress += 1 + if ( + self.steps_since_progress >= STUCK_TRIGGER_STEPS + and ( + self.steps_since_progress - STUCK_TRIGGER_STEPS + ) + % STUCK_SWITCH_STEPS + == 0 + ): + self.stuck_offset_index += 1 + self.previous_bricks = bricks.copy() + + def _update_ball(self, ball: tuple[float, float] | None) -> None: + if ball is None: + self.missing_ball_frames += 1 + if self.missing_ball_frames > MAX_MISSING_BALL_FRAMES: + self.prev_ball = None + self.velocity = None + return + + self.missing_ball_frames = 0 + if self.prev_ball is not None: + dx = ball[0] - self.prev_ball[0] + dy = ball[1] - self.prev_ball[1] + if ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ): + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * self.velocity[0] + 0.5 * dx, + 0.5 * self.velocity[1] + 0.5 * dy, + ) + else: + self.velocity = None + self.prev_ball = ball + + def _serve(self, paddle_x: float | None) -> int: + if paddle_x is None: + return FIRE + target_x = HOME_X + self.launch_sign * LAUNCH_OFFSET + error = target_x - paddle_x + if error > DEADBAND: + return RIGHT + if error < -DEADBAND: + return LEFT + return FIRE + + def _target_x(self, ball: tuple[float, float]) -> float: + ball_x, ball_y = ball + if self.velocity is None: + return _clip(ball_x, PADDLE_MIN_X, PADDLE_MAX_X) + + vx, vy = self.velocity + if vy > 0.1 and ball_y <= PADDLE_Y: + steps_to_paddle = max((PADDLE_Y - ball_y) / vy, 0.0) + target_x = _reflect( + ball_x + vx * steps_to_paddle, + FIELD_LEFT, + FIELD_RIGHT, + ) + self._stuck_offset() + elif vy >= FAST_BALL_MIN_VY: + target_x = ball_x + FAST_LOW_BALL_LEAD_STEPS * vx + else: + target_x = ball_x + CHASE_LEAD_STEPS * vx + return _clip(target_x, PADDLE_MIN_X, PADDLE_MAX_X) + + def _stuck_offset(self) -> float: + if self.steps_since_progress < STUCK_TRIGGER_STEPS: + return 0.0 + phase = self.stuck_offset_index % 4 + direction = 1.0 if phase in (0, 2) else -1.0 + magnitude = STUCK_OFFSET if phase in (0, 1) else 0.5 * STUCK_OFFSET + return direction * magnitude diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/task.md b/tasks/simulation_heuristics_breakout_rgb_v1/task.md new file mode 100644 index 0000000..3ad7fad --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/task.md @@ -0,0 +1,145 @@ +--- +schema_version: "1.3" +task: + name: genesisbench/simulation_heuristics_breakout_rgb_v1 + description: Improve a pixel-only Atari Breakout policy toward the article's 864-point RAM-to-RGB transfer result. + authors: + - name: GenesisBench Contributors + keywords: + - simulation + - heuristics + - breakout + - atari + - computer-vision +metadata: + category: game-control + difficulty: hard + tags: + - atari + - discrete-control + - policy-search + - rgb-observation + - computer-vision + reference_task: false + genesisbench: + starter: + path: starter_policy + submission: + directory: final_policy + entrypoint: policy.py + development: + episodes: 1 + max_steps: 30000 + seeds: [0] + observation_mode: rgb + verifier: + reproduction_config: verifier/config.toml + anchors: verifier/anchors.json + supports_private_config: true +agent: + timeout_sec: 1800 + user: agent + network_mode: public +verifier: + timeout_sec: 300 + user: root + network_mode: no-network +environment: + build_timeout_sec: 1200 + cpus: 4 + memory_mb: 4096 + storage_mb: 10240 + network_mode: public + allow_internet: true + workdir: /app +benchflow: + document_version: "0.6" +--- +# Improve the pixel-only Breakout policy + +You are given the article's initial 310-point programmatic vision controller +for EnvPool `Breakout-v5`. Improve it through autonomous research, coding, and +experimentation. + +## Objective + +Maximize the native Breakout game score. The article first developed the +controller structure with RAM, then transferred the state-reading layer to +pixels. The RGB transfer progressed through `310 -> 428 -> 864`; this package +preserves the pixel-only `864` target. + +## Observation and action contract + +The policy receives only one EnvPool RGB `uint8` tensor with shape +`(3, 210, 160)`. It +does not receive RAM, reward, `info`, lives, emulator state, object labels, or +hidden evaluator configuration. + +Return one integer action: + +```text +0 = NOOP +1 = FIRE +2 = RIGHT +3 = LEFT +``` + +The evaluator runs one ALE frame per policy action with no frame skipping. + +## Starting point + +`starter_policy/policy.py` reproduces the article's `310` RGB node. The +intermediate `428` node increases ordinary chase lookahead; the `864` node also +applies stuck-offset release and lag compensation throughout the transfer +policy. + +## Development evaluation + +Run: + +```bash +python evaluate.py --policy starter_policy/policy.py +``` + +Run the article-scale RGB horizon with: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --episodes 1 \ + --max-steps 30000 +``` + +The final evaluator uses unseen seeds and a shifted initial paddle state. A +memorized action tape from the public trajectory is unlikely to transfer. + +## Required submission + +Store your best policy at: + +```text +final_policy/policy.py +``` + +It must define either: + +```python +class Policy: + def __init__(self, seed: int = 0): ... + def reset(self, seed: int = 0) -> None: ... + def act(self, observation) -> int: ... +``` + +or `make_policy(seed=0)` returning an object with `reset` and `act`. + +## Rules + +1. Work autonomously. Do not ask for user feedback. +2. Do not modify `evaluate.py` or trusted runtime files. +3. Do not access `/oracle`, `/verifier`, or hidden configuration. +4. Do not copy a completed Breakout solution from the internet. +5. The final policy receives pixels only. Reading RAM or emulator internals is + prohibited. +6. The final policy must run offline with the installed packages. +7. Before finishing, evaluate `final_policy/policy.py` and leave the best + working version in place. diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/task_context/article_progression.md b/tasks/simulation_heuristics_breakout_rgb_v1/task_context/article_progression.md new file mode 100644 index 0000000..4b41379 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/task_context/article_progression.md @@ -0,0 +1,18 @@ +# Article progression + +The RGB package represents transfer of the already-developed geometry +controller to pixels: + +| Node | Score | Mechanism | +| --- | ---: | --- | +| Starter | 310 | Pixel segmentation with ordinary chase lookahead of six | +| Intermediate | 428 | Increase ordinary chase lookahead from six to eight | +| Reference | 864 | Apply stuck-offset release and paddle-lag compensation throughout | + +The checked-in starter is the `310` node and the trusted reference is the +`864` node. The intermediate `428` behavior is regression-tested by changing +only `CHASE_LEAD_STEPS` from `6.0` to `8.0` in the starter policy. + +All three nodes receive channel-first RGB pixels only. Brick changes, ball +velocity, and no-progress duration are inferred from successive frames; RAM, +reward, and `info` are never passed to the policy. diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/task_context/evaluation.md b/tasks/simulation_heuristics_breakout_rgb_v1/task_context/evaluation.md new file mode 100644 index 0000000..dc13369 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/task_context/evaluation.md @@ -0,0 +1,24 @@ +# Evaluation + +The environment is EnvPool `1.1.1` `Breakout-v5` with the article's settings: + +- raw channel-first `3 x 210 x 160` RGB observations; +- the reduced four-action set; +- `frameskip=1`; +- `noop_max=1`; +- `use_fire_reset=True`; +- `episodic_life=False`; +- `repeat_action_probability=0`; +- `reward_clip=False`; +- a final horizon of 30,000 policy actions. + +The public evaluator reports mean/minimum/maximum return, 864-point completion +rate, invalid-policy rate, action latency, and per-episode details. + +The checked-in final suite combines a nominal start with a shifted initial +paddle position. The zero-reward prefix changes action timing before policy +control begins and penalizes fixed single-trajectory replays while preserving +the pixel-only observation contract. + +Official hosted evaluation may inject a private config with different seeds +and prefixes. Invalid actions receive the configured failure return. diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/task_context/policy_api.md b/tasks/simulation_heuristics_breakout_rgb_v1/task_context/policy_api.md new file mode 100644 index 0000000..aad81d6 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/task_context/policy_api.md @@ -0,0 +1,28 @@ +# Policy API + +The evaluator imports `final_policy/policy.py` in a clean Python process. + +Define `Policy` or `make_policy`. The policy is instantiated once per episode +and reset before the first scored action: + +```python +class Policy: + def __init__(self, seed: int = 0): + ... + + def reset(self, seed: int = 0) -> None: + ... + + def act(self, observation): + ... +``` + +`observation` is exactly one channel-first `uint8` RGB frame from EnvPool +`1.1.1` `Breakout-v5`, with shape `(3, 210, 160)`. + +The only valid outputs are scalar integer actions `0`, `1`, `2`, and `3`, +corresponding to `NOOP`, `FIRE`, `RIGHT`, and `LEFT`. The policy may keep +recurrent state and load files stored below `final_policy/`. + +No RAM, reward, `info`, lives counter, object labels, emulator object, or +hidden variant metadata is passed to the policy. diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/task_context/provenance.md b/tasks/simulation_heuristics_breakout_rgb_v1/task_context/provenance.md new file mode 100644 index 0000000..16ddd0f --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/task_context/provenance.md @@ -0,0 +1,22 @@ +# Provenance and licensing + +This task reimplements the RGB transfer experiment described in **Learning +Beyond Gradients**. + +- Upstream project: `Trinkle23897/learning-beyond-gradients` +- Reviewed revision: `3555c2956c257d49a5015b782cbe485b14fd659e` +- Reviewed artifact: `atari/breakout/heuristic_breakout.py` +- Reported RGB transfer progression: `310 -> 428 -> 864` + +The article explicitly describes this as transfer of an already-developed +geometry controller from RAM state reading to RGB segmentation, not image-only +learning from scratch. + +The starter and reference policies adapt the published controller structure +and retain its `Copyright 2021 Garena Online Private Limited` and Apache +License 2.0 notice. GenesisBench includes the Apache 2.0 text at +`LICENSES/Apache-2.0.txt`. + +No Atari ROM file is copied into this repository. The task image pins +`envpool==1.1.1`, the runtime used by the article, and relies on the Atari +assets distributed with that EnvPool wheel. diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchor_policies/reference_policy.py b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchor_policies/reference_policy.py new file mode 100644 index 0000000..6ae7733 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchor_policies/reference_policy.py @@ -0,0 +1,310 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen pixel-only reference for the Breakout RGB transfer experiment.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +FIRE = 1 +RIGHT = 2 +LEFT = 3 + +RED = np.asarray([200, 72, 72], dtype=np.uint8) +BLACK = np.asarray([0, 0, 0], dtype=np.uint8) +WALL = np.asarray([142, 142, 142], dtype=np.uint8) + +FIELD_LEFT = 8.0 +FIELD_RIGHT = 151.0 +PADDLE_MIN_X = 15.5 +PADDLE_MAX_X = 152.5 +PADDLE_Y = 189.5 +HOME_X = 106.5 +DEADBAND = 3.0 +CHASE_LEAD_STEPS = 8.0 +LAUNCH_OFFSET = 24.0 +FAST_LOW_BALL_LEAD_STEPS = 3.0 +FAST_BALL_MIN_VY = 3.0 +MAX_VELOCITY_JUMP = 24.0 +STUCK_TRIGGER_STEPS = 1024 +STUCK_SWITCH_STEPS = 256 +STUCK_OFFSET = 12.0 +STUCK_RELEASE_HORIZON = 8.0 +BRICK_BALANCE_DEADZONE = 0.01 +PADDLE_LAG = 2.0 +LAG_BALL_Y = 170.0 +MAX_MISSING_BALL_FRAMES = 8 + + +def _clip(value: float, lower: float, upper: float) -> float: + return min(max(value, lower), upper) + + +def _reflect(value: float, lower: float, upper: float) -> float: + span = upper - lower + shifted = (value - lower) % (2.0 * span) + if shifted <= span: + return lower + shifted + return upper - (shifted - span) + + +def _paddle_center(mask: np.ndarray, x_offset: int) -> float | None: + best_length = 0 + best_center = None + for row in mask: + padded = np.pad(row.astype(np.int8), (1, 1)) + edges = np.diff(padded) + starts = np.flatnonzero(edges == 1) + ends = np.flatnonzero(edges == -1) + for start, end in zip(starts, ends, strict=True): + length = int(end - start) + if length > best_length: + best_length = length + best_center = x_offset + 0.5 * (start + end - 1) + return float(best_center) if best_length >= 8 else None + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.prev_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + self.last_action = NOOP + self.launch_sign = 1.0 + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + self.missing_ball_frames = 0 + self.previous_bricks: np.ndarray | None = None + + def act(self, observation: np.ndarray) -> int: + pixels = np.asarray(observation, dtype=np.uint8) + if pixels.shape != (3, 210, 160): + raise ValueError( + f"Expected RGB shape (3, 210, 160), got {pixels.shape}" + ) + frame = np.moveaxis(pixels, 0, -1) + + ball, paddle_x, brick_balance, bricks = self._detect(frame) + self._update_progress(bricks) + self._update_ball(ball) + + if ball is None: + action = self._serve(paddle_x) + elif paddle_x is None: + action = NOOP + else: + target_x = self._target_x(ball, brick_balance) + control_x = self._control_paddle_x(paddle_x, ball) + error = target_x - control_x + if error > DEADBAND: + action = RIGHT + elif error < -DEADBAND: + action = LEFT + else: + action = NOOP + + self.last_action = action + return action + + def _detect( + self, + frame: np.ndarray, + ) -> tuple[ + tuple[float, float] | None, + float | None, + float, + np.ndarray, + ]: + red = np.all(frame == RED, axis=-1) + paddle_x = _paddle_center(red[184:196, 8:152], 8) + + active = np.logical_and( + ~np.all(frame == BLACK, axis=-1), + ~np.all(frame == WALL, axis=-1), + ) + active[:30, :] = False + active[196:, :] = False + active[:, :8] = False + active[:, 152:] = False + if paddle_x is not None: + x0 = max(8, int(round(paddle_x)) - 12) + x1 = min(152, int(round(paddle_x)) + 13) + active[184:196, x0:x1] = False + + brick_active = active[57:93, 8:152] + cells = brick_active.reshape(6, 6, 18, 8).sum(axis=(1, 3)) > 24 + occupied_pixels = np.repeat( + np.repeat(cells, 6, axis=0), + 8, + axis=1, + ) + ball_mask = active.copy() + ball_mask[57:93, 8:152] &= ~occupied_pixels + ys, xs = np.nonzero(ball_mask) + ball = None + if ( + 2 <= xs.size <= 8 + and int(xs.max() - xs.min() + 1) <= 4 + and int(ys.max() - ys.min() + 1) <= 6 + ): + neighbors = np.zeros_like(ball_mask) + neighbors[1:, :] |= ball_mask[:-1, :] + neighbors[:-1, :] |= ball_mask[1:, :] + neighbors[:, 1:] |= ball_mask[:, :-1] + neighbors[:, :-1] |= ball_mask[:, 1:] + touches_other_object = np.any( + np.logical_and( + neighbors, + np.logical_and(active, ~ball_mask), + ) + ) + if not touches_other_object: + ball = (float(xs.mean()), float(ys.mean())) + + split = brick_active.shape[1] // 2 + left = float(np.count_nonzero(brick_active[:, :split])) + right = float(np.count_nonzero(brick_active[:, split:])) + brick_balance = (left - right) / max(left + right, 1.0) + return ball, paddle_x, brick_balance, cells + + def _update_progress(self, bricks: np.ndarray) -> None: + if self.previous_bricks is None: + self.previous_bricks = bricks.copy() + return + if not np.array_equal(bricks, self.previous_bricks): + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + else: + self.steps_since_progress += 1 + if ( + self.steps_since_progress >= STUCK_TRIGGER_STEPS + and ( + self.steps_since_progress - STUCK_TRIGGER_STEPS + ) + % STUCK_SWITCH_STEPS + == 0 + ): + self.stuck_offset_index += 1 + self.previous_bricks = bricks.copy() + + def _update_ball(self, ball: tuple[float, float] | None) -> None: + if ball is None: + self.missing_ball_frames += 1 + if self.missing_ball_frames > MAX_MISSING_BALL_FRAMES: + self.prev_ball = None + self.velocity = None + return + + self.missing_ball_frames = 0 + if self.prev_ball is not None: + dx = ball[0] - self.prev_ball[0] + dy = ball[1] - self.prev_ball[1] + if ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ): + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * self.velocity[0] + 0.5 * dx, + 0.5 * self.velocity[1] + 0.5 * dy, + ) + else: + self.velocity = None + self.prev_ball = ball + + def _serve(self, paddle_x: float | None) -> int: + if paddle_x is None: + return FIRE + target_x = HOME_X + self.launch_sign * LAUNCH_OFFSET + error = target_x - paddle_x + if error > DEADBAND: + return RIGHT + if error < -DEADBAND: + return LEFT + return FIRE + + def _target_x(self, ball: tuple[float, float], balance: float) -> float: + ball_x, ball_y = ball + if self.velocity is None: + return _clip(ball_x, PADDLE_MIN_X, PADDLE_MAX_X) + + vx, vy = self.velocity + if vy > 0.1 and ball_y <= PADDLE_Y: + steps_to_paddle = max((PADDLE_Y - ball_y) / vy, 0.0) + intercept = _reflect( + ball_x + vx * steps_to_paddle, + FIELD_LEFT, + FIELD_RIGHT, + ) + target_x = intercept + self._stuck_offset( + steps_to_paddle, + balance, + ) + elif vy >= FAST_BALL_MIN_VY: + target_x = ball_x + FAST_LOW_BALL_LEAD_STEPS * vx + else: + target_x = ball_x + CHASE_LEAD_STEPS * vx + return _clip(target_x, PADDLE_MIN_X, PADDLE_MAX_X) + + def _control_paddle_x( + self, + paddle_x: float, + ball: tuple[float, float], + ) -> float: + if ( + self.velocity is None + or self.velocity[1] <= 0.1 + or ball[1] < LAG_BALL_Y + ): + return paddle_x + if self.last_action == RIGHT: + return _clip( + paddle_x + PADDLE_LAG, + PADDLE_MIN_X, + PADDLE_MAX_X, + ) + if self.last_action == LEFT: + return _clip( + paddle_x - PADDLE_LAG, + PADDLE_MIN_X, + PADDLE_MAX_X, + ) + return paddle_x + + def _stuck_offset(self, steps_to_paddle: float, balance: float) -> float: + if self.steps_since_progress < STUCK_TRIGGER_STEPS: + return 0.0 + + phase = self.stuck_offset_index % 4 + direction = 1.0 if phase in (0, 2) else -1.0 + magnitude = STUCK_OFFSET if phase in (0, 1) else 0.5 * STUCK_OFFSET + if balance > BRICK_BALANCE_DEADZONE: + direction = 1.0 + elif balance < -BRICK_BALANCE_DEADZONE: + direction = -1.0 + release = _clip( + steps_to_paddle / STUCK_RELEASE_HORIZON, + 0.0, + 1.0, + ) + return direction * magnitude * release diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchor_policies/starter_policy.py b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchor_policies/starter_policy.py new file mode 100644 index 0000000..dbeba26 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchor_policies/starter_policy.py @@ -0,0 +1,253 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Article-faithful 310-point pixel-only starter for Breakout.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +FIRE = 1 +RIGHT = 2 +LEFT = 3 + +RED = np.asarray([200, 72, 72], dtype=np.uint8) +BLACK = np.asarray([0, 0, 0], dtype=np.uint8) +WALL = np.asarray([142, 142, 142], dtype=np.uint8) + +FIELD_LEFT = 8.0 +FIELD_RIGHT = 151.0 +PADDLE_MIN_X = 15.5 +PADDLE_MAX_X = 152.5 +PADDLE_Y = 189.5 +HOME_X = 106.5 +DEADBAND = 3.0 +CHASE_LEAD_STEPS = 6.0 +LAUNCH_OFFSET = 24.0 +FAST_LOW_BALL_LEAD_STEPS = 3.0 +FAST_BALL_MIN_VY = 3.0 +MAX_VELOCITY_JUMP = 24.0 +STUCK_TRIGGER_STEPS = 1024 +STUCK_SWITCH_STEPS = 256 +STUCK_OFFSET = 12.0 +MAX_MISSING_BALL_FRAMES = 8 + + +def _clip(value: float, lower: float, upper: float) -> float: + return min(max(value, lower), upper) + + +def _reflect(value: float, lower: float, upper: float) -> float: + span = upper - lower + shifted = (value - lower) % (2.0 * span) + if shifted <= span: + return lower + shifted + return upper - (shifted - span) + + +def _paddle_center(mask: np.ndarray, x_offset: int) -> float | None: + best_length = 0 + best_center = None + for row in mask: + padded = np.pad(row.astype(np.int8), (1, 1)) + edges = np.diff(padded) + starts = np.flatnonzero(edges == 1) + ends = np.flatnonzero(edges == -1) + for start, end in zip(starts, ends, strict=True): + length = int(end - start) + if length > best_length: + best_length = length + best_center = x_offset + 0.5 * (start + end - 1) + return float(best_center) if best_length >= 8 else None + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.prev_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + self.last_action = NOOP + self.launch_sign = 1.0 + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + self.missing_ball_frames = 0 + self.previous_bricks: np.ndarray | None = None + + def act(self, observation: np.ndarray) -> int: + pixels = np.asarray(observation, dtype=np.uint8) + if pixels.shape != (3, 210, 160): + raise ValueError( + f"Expected RGB shape (3, 210, 160), got {pixels.shape}" + ) + frame = np.moveaxis(pixels, 0, -1) + + ball, paddle_x, bricks = self._detect(frame) + self._update_progress(bricks) + self._update_ball(ball) + + if ball is None: + action = self._serve(paddle_x) + elif paddle_x is None: + action = NOOP + else: + error = self._target_x(ball) - paddle_x + if error > DEADBAND: + action = RIGHT + elif error < -DEADBAND: + action = LEFT + else: + action = NOOP + self.last_action = action + return action + + def _detect( + self, + frame: np.ndarray, + ) -> tuple[tuple[float, float] | None, float | None, np.ndarray]: + red = np.all(frame == RED, axis=-1) + paddle_x = _paddle_center(red[184:196, 8:152], 8) + active = np.logical_and( + ~np.all(frame == BLACK, axis=-1), + ~np.all(frame == WALL, axis=-1), + ) + active[:30, :] = False + active[196:, :] = False + active[:, :8] = False + active[:, 152:] = False + if paddle_x is not None: + x0 = max(8, int(round(paddle_x)) - 12) + x1 = min(152, int(round(paddle_x)) + 13) + active[184:196, x0:x1] = False + + brick_active = active[57:93, 8:152] + cells = brick_active.reshape(6, 6, 18, 8).sum(axis=(1, 3)) > 24 + occupied_pixels = np.repeat( + np.repeat(cells, 6, axis=0), + 8, + axis=1, + ) + ball_mask = active.copy() + ball_mask[57:93, 8:152] &= ~occupied_pixels + ys, xs = np.nonzero(ball_mask) + ball = None + if ( + 2 <= xs.size <= 8 + and int(xs.max() - xs.min() + 1) <= 4 + and int(ys.max() - ys.min() + 1) <= 6 + ): + neighbors = np.zeros_like(ball_mask) + neighbors[1:, :] |= ball_mask[:-1, :] + neighbors[:-1, :] |= ball_mask[1:, :] + neighbors[:, 1:] |= ball_mask[:, :-1] + neighbors[:, :-1] |= ball_mask[:, 1:] + touches_other_object = np.any( + np.logical_and( + neighbors, + np.logical_and(active, ~ball_mask), + ) + ) + if not touches_other_object: + ball = (float(xs.mean()), float(ys.mean())) + return ball, paddle_x, cells + + def _update_progress(self, bricks: np.ndarray) -> None: + if self.previous_bricks is None: + self.previous_bricks = bricks.copy() + return + if not np.array_equal(bricks, self.previous_bricks): + self.steps_since_progress = 0 + self.stuck_offset_index = 0 + else: + self.steps_since_progress += 1 + if ( + self.steps_since_progress >= STUCK_TRIGGER_STEPS + and ( + self.steps_since_progress - STUCK_TRIGGER_STEPS + ) + % STUCK_SWITCH_STEPS + == 0 + ): + self.stuck_offset_index += 1 + self.previous_bricks = bricks.copy() + + def _update_ball(self, ball: tuple[float, float] | None) -> None: + if ball is None: + self.missing_ball_frames += 1 + if self.missing_ball_frames > MAX_MISSING_BALL_FRAMES: + self.prev_ball = None + self.velocity = None + return + + self.missing_ball_frames = 0 + if self.prev_ball is not None: + dx = ball[0] - self.prev_ball[0] + dy = ball[1] - self.prev_ball[1] + if ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ): + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * self.velocity[0] + 0.5 * dx, + 0.5 * self.velocity[1] + 0.5 * dy, + ) + else: + self.velocity = None + self.prev_ball = ball + + def _serve(self, paddle_x: float | None) -> int: + if paddle_x is None: + return FIRE + target_x = HOME_X + self.launch_sign * LAUNCH_OFFSET + error = target_x - paddle_x + if error > DEADBAND: + return RIGHT + if error < -DEADBAND: + return LEFT + return FIRE + + def _target_x(self, ball: tuple[float, float]) -> float: + ball_x, ball_y = ball + if self.velocity is None: + return _clip(ball_x, PADDLE_MIN_X, PADDLE_MAX_X) + + vx, vy = self.velocity + if vy > 0.1 and ball_y <= PADDLE_Y: + steps_to_paddle = max((PADDLE_Y - ball_y) / vy, 0.0) + target_x = _reflect( + ball_x + vx * steps_to_paddle, + FIELD_LEFT, + FIELD_RIGHT, + ) + self._stuck_offset() + elif vy >= FAST_BALL_MIN_VY: + target_x = ball_x + FAST_LOW_BALL_LEAD_STEPS * vx + else: + target_x = ball_x + CHASE_LEAD_STEPS * vx + return _clip(target_x, PADDLE_MIN_X, PADDLE_MAX_X) + + def _stuck_offset(self) -> float: + if self.steps_since_progress < STUCK_TRIGGER_STEPS: + return 0.0 + phase = self.stuck_offset_index % 4 + direction = 1.0 if phase in (0, 2) else -1.0 + magnitude = STUCK_OFFSET if phase in (0, 1) else 0.5 * STUCK_OFFSET + return direction * magnitude diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchors.json b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchors.json new file mode 100644 index 0000000..1b54ea1 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/anchors.json @@ -0,0 +1,13 @@ +{ + "environment": "EnvPool 1.1.1 Breakout-v5, RGB-only policy input, frame_skip 1", + "max_steps": 30000, + "target_score": 864, + "reference_policy": { + "description": "Frozen pixel-only geometric controller reaching 864", + "path": "anchor_policies/reference_policy.py" + }, + "starter_policy": { + "description": "Article-faithful 310-point pixel-only controller", + "path": "anchor_policies/starter_policy.py" + } +} diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/config.toml b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/config.toml new file mode 100644 index 0000000..330c67a --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/config.toml @@ -0,0 +1,29 @@ +version = "1.0" + +[evaluation] +observation_mode = "rgb" +max_steps = 30000 +failure_return = -1.0 +target_score = 864.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.5 +seeds = [101] +variants = ["nominal"] + +[[evaluation.suites]] +name = "hidden_shifted_start" +weight = 0.5 +seeds = [211] +variants = ["right_6"] + +[[evaluation.variants]] +name = "nominal" +repeat_action_probability = 0.0 +initial_actions = [] + +[[evaluation.variants]] +name = "right_6" +repeat_action_probability = 0.0 +initial_actions = [2, 2, 2, 2, 2, 2] diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/evaluate_hidden.py new file mode 100644 index 0000000..986709e --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/evaluate_hidden.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +import tomllib +from pathlib import Path + + +VERIFIER_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] +RUNTIME_CANDIDATES.extend(ancestor / "src" for ancestor in VERIFIER_DIR.parents) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.breakout import ( # noqa: E402 + BreakoutEvaluation, + BreakoutVariant, + evaluate_breakout_policy, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run hidden pixel-only Breakout evaluation." + ) + parser.add_argument("policy", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument( + "--config", + type=Path, + default=Path(__file__).parent / "config.toml", + help="Evaluation config; production can inject a private file.", + ) + parser.add_argument( + "--anchors", + type=Path, + default=Path(__file__).parent / "anchors.json", + help="Normalization anchors matching the selected suite.", + ) + return parser.parse_args() + + +def _variants(evaluation: dict) -> dict[str, BreakoutVariant]: + return { + variant["name"]: BreakoutVariant( + name=variant["name"], + repeat_action_probability=float( + variant.get("repeat_action_probability", 0.0) + ), + initial_actions=tuple(int(action) for action in variant["initial_actions"]), + ) + for variant in evaluation["variants"] + } + + +def _evaluate_raw( + policy: Path, + *, + evaluation: dict, +) -> tuple[dict[str, BreakoutEvaluation], float]: + variants = _variants(evaluation) + results: dict[str, BreakoutEvaluation] = {} + score = 0.0 + for suite in evaluation["suites"]: + suite_variants = tuple(variants[name] for name in suite["variants"]) + result = evaluate_breakout_policy( + policy, + observation_mode=evaluation["observation_mode"], + seeds=suite["seeds"], + max_steps=evaluation["max_steps"], + variants=suite_variants, + failure_return=evaluation["failure_return"], + ) + results[suite["name"]] = result + score += float(suite["weight"]) * result.mean_return + return results, score + + +def _anchor_score( + anchors: dict, + name: str, + *, + anchors_path: Path, + evaluation: dict, +) -> float: + anchor = anchors[name] + score = anchor.get("score") + if isinstance(score, int | float): + return float(score) + relative_path = anchor.get("path") + if not isinstance(relative_path, str) or not relative_path: + raise ValueError(f"{name} must declare score or path") + _, calibrated_score = _evaluate_raw( + anchors_path.parent / relative_path, + evaluation=evaluation, + ) + return calibrated_score + + +def main() -> None: + args = parse_args() + config = tomllib.loads(args.config.read_text()) + anchors = json.loads(args.anchors.read_text()) + evaluation = config["evaluation"] + suite_results, score = _evaluate_raw(args.policy, evaluation=evaluation) + starter_score = _anchor_score( + anchors, + "starter_policy", + anchors_path=args.anchors, + evaluation=evaluation, + ) + reference_score = _anchor_score( + anchors, + "reference_policy", + anchors_path=args.anchors, + evaluation=evaluation, + ) + if reference_score == starter_score: + raise ValueError("starter and reference anchors must have different scores") + normalized_score = round( + 100.0 * (score - starter_score) / (reference_score - starter_score), + 9, + ) + payload = { + "score": score, + "normalized_score": normalized_score, + "starter_score": starter_score, + "reference_score": reference_score, + "target_score": evaluation["target_score"], + "target_gap": score - float(evaluation["target_score"]), + "suites": { + name: result.to_dict() for name, result in suite_results.items() + }, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/rubrics/verifier.md new file mode 100644 index 0000000..e18ad1b --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/rubrics/verifier.md @@ -0,0 +1,9 @@ +# Simulation Heuristics Breakout RGB v1 Verifier Rubric + +- `task_success`: weighted hidden-suite Breakout return normalized so the + public starter maps to `0` and the frozen 864-point pixel-only reference maps + to `1`. +- Any attempt to return malformed/out-of-range actions fails the affected + episode; the policy process receives RGB frames only. +- The canonical reward is clamped to `[0, 1]`; detailed JSON retains raw and + unclamped normalized scores. diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/rubrics/verifier.toml b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/rubrics/verifier.toml new file mode 100644 index 0000000..cd992b5 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/rubrics/verifier.toml @@ -0,0 +1,9 @@ +version = "0.1" + +[[criteria]] +id = "task_success" +description = "Normalized hidden-suite pixel-only Breakout return." +weight = 1.0 + +[scoring] +method = "weighted_mean" diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/test.sh b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/test.sh new file mode 100755 index 0000000..bd98ee1 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/test.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier +policy=/app/final_policy/policy.py + +if [ ! -f "$policy" ]; then + printf '0.0\n' > /logs/verifier/reward.txt + printf '{"reward": 0.0}\n' > /logs/verifier/reward.json + exit 0 +fi + +python /verifier/evaluate_hidden.py \ + "$policy" \ + --config /verifier/config.toml \ + --anchors /verifier/anchors.json \ + --output /logs/verifier/genesis-score.json + +python - <<'PY' +import json +from pathlib import Path + +score = json.loads(Path("/logs/verifier/genesis-score.json").read_text()) +reward = max(0.0, min(1.0, float(score["normalized_score"]) / 100.0)) +Path("/logs/verifier/reward.txt").write_text(f"{reward}\n") +Path("/logs/verifier/reward.json").write_text( + json.dumps({"reward": reward}, indent=2) + "\n" +) +PY diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/verifier/verifier.md b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/verifier.md new file mode 100644 index 0000000..824c888 --- /dev/null +++ b/tasks/simulation_heuristics_breakout_rgb_v1/verifier/verifier.md @@ -0,0 +1,40 @@ +--- +document_version: "0.3" +verifier: + name: simulation_heuristics_breakout_rgb_v1 + default_strategy: deterministic + strategies: + deterministic: + type: script + command: ./test.sh + rubric: + combine: weighted_mean + dimensions: + task_success: + weight: 1.0 + source: deterministic + outputs: + reward_text: /logs/verifier/reward.txt + reward_json: /logs/verifier/reward.json + details_json: /logs/verifier/genesis-score.json + aggregate_policy: + method: weighted_mean + metrics: + task_success: 1.0 +--- + +## Verifier intent + +Run the submitted policy on raw RGB frames only, using a full 30,000-frame +nominal episode and a shifted-start episode. The shift moves the paddle before +the policy is instantiated, changing action timing without exposing RAM, +reward, `info`, or hidden state. + +The raw score is a weighted native Breakout return. The normalized score maps +the trusted RGB starter to `0` and the trusted pixel-only 864-point reference +to `100`. BenchFlow's reward clamps normalized score divided by 100 to `[0, 1]`; +full raw, anchor, suite, completion, invalid-action, and latency metrics remain +in `genesis-score.json`. + +The checked-in suite is reproducible. Hosted evaluation should inject private +seeds, prefixes, and matching anchors. diff --git a/tasks/simulation_heuristics_halfcheetah_v1/README.md b/tasks/simulation_heuristics_halfcheetah_v1/README.md new file mode 100644 index 0000000..a8abbb2 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/README.md @@ -0,0 +1,131 @@ +# Simulation Heuristics HalfCheetah v1 + +`simulation_heuristics_halfcheetah_v1` packages the MuJoCo HalfCheetah article +experiment from *Learning Beyond Gradients* as a native GenesisBench task. + +The research progression remains visible in the benchmark: + +```text +asymmetric Fourier CPG +→ joint-space PD tracking +→ copied-model one-step action scoring +→ top-K two-level online tree +→ staged swing-amplitude schedule +``` + +## Task contract + +The agent receives: + +- `task.md`: BenchFlow-native config and complete instruction; +- `starter_policy/policy.py`: the asymmetric CPG/PD baseline; +- `evaluate.py`: queryable development feedback; +- `task_context/`: reward, policy API, and article context; +- `_runtime/`: copied into the isolated workspace by the preparation script. + +The preparation step creates `final_policy/policy.py` from the starter. The +clean verifier and oracle are excluded from the prepared agent workspace. + +## Published reference + +The article reports the staged-tree policy on seeds `100..104`: + +```text +mean 11836.693 +min 11735.0 +max 12041.2 +``` + +The trusted policy reimplements the published controller while adapting it to +an observation-only action interface. The optional configuration hook supplies +a copied MuJoCo XML path, not the live environment. + +## Score + +```text +raw score = + 0.70 * hidden nominal mean return + + 0.30 * hidden dynamics-robustness mean return +``` + +The normalized score maps: + +- the checked-in asymmetric CPG/PD starter to `0`; +- the checked-in staged-tree reference to `100`. + +Both anchors are evaluated locally with the same runtime and dynamics suite as +the submitted policy. Scores above `100` are valid. + +## Run locally + +```bash +uv sync --extra dev + +uv run python tasks/simulation_heuristics_halfcheetah_v1/evaluate.py \ + --policy tasks/simulation_heuristics_halfcheetah_v1/starter_policy/policy.py +``` + +Run a one-step reference smoke: + +```bash +uv run python tasks/simulation_heuristics_halfcheetah_v1/evaluate.py \ + --policy tasks/simulation_heuristics_halfcheetah_v1/oracle/policy.py \ + --episodes 1 \ + --max-steps 1 \ + --seed 100 +``` + +Run the full article reproduction: + +```bash +uv run python tasks/simulation_heuristics_halfcheetah_v1/evaluate.py \ + --policy tasks/simulation_heuristics_halfcheetah_v1/oracle/policy.py \ + --episodes 5 \ + --max-steps 1000 \ + --seed 100 +``` + +The exact tree planner is CPU-heavy. One 1,000-step seed took roughly six +minutes in the source script and eight minutes through the benchmark evaluator +on the development host, so the five-seed and full hidden suites are +long-running publication checks rather than unit tests. + +Validate the native task: + +```bash +uv run python scripts/validate_tasks.py \ + --task simulation_heuristics_halfcheetah_v1 + +uv run bench tasks check \ + tasks/simulation_heuristics_halfcheetah_v1 \ + --level publication-grade +``` + +Run the trusted oracle through BenchFlow: + +```bash +uv run bench eval run \ + --tasks-dir tasks/simulation_heuristics_halfcheetah_v1 \ + --agent oracle \ + --sandbox docker \ + --context-root . +``` + +Prepare the public workspace: + +```bash +uv run python scripts/prepare_task.py \ + simulation_heuristics_halfcheetah_v1 \ + /tmp/genesisbench-simulation-heuristics-halfcheetah-v1 \ + --force +``` + +The prepared workspace must not contain `verifier/`, `oracle/`, or `evidence/`. + +## Provenance + +The inspected article artifact repository did not contain an explicit license +at commit `3555c2956c257d49a5015b782cbe485b14fd659e`. No upstream source file is +vendored. The implementation is a GPL-3.0-or-later policy-API adaptation; the +published numeric controller parameters are retained as reproducibility data. +See `evidence/source_provenance.md` for the exact evidence trail. diff --git a/tasks/simulation_heuristics_halfcheetah_v1/environment/Dockerfile b/tasks/simulation_heuristics_halfcheetah_v1/environment/Dockerfile new file mode 100644 index 0000000..acc3441 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/environment/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12-slim + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends bash ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir \ + "gymnasium[mujoco]>=1.2.2,<2" \ + "numpy>=1.26,<3" \ + "pyyaml>=6,<7" + +RUN useradd --create-home --uid 1000 agent \ + && mkdir -p /app /opt/genesisbench + +COPY src/genesisbench /opt/genesisbench/genesisbench +COPY tasks/simulation_heuristics_halfcheetah_v1/evaluate.py /app/evaluate.py +COPY tasks/simulation_heuristics_halfcheetah_v1/starter_policy /app/starter_policy +COPY tasks/simulation_heuristics_halfcheetah_v1/task_context /app/task_context + +RUN cp -R /app/starter_policy /app/final_policy \ + && chown -R root:root /opt/genesisbench \ + && chmod -R a-w /opt/genesisbench \ + && chown -R agent:agent /app + +ENV PYTHONPATH="/opt/genesisbench" + +WORKDIR /app + +RUN mkdir -p /logs/verifier /logs/agent /logs/artifacts diff --git a/tasks/simulation_heuristics_halfcheetah_v1/evaluate.py b/tasks/simulation_heuristics_halfcheetah_v1/evaluate.py new file mode 100755 index 0000000..a304875 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/evaluate.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +TASK_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = ( + TASK_DIR / "_runtime", + TASK_DIR.parent / "src", + TASK_DIR.parent.parent / "src", +) +for candidate in RUNTIME_CANDIDATES: + if candidate.is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.halfcheetah import ( # noqa: E402 + DynamicsVariant, + evaluate_halfcheetah_policy, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate a HalfCheetah program policy." + ) + parser.add_argument( + "--policy", + type=Path, + default=Path("final_policy/policy.py"), + ) + parser.add_argument("--episodes", type=int, default=3) + parser.add_argument("--max-steps", type=int, default=300) + parser.add_argument("--seed", type=int, default=100) + parser.add_argument("--json-output-file", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + result = evaluate_halfcheetah_policy( + args.policy, + seeds=range(args.seed, args.seed + args.episodes), + max_steps=args.max_steps, + variants=(DynamicsVariant(),), + ) + rendered = result.to_json() + print(rendered) + if args.json_output_file is not None: + args.json_output_file.parent.mkdir(parents=True, exist_ok=True) + args.json_output_file.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_halfcheetah_v1/evidence/article_reproduction.json b/tasks/simulation_heuristics_halfcheetah_v1/evidence/article_reproduction.json new file mode 100644 index 0000000..e8d690b --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/evidence/article_reproduction.json @@ -0,0 +1,33 @@ +{ + "article": { + "expected_max_return": 12041.2, + "expected_mean_return": 11836.693, + "expected_min_return": 11735.0, + "seeds": [ + 100, + 101, + 102, + 103, + 104 + ] + }, + "command": "uv run python tasks/simulation_heuristics_halfcheetah_v1/evaluate.py --policy tasks/simulation_heuristics_halfcheetah_v1/oracle/policy.py --episodes 5 --max-steps 1000 --seed 100", + "observed": { + "episode_returns": [ + 12041.189857475818, + 11735.02927325886, + 11854.710591778263, + 11767.164473961016, + 11785.373052623192 + ], + "invalid_episode_rate": 0.0, + "max_return": 12041.189857475818, + "mean_return": 11836.693449819431, + "min_return": 11735.02927325886 + }, + "source": { + "artifact": "mujoco/halfcheetah/heuristic_halfcheetah_v5.py", + "repository": "Trinkle23897/learning-beyond-gradients", + "revision": "3555c2956c257d49a5015b782cbe485b14fd659e" + } +} diff --git a/tasks/simulation_heuristics_halfcheetah_v1/evidence/source_provenance.md b/tasks/simulation_heuristics_halfcheetah_v1/evidence/source_provenance.md new file mode 100644 index 0000000..b04d862 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/evidence/source_provenance.md @@ -0,0 +1,52 @@ +# Source provenance and licensing review + +## Evidence inspected + +- Artifact repository: + `https://github.com/Trinkle23897/learning-beyond-gradients` +- Inspected commit: + `3555c2956c257d49a5015b782cbe485b14fd659e` +- Article source: + `learning-beyond-gradient.en.md` +- Experiment source: + `mujoco/halfcheetah/heuristic_halfcheetah_v5.py` +- Iteration record: + `mujoco/halfcheetah/heuristic_halfcheetah_v5_log.md` + +The article identifies the work as *Learning Beyond Gradients*, Jiayi Weng, +May 2026. Its appendix reports a five-episode mean of `11836.693` for seeds +`100..104`. + +## Licensing treatment + +No `LICENSE`, `COPYING`, or equivalent repository-level license was present in +the inspected source tree at the commit above. Therefore GenesisBench does not +vendor the upstream Python file or present it as licensed source code. + +The task contains a newly written policy-API adaptation under GenesisBench's +GPL-3.0-or-later licensing. It preserves the published algorithmic semantics +and numeric controller parameters as experiment data required for +reproducibility: + +- asymmetric two-rate CPG/PD gait; +- copied-model, short-horizon scoring; +- top-K two-level action tree; +- staged swing-amplitude schedule; +- published seeds and aggregate target. + +The implementation reconstructs a local MuJoCo planning state from the public +17-value observation rather than receiving the live environment object used by +the original script. + +## Local source check + +On July 12, 2026, the unmodified source command for seed 100 produced +`12041.189857475818`. A 20-step parity probe showed the GenesisBench oracle and +source planner selecting identical actions at every checked step. The complete +GenesisBench oracle run for seed 100 also returned exactly +`12041.189857475818` over 1,000 steps. + +The full packaged five-seed reproduction subsequently produced mean +`11836.693449819431`, minimum `11735.02927325886`, and maximum +`12041.189857475818`. See `evidence/article_reproduction.json` for the +machine-readable command and per-seed returns. diff --git a/tasks/simulation_heuristics_halfcheetah_v1/oracle/policy.py b/tasks/simulation_heuristics_halfcheetah_v1/oracle/policy.py new file mode 100644 index 0000000..efa70c0 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/oracle/policy.py @@ -0,0 +1,455 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Staged asymmetric CPG/PD controller with two-level online tree MPC.""" + +from __future__ import annotations + +import math + +import mujoco +import mujoco.rollout +import numpy as np + + +BASE_GAIT = np.asarray( + [ + 2.0720429776207037, + 2.0890962589603244, + 0.7454172634828156, + -0.06577692937619559, + 0.6454089230776165, + -0.6746024815060896, + -0.5255276137795621, + -0.776911900431121, + 0.4784639889085752, + -1.5344226994990133, + 0.8642039688512475, + -0.6508749004237321, + -0.8139980805865409, + -0.43273458554686073, + 0.9311256504045836, + -0.3884941211535119, + 0.5058403771772146, + 0.29329785944557646, + -0.41630444646878295, + -0.17793318904003722, + 0.25287195521722133, + 0.1864602367775759, + -0.2174643453198326, + 0.32053306369231493, + 0.18857020059058224, + 0.2877837436012836, + -0.46494131953713835, + -0.23362679675599532, + -0.179250145837136, + -0.2538333727428432, + -0.19506084586961667, + -0.3065628153040692, + ], + dtype=np.float64, +) +PHASE_SUBSTEPS = 10 +PLANNING_HORIZON = 14 +TERMINAL_VELOCITY_WEIGHT = 0.5 +PD_KP = 8.0 +PD_KD = 0.0 +TREE_WIDTH = 8 +FAST_STAGE_STEP = 300 +FINAL_STAGE_STEP = 900 +COORDINATE_DELTAS = ( + -0.10, + 0.10, + -0.25, + 0.25, + -0.50, + 0.50, + -0.75, + 0.75, + -1.00, + 1.00, +) +RANDOM_BLOCKS = ((0.15, 64), (0.35, 128), (0.70, 192)) +BANG_BANG_ACTIONS = np.asarray( + np.meshgrid(*[[-1.0, 1.0]] * 6), dtype=np.float64 +).T.reshape(-1, 6) + + +def _frequency(raw_value: np.ndarray) -> np.ndarray: + sigmoid = 1.0 / (1.0 + np.exp(-raw_value)) + return 0.5 + 4.5 * sigmoid + + +def _stage_gait(amplitude: float, front_lower_leg_bias: float) -> np.ndarray: + gait = BASE_GAIT.copy() + gait[0] -= 1.80 + gait[1] -= 1.10 + coefficients = gait[2:].reshape(5, 6).copy() + coefficients[1:] *= amplitude + coefficients[0, 4:] += front_lower_leg_bias + gait[2:] = coefficients.reshape(-1) + return gait + + +START_GAIT = _stage_gait(1.15, 0.15) +FAST_GAIT = _stage_gait(1.18, 0.20) +FINAL_GAIT = START_GAIT.copy() + + +def _advance_phase( + gait: np.ndarray, + phase: np.ndarray, + dt: float, +) -> np.ndarray: + stance_frequency = _frequency(gait[:, 0]) + swing_frequency = _frequency(gait[:, 1]) + updated = phase.copy() + for _ in range(PHASE_SUBSTEPS): + frequency = np.where( + np.sin(updated) > 0.0, + swing_frequency, + stance_frequency, + ) + updated = (updated + 2.0 * math.pi * frequency * (dt / PHASE_SUBSTEPS)) % ( + 2.0 * math.pi + ) + return updated + + +def _cpg_actions( + gait: np.ndarray, + observation: np.ndarray, + phase: np.ndarray, + dt: float, +) -> tuple[np.ndarray, np.ndarray]: + next_phase = _advance_phase(gait, phase, dt) + features = np.stack( + ( + np.ones_like(next_phase), + np.sin(next_phase), + np.cos(next_phase), + np.sin(2.0 * next_phase), + np.cos(2.0 * next_phase), + ), + axis=1, + ) + target = np.einsum( + "nf,nfa->na", + features, + gait[:, 2:].reshape(-1, 5, 6), + ) + action = PD_KP * (target - observation[:, 2:8]) - PD_KD * observation[:, 11:17] + return np.clip(action, -1.0, 1.0).astype(np.float32), next_phase + + +def _single_cpg_action( + gait: np.ndarray, + observation: np.ndarray, + phase: float, + dt: float, +) -> tuple[np.ndarray, float]: + next_phase = _advance_phase( + gait[None, :], + np.asarray([phase], dtype=np.float64), + dt, + ) + phase_value = float(next_phase[0]) + features = np.asarray( + [ + 1.0, + math.sin(phase_value), + math.cos(phase_value), + math.sin(2.0 * phase_value), + math.cos(2.0 * phase_value), + ], + dtype=np.float64, + ) + target = features @ gait[2:].reshape(5, 6) + action = PD_KP * (target - observation[2:8]) - PD_KD * observation[11:17] + return action, phase_value + + +def _model_state(data: mujoco.MjData) -> np.ndarray: + return np.concatenate(([data.time], data.qpos.copy(), data.qvel.copy())).astype( + np.float64 + ) + + +def _observations_from_states( + model: mujoco.MjModel, + state: np.ndarray, +) -> np.ndarray: + qpos_start = 1 + qvel_start = qpos_start + model.nq + qpos = state[:, qpos_start:qvel_start] + qvel = state[:, qvel_start : qvel_start + model.nv] + return np.concatenate((qpos[:, 1:], qvel), axis=1) + + +def _step_batch( + model: mujoco.MjModel, + rollout_data: mujoco.MjData, + state: np.ndarray, + action: np.ndarray, + frame_skip: int, + dt: float, +) -> tuple[np.ndarray, np.ndarray]: + controls = np.repeat( + action[:, None, :], + frame_skip, + axis=1, + ).astype(np.float64) + states, _ = mujoco.rollout.rollout( + model, + rollout_data, + initial_state=state, + control=controls, + ) + next_state = states[:, -1, :] + forward_reward = (next_state[:, 1] - state[:, 1]) / dt + control_cost = 0.1 * np.sum(action * action, axis=1) + return next_state, forward_reward - control_cost + + +def _score_actions( + *, + gait: np.ndarray, + model: mujoco.MjModel, + rollout_data: mujoco.MjData, + initial_state: np.ndarray, + phase: np.ndarray, + first_actions: np.ndarray, + frame_skip: int, + dt: float, +) -> np.ndarray: + count = len(first_actions) + repeated_gait = np.repeat(gait[None, :], count, axis=0) + simulated_phase = _advance_phase(repeated_gait, phase, dt) + state, score = _step_batch( + model, + rollout_data, + initial_state, + np.clip(first_actions, -1.0, 1.0), + frame_skip, + dt, + ) + + for _ in range(1, PLANNING_HORIZON): + action, simulated_phase = _cpg_actions( + repeated_gait, + _observations_from_states(model, state), + simulated_phase, + dt, + ) + state, reward = _step_batch( + model, + rollout_data, + state, + action, + frame_skip, + dt, + ) + score += reward + + qvel_start = 1 + model.nq + return score + TERMINAL_VELOCITY_WEIGHT * state[:, qvel_start] + + +def _candidate_actions( + base_action: np.ndarray, + rng: np.random.Generator, + *, + include_random: bool, +) -> np.ndarray: + candidates = [base_action] + for joint in range(6): + for delta in COORDINATE_DELTAS: + candidate = base_action.copy() + candidate[joint] = np.clip( + candidate[joint] + delta, + -1.0, + 1.0, + ) + candidates.append(candidate) + + if include_random: + for standard_deviation, count in RANDOM_BLOCKS: + samples = base_action + standard_deviation * rng.standard_normal((count, 6)) + candidates.extend(np.clip(samples, -1.0, 1.0)) + + candidates.extend(BANG_BANG_ACTIONS) + return np.asarray(candidates, dtype=np.float64) + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self._model: mujoco.MjModel | None = None + self._state_data: mujoco.MjData | None = None + self._rollout_data: mujoco.MjData | None = None + self._frame_skip = 5 + self._dt = 0.05 + self.reset(seed) + + def configure_simulator( + self, + *, + model_xml_path: str, + frame_skip: int = 5, + ) -> None: + model = mujoco.MjModel.from_xml_path(model_xml_path) + if (model.nq, model.nv, model.nu) != (9, 9, 6): + raise ValueError( + "Expected HalfCheetah model dimensions (9, 9, 6), " + f"got {(model.nq, model.nv, model.nu)}" + ) + self._model = model + self._state_data = mujoco.MjData(model) + self._rollout_data = mujoco.MjData(model) + self._frame_skip = int(frame_skip) + self._dt = float(model.opt.timestep) * self._frame_skip + + def reset(self, seed: int = 0) -> None: + self.seed = int(seed) + self.phase = 0.0 + self.step = 0 + self.rng = np.random.default_rng(self.seed + 333) + + def _gait(self) -> np.ndarray: + if self.step < FAST_STAGE_STEP: + return START_GAIT + if self.step < FINAL_STAGE_STEP: + return FAST_GAIT + return FINAL_GAIT + + def _sync_observation(self, observation: np.ndarray) -> None: + if self._model is None or self._state_data is None: + raise RuntimeError("configure_simulator must be called before act") + if observation.shape != (17,): + raise ValueError( + f"Expected observation shape (17,), got {observation.shape}" + ) + self._state_data.time = self.step * self._dt + self._state_data.qpos[0] = 0.0 + self._state_data.qpos[1:] = observation[:8] + self._state_data.qvel[:] = observation[8:] + self._state_data.ctrl[:] = 0.0 + mujoco.mj_forward(self._model, self._state_data) + + def act(self, observation: np.ndarray) -> np.ndarray: + observation = np.asarray(observation, dtype=np.float64) + self._sync_observation(observation) + assert self._model is not None + assert self._state_data is not None + assert self._rollout_data is not None + + gait = self._gait() + base_action, next_phase = _single_cpg_action( + gait, + observation, + self.phase, + self._dt, + ) + first_candidates = _candidate_actions( + np.clip(base_action, -1.0, 1.0), + self.rng, + include_random=True, + ) + initial_state = np.repeat( + _model_state(self._state_data)[None, :], + len(first_candidates), + axis=0, + ) + first_scores = _score_actions( + gait=gait, + model=self._model, + rollout_data=self._rollout_data, + initial_state=initial_state, + phase=np.full( + len(first_candidates), + self.phase, + dtype=np.float64, + ), + first_actions=first_candidates, + frame_skip=self._frame_skip, + dt=self._dt, + ) + top_indices = np.argsort(first_scores)[-TREE_WIDTH:] + top_first_actions = first_candidates[top_indices] + + top_count = len(top_first_actions) + top_initial_state = np.repeat( + _model_state(self._state_data)[None, :], + top_count, + axis=0, + ) + second_state, first_reward = _step_batch( + self._model, + self._rollout_data, + top_initial_state, + top_first_actions, + self._frame_skip, + self._dt, + ) + second_phase = np.full( + top_count, + next_phase, + dtype=np.float64, + ) + repeated_gait = np.repeat(gait[None, :], top_count, axis=0) + second_base_actions, _ = _cpg_actions( + repeated_gait, + _observations_from_states(self._model, second_state), + second_phase.copy(), + self._dt, + ) + + second_action_sets = [ + _candidate_actions( + second_base_actions[branch].astype(np.float64), + self.rng, + include_random=False, + ) + for branch in range(top_count) + ] + branch_indices = np.concatenate( + [ + np.full(len(actions), branch, dtype=np.int64) + for branch, actions in enumerate(second_action_sets) + ] + ) + second_actions = np.concatenate(second_action_sets, axis=0) + second_initial_state = np.concatenate( + [ + np.repeat( + second_state[branch : branch + 1], + len(actions), + axis=0, + ) + for branch, actions in enumerate(second_action_sets) + ], + axis=0, + ) + second_initial_phase = np.concatenate( + [ + np.full( + len(actions), + second_phase[branch], + dtype=np.float64, + ) + for branch, actions in enumerate(second_action_sets) + ] + ) + tree_scores = first_reward[branch_indices] + _score_actions( + gait=gait, + model=self._model, + rollout_data=self._rollout_data, + initial_state=second_initial_state, + phase=second_initial_phase, + first_actions=second_actions, + frame_skip=self._frame_skip, + dt=self._dt, + ) + + best_branch = branch_indices[int(np.argmax(tree_scores))] + action = top_first_actions[best_branch] + self.phase = next_phase + self.step += 1 + return np.clip(action, -1.0, 1.0) diff --git a/tasks/simulation_heuristics_halfcheetah_v1/oracle/solve.sh b/tasks/simulation_heuristics_halfcheetah_v1/oracle/solve.sh new file mode 100755 index 0000000..1af9fd8 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/oracle/solve.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /app/final_policy +cp /oracle/policy.py /app/final_policy/policy.py diff --git a/tasks/simulation_heuristics_halfcheetah_v1/starter_policy/policy.py b/tasks/simulation_heuristics_halfcheetah_v1/starter_policy/policy.py new file mode 100644 index 0000000..13d1e73 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/starter_policy/policy.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Asymmetric two-rate CPG with joint-space PD feedback.""" + +from __future__ import annotations + +import math + +import numpy as np + + +GAIT = np.asarray( + [ + 2.0720429776207037, + 2.0890962589603244, + 0.7454172634828156, + -0.06577692937619559, + 0.6454089230776165, + -0.6746024815060896, + -0.5255276137795621, + -0.776911900431121, + 0.4784639889085752, + -1.5344226994990133, + 0.8642039688512475, + -0.6508749004237321, + -0.8139980805865409, + -0.43273458554686073, + 0.9311256504045836, + -0.3884941211535119, + 0.5058403771772146, + 0.29329785944557646, + -0.41630444646878295, + -0.17793318904003722, + 0.25287195521722133, + 0.1864602367775759, + -0.2174643453198326, + 0.32053306369231493, + 0.18857020059058224, + 0.2877837436012836, + -0.46494131953713835, + -0.23362679675599532, + -0.179250145837136, + -0.2538333727428432, + -0.19506084586961667, + -0.3065628153040692, + ], + dtype=np.float64, +) +DT = 0.05 +PHASE_SUBSTEPS = 10 +KP = 1.0 +KD = 0.02 + + +def _frequency(raw_value: np.ndarray) -> np.ndarray: + sigmoid = 1.0 / (1.0 + np.exp(-raw_value)) + return 0.5 + 4.5 * sigmoid + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = int(seed) + self.phase = 0.0 + + def act(self, observation: np.ndarray) -> np.ndarray: + observation = np.asarray(observation, dtype=np.float64) + stance_frequency = _frequency(GAIT[0:1]) + swing_frequency = _frequency(GAIT[1:2]) + phase = np.asarray([self.phase], dtype=np.float64) + + for _ in range(PHASE_SUBSTEPS): + frequency = np.where( + np.sin(phase) > 0.0, + swing_frequency, + stance_frequency, + ) + phase = (phase + 2.0 * math.pi * frequency * (DT / PHASE_SUBSTEPS)) % ( + 2.0 * math.pi + ) + self.phase = float(phase[0]) + + features = np.asarray( + [ + 1.0, + math.sin(self.phase), + math.cos(self.phase), + math.sin(2.0 * self.phase), + math.cos(2.0 * self.phase), + ], + dtype=np.float64, + ) + target = features @ GAIT[2:].reshape(5, 6) + action = KP * (target - observation[2:8]) - KD * observation[11:17] + return np.clip(action, -1.0, 1.0) diff --git a/tasks/simulation_heuristics_halfcheetah_v1/task.md b/tasks/simulation_heuristics_halfcheetah_v1/task.md new file mode 100644 index 0000000..ad935f1 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/task.md @@ -0,0 +1,188 @@ +--- +schema_version: "1.3" +task: + name: genesisbench/simulation_heuristics_halfcheetah_v1 + description: Improve an interpretable HalfCheetah controller under a fixed autonomous research budget. + authors: + - name: GenesisBench Contributors + keywords: + - simulation + - heuristics + - halfcheetah + - mujoco + - model-predictive-control +metadata: + category: locomotion + difficulty: hard + tags: + - mujoco + - continuous-control + - policy-search + - model-predictive-control + - robotics + reference_task: false + genesisbench: + starter: + path: starter_policy + submission: + directory: final_policy + entrypoint: policy.py + development: + episodes: 3 + max_steps: 300 + seeds: [100, 101, 102] + verifier: + reproduction_config: verifier/config.toml + anchors: verifier/anchors.json + supports_private_config: true +agent: + timeout_sec: 3600 + user: agent + network_mode: public +verifier: + timeout_sec: 4200 + user: root + network_mode: no-network +environment: + build_timeout_sec: 1200 + cpus: 4 + memory_mb: 4096 + storage_mb: 10240 + network_mode: public + allow_internet: true + workdir: /app +benchflow: + document_version: "0.6" +--- +# Improve the HalfCheetah locomotion policy + +You are given a strong, interpretable programmatic controller for Gymnasium +`HalfCheetah-v5`. Improve it as much as possible through autonomous research, +coding, and experimentation. The policy must remain a code-based controller: +do not train or embed a neural network. + +## Objective + +Maximize the final HalfCheetah episode return: + +```text +x velocity - 0.1 * sum(action ** 2) +``` + +The environment has: + +- a 17-dimensional observation; +- six continuous actions in `[-1, 1]`; +- a maximum of 1,000 steps per episode; +- no health termination in the standard task. + +## Starting point + +`starter_policy/policy.py` contains the article's non-MPC baseline: a +two-rate asymmetric central pattern generator whose Fourier joint targets are +tracked by a PD controller. The published artifact reported about `4,799.7` +mean return over seeds `100..109` for this controller family. + +The reference result adds online staged-tree MPC around that interpretable +gait. The article's five-episode rerun on seeds `100..104` reported: + +```text +mean 11836.693, min 11735.0, max 12041.2 +``` + +The trusted oracle reimplements that controller: top-K two-level action-tree +search, a 14-step closed-loop CPG/PD tail, and a swing-amplitude schedule that +changes at steps 300 and 900. + +## Development evaluation + +Run: + +```bash +python evaluate.py --policy final_policy/policy.py +``` + +For a quick iteration: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --episodes 1 \ + --max-steps 100 +``` + +For the full published five-seed reproduction: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --episodes 5 \ + --max-steps 1000 \ + --seed 100 +``` + +The full staged-tree reference is intentionally compute-intensive. Internal +MuJoCo planning rollouts do not count as external episode steps, but they do +consume your wall-clock budget and appear in action-latency metrics. + +The public evaluator uses visible development seeds. The final evaluator uses +unseen seeds and conservative unseen dynamics variants. A policy that memorizes +the development episodes is unlikely to score well. + +## Required submission + +Store your best policy at: + +```text +final_policy/policy.py +``` + +It must define either: + +```python +class Policy: + def __init__(self, seed: int = 0): ... + def reset(self, seed: int = 0) -> None: ... + def act(self, observation) -> numpy.ndarray: ... +``` + +or: + +```python +def make_policy(seed: int = 0): ... +``` + +The returned object must implement `reset` and `act`. + +Each action must be a finite NumPy-compatible array with shape `(6,)`. Values +outside `[-1, 1]` are clipped. + +For model-based planning, a policy may optionally implement: + +```python +def configure_simulator( + self, + *, + model_xml_path: str, + frame_skip: int, +) -> None: ... +``` + +The evaluator calls this once per episode with a read-only path to the +episode's MuJoCo model. It does not expose the live environment, reward, or +hidden evaluation configuration. + +## Rules + +1. Work autonomously. Do not ask for user feedback. +2. You have a fixed wall-clock budget. Use it for iterative improvement. +3. Keep the final controller interpretable and non-neural. +4. Do not modify `evaluate.py` or trusted runtime files. +5. Do not access `/oracle`, `/verifier`, or reconstruct hidden seeds or + dynamics configuration. +6. Do not copy a completed HalfCheetah solution from the internet. +7. The final policy receives observations, its reset seed, and the optional + simulator-model configuration hook only. +8. The final policy must run offline with the packages already installed. +9. Before finishing, evaluate `final_policy/policy.py` and leave the best + working version in place. diff --git a/tasks/simulation_heuristics_halfcheetah_v1/task_context/article_context.md b/tasks/simulation_heuristics_halfcheetah_v1/task_context/article_context.md new file mode 100644 index 0000000..b57414b --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/task_context/article_context.md @@ -0,0 +1,34 @@ +# Article experiment context + +This task adapts the MuJoCo HalfCheetah experiment published with *Learning +Beyond Gradients* (Jiayi Weng, May 2026). + +The source artifact describes an interpretable progression: + +1. periodic Fourier torque rules; +2. proprioceptive reflexes; +3. a symmetric PD/CPG target-angle gait; +4. an asymmetric two-rate PD/CPG gait; +5. one-step model-predictive action selection; +6. a two-level top-K action tree; +7. staged swing-amplitude target schedules. + +The final named policy, +`mpc-staged-tree-asym-pd-cpg`, uses a 14-step closed-loop CPG/PD tail, a +terminal velocity term, top-K width 8, high-gain undamped PD tracking inside +the planner, and target schedules: + +```text +steps 0..299: harmonic amplitude 1.15, front lower-leg bias 0.15 +steps 300..899: harmonic amplitude 1.18, front lower-leg bias 0.20 +steps 900..999: harmonic amplitude 1.15, front lower-leg bias 0.15 +``` + +The article reports five seeds `100..104` with mean return `11836.693`, minimum +`11735.0`, and maximum `12041.2`. + +No upstream source file is vendored into the public workspace. The GenesisBench +implementation is a policy-API adaptation under this repository's GPL-3.0 +license. Numeric controller parameters are retained as experiment data needed +to reproduce the published result. Detailed provenance is kept outside the +agent workspace under `evidence/`. diff --git a/tasks/simulation_heuristics_halfcheetah_v1/task_context/evaluation.md b/tasks/simulation_heuristics_halfcheetah_v1/task_context/evaluation.md new file mode 100644 index 0000000..cafb74c --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/task_context/evaluation.md @@ -0,0 +1,51 @@ +# Evaluation + +Per-step reward: + +```text +reward = x_velocity - 0.1 * sum(action ** 2) +``` + +Standard `HalfCheetah-v5` runs for 1,000 steps without health termination. +The public evaluator reports: + +- mean, minimum, and maximum return; +- mean final x-position; +- mean accumulated forward reward and control cost; +- invalid-policy rate; +- mean action latency; +- per-episode metrics. + +Actions are clipped to `[-1, 1]` and quantized to `float32`, matching the +published experiment's execution path. + +## Public and hidden suites + +The public development defaults are three 300-step episodes on seeds +`100..102`. Use `--max-steps 1000` for full episodes. + +The checked-in hidden reproducibility suite runs full 1,000-step episodes on +unseen nominal seeds and on conservative mass, friction, damping, and actuator +variants. Its raw score is: + +```text +0.70 * hidden nominal mean return ++ 0.30 * hidden dynamics-robustness mean return +``` + +The score is normalized on the evaluation machine so the public asymmetric +CPG/PD starter maps to `0` and the trusted staged-tree reference maps to `100`. +Scores above `100` are valid. + +The checked-in suite is reproducible, not secret. An official leaderboard +should inject a private config and matching private anchors. + +## Planning cost + +Calls made by a policy inside its own copied MuJoCo model are internal planning +rollouts. They do not increment the external episode-step count. They do count +against wall-clock limits and are reflected in mean action latency. + +The trusted article controller evaluates hundreds of first actions plus a +second tree level at every external step, so a full five-seed reproduction can +take tens of minutes on a CPU. diff --git a/tasks/simulation_heuristics_halfcheetah_v1/task_context/policy_api.md b/tasks/simulation_heuristics_halfcheetah_v1/task_context/policy_api.md new file mode 100644 index 0000000..2887648 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/task_context/policy_api.md @@ -0,0 +1,65 @@ +# Policy API + +The evaluator imports `final_policy/policy.py` in a clean Python process. + +Define `Policy` or `make_policy`. The policy is instantiated once per episode +and reset before the first action. + +```python +class Policy: + def __init__(self, seed: int = 0): + ... + + def reset(self, seed: int = 0) -> None: + ... + + def act(self, observation): + ... +``` + +`observation` is a one-dimensional NumPy array with 17 values: + +```text +qpos[1:] # 8 values: root z, torso pitch, six joint angles +qvel[:] # 9 values: root x/z velocity, pitch rate, six joint velocities +``` + +The global x-position is omitted. The six actions are ordered: + +```text +bthigh, bshin, bfoot, fthigh, fshin, ffoot +``` + +Each action must have shape `(6,)`. The policy may keep recurrent state and +may load files stored below `final_policy/`. + +## Optional simulator configuration + +Model-based policies may define: + +```python +def configure_simulator( + self, + *, + model_xml_path: str, + frame_skip: int, +) -> None: + ... +``` + +Call order is: + +```text +instantiate policy +configure_simulator(...) # when implemented +reset(seed=...) +act(observation) repeated for the episode +``` + +`model_xml_path` identifies the dynamics used by that episode, including a +hidden robustness variant when applicable. The policy receives no live +`gymnasium.Env`, `MjData`, reward, or `info` object. + +HalfCheetah's omitted x-position is dynamically translation-invariant, so a +planner can reconstruct a local `MjData` state by setting x to any convenient +origin and copying the 17 observed values into the remaining qpos/qvel fields. diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchor_policies/reference_policy.py b/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchor_policies/reference_policy.py new file mode 100644 index 0000000..efa70c0 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchor_policies/reference_policy.py @@ -0,0 +1,455 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Staged asymmetric CPG/PD controller with two-level online tree MPC.""" + +from __future__ import annotations + +import math + +import mujoco +import mujoco.rollout +import numpy as np + + +BASE_GAIT = np.asarray( + [ + 2.0720429776207037, + 2.0890962589603244, + 0.7454172634828156, + -0.06577692937619559, + 0.6454089230776165, + -0.6746024815060896, + -0.5255276137795621, + -0.776911900431121, + 0.4784639889085752, + -1.5344226994990133, + 0.8642039688512475, + -0.6508749004237321, + -0.8139980805865409, + -0.43273458554686073, + 0.9311256504045836, + -0.3884941211535119, + 0.5058403771772146, + 0.29329785944557646, + -0.41630444646878295, + -0.17793318904003722, + 0.25287195521722133, + 0.1864602367775759, + -0.2174643453198326, + 0.32053306369231493, + 0.18857020059058224, + 0.2877837436012836, + -0.46494131953713835, + -0.23362679675599532, + -0.179250145837136, + -0.2538333727428432, + -0.19506084586961667, + -0.3065628153040692, + ], + dtype=np.float64, +) +PHASE_SUBSTEPS = 10 +PLANNING_HORIZON = 14 +TERMINAL_VELOCITY_WEIGHT = 0.5 +PD_KP = 8.0 +PD_KD = 0.0 +TREE_WIDTH = 8 +FAST_STAGE_STEP = 300 +FINAL_STAGE_STEP = 900 +COORDINATE_DELTAS = ( + -0.10, + 0.10, + -0.25, + 0.25, + -0.50, + 0.50, + -0.75, + 0.75, + -1.00, + 1.00, +) +RANDOM_BLOCKS = ((0.15, 64), (0.35, 128), (0.70, 192)) +BANG_BANG_ACTIONS = np.asarray( + np.meshgrid(*[[-1.0, 1.0]] * 6), dtype=np.float64 +).T.reshape(-1, 6) + + +def _frequency(raw_value: np.ndarray) -> np.ndarray: + sigmoid = 1.0 / (1.0 + np.exp(-raw_value)) + return 0.5 + 4.5 * sigmoid + + +def _stage_gait(amplitude: float, front_lower_leg_bias: float) -> np.ndarray: + gait = BASE_GAIT.copy() + gait[0] -= 1.80 + gait[1] -= 1.10 + coefficients = gait[2:].reshape(5, 6).copy() + coefficients[1:] *= amplitude + coefficients[0, 4:] += front_lower_leg_bias + gait[2:] = coefficients.reshape(-1) + return gait + + +START_GAIT = _stage_gait(1.15, 0.15) +FAST_GAIT = _stage_gait(1.18, 0.20) +FINAL_GAIT = START_GAIT.copy() + + +def _advance_phase( + gait: np.ndarray, + phase: np.ndarray, + dt: float, +) -> np.ndarray: + stance_frequency = _frequency(gait[:, 0]) + swing_frequency = _frequency(gait[:, 1]) + updated = phase.copy() + for _ in range(PHASE_SUBSTEPS): + frequency = np.where( + np.sin(updated) > 0.0, + swing_frequency, + stance_frequency, + ) + updated = (updated + 2.0 * math.pi * frequency * (dt / PHASE_SUBSTEPS)) % ( + 2.0 * math.pi + ) + return updated + + +def _cpg_actions( + gait: np.ndarray, + observation: np.ndarray, + phase: np.ndarray, + dt: float, +) -> tuple[np.ndarray, np.ndarray]: + next_phase = _advance_phase(gait, phase, dt) + features = np.stack( + ( + np.ones_like(next_phase), + np.sin(next_phase), + np.cos(next_phase), + np.sin(2.0 * next_phase), + np.cos(2.0 * next_phase), + ), + axis=1, + ) + target = np.einsum( + "nf,nfa->na", + features, + gait[:, 2:].reshape(-1, 5, 6), + ) + action = PD_KP * (target - observation[:, 2:8]) - PD_KD * observation[:, 11:17] + return np.clip(action, -1.0, 1.0).astype(np.float32), next_phase + + +def _single_cpg_action( + gait: np.ndarray, + observation: np.ndarray, + phase: float, + dt: float, +) -> tuple[np.ndarray, float]: + next_phase = _advance_phase( + gait[None, :], + np.asarray([phase], dtype=np.float64), + dt, + ) + phase_value = float(next_phase[0]) + features = np.asarray( + [ + 1.0, + math.sin(phase_value), + math.cos(phase_value), + math.sin(2.0 * phase_value), + math.cos(2.0 * phase_value), + ], + dtype=np.float64, + ) + target = features @ gait[2:].reshape(5, 6) + action = PD_KP * (target - observation[2:8]) - PD_KD * observation[11:17] + return action, phase_value + + +def _model_state(data: mujoco.MjData) -> np.ndarray: + return np.concatenate(([data.time], data.qpos.copy(), data.qvel.copy())).astype( + np.float64 + ) + + +def _observations_from_states( + model: mujoco.MjModel, + state: np.ndarray, +) -> np.ndarray: + qpos_start = 1 + qvel_start = qpos_start + model.nq + qpos = state[:, qpos_start:qvel_start] + qvel = state[:, qvel_start : qvel_start + model.nv] + return np.concatenate((qpos[:, 1:], qvel), axis=1) + + +def _step_batch( + model: mujoco.MjModel, + rollout_data: mujoco.MjData, + state: np.ndarray, + action: np.ndarray, + frame_skip: int, + dt: float, +) -> tuple[np.ndarray, np.ndarray]: + controls = np.repeat( + action[:, None, :], + frame_skip, + axis=1, + ).astype(np.float64) + states, _ = mujoco.rollout.rollout( + model, + rollout_data, + initial_state=state, + control=controls, + ) + next_state = states[:, -1, :] + forward_reward = (next_state[:, 1] - state[:, 1]) / dt + control_cost = 0.1 * np.sum(action * action, axis=1) + return next_state, forward_reward - control_cost + + +def _score_actions( + *, + gait: np.ndarray, + model: mujoco.MjModel, + rollout_data: mujoco.MjData, + initial_state: np.ndarray, + phase: np.ndarray, + first_actions: np.ndarray, + frame_skip: int, + dt: float, +) -> np.ndarray: + count = len(first_actions) + repeated_gait = np.repeat(gait[None, :], count, axis=0) + simulated_phase = _advance_phase(repeated_gait, phase, dt) + state, score = _step_batch( + model, + rollout_data, + initial_state, + np.clip(first_actions, -1.0, 1.0), + frame_skip, + dt, + ) + + for _ in range(1, PLANNING_HORIZON): + action, simulated_phase = _cpg_actions( + repeated_gait, + _observations_from_states(model, state), + simulated_phase, + dt, + ) + state, reward = _step_batch( + model, + rollout_data, + state, + action, + frame_skip, + dt, + ) + score += reward + + qvel_start = 1 + model.nq + return score + TERMINAL_VELOCITY_WEIGHT * state[:, qvel_start] + + +def _candidate_actions( + base_action: np.ndarray, + rng: np.random.Generator, + *, + include_random: bool, +) -> np.ndarray: + candidates = [base_action] + for joint in range(6): + for delta in COORDINATE_DELTAS: + candidate = base_action.copy() + candidate[joint] = np.clip( + candidate[joint] + delta, + -1.0, + 1.0, + ) + candidates.append(candidate) + + if include_random: + for standard_deviation, count in RANDOM_BLOCKS: + samples = base_action + standard_deviation * rng.standard_normal((count, 6)) + candidates.extend(np.clip(samples, -1.0, 1.0)) + + candidates.extend(BANG_BANG_ACTIONS) + return np.asarray(candidates, dtype=np.float64) + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self._model: mujoco.MjModel | None = None + self._state_data: mujoco.MjData | None = None + self._rollout_data: mujoco.MjData | None = None + self._frame_skip = 5 + self._dt = 0.05 + self.reset(seed) + + def configure_simulator( + self, + *, + model_xml_path: str, + frame_skip: int = 5, + ) -> None: + model = mujoco.MjModel.from_xml_path(model_xml_path) + if (model.nq, model.nv, model.nu) != (9, 9, 6): + raise ValueError( + "Expected HalfCheetah model dimensions (9, 9, 6), " + f"got {(model.nq, model.nv, model.nu)}" + ) + self._model = model + self._state_data = mujoco.MjData(model) + self._rollout_data = mujoco.MjData(model) + self._frame_skip = int(frame_skip) + self._dt = float(model.opt.timestep) * self._frame_skip + + def reset(self, seed: int = 0) -> None: + self.seed = int(seed) + self.phase = 0.0 + self.step = 0 + self.rng = np.random.default_rng(self.seed + 333) + + def _gait(self) -> np.ndarray: + if self.step < FAST_STAGE_STEP: + return START_GAIT + if self.step < FINAL_STAGE_STEP: + return FAST_GAIT + return FINAL_GAIT + + def _sync_observation(self, observation: np.ndarray) -> None: + if self._model is None or self._state_data is None: + raise RuntimeError("configure_simulator must be called before act") + if observation.shape != (17,): + raise ValueError( + f"Expected observation shape (17,), got {observation.shape}" + ) + self._state_data.time = self.step * self._dt + self._state_data.qpos[0] = 0.0 + self._state_data.qpos[1:] = observation[:8] + self._state_data.qvel[:] = observation[8:] + self._state_data.ctrl[:] = 0.0 + mujoco.mj_forward(self._model, self._state_data) + + def act(self, observation: np.ndarray) -> np.ndarray: + observation = np.asarray(observation, dtype=np.float64) + self._sync_observation(observation) + assert self._model is not None + assert self._state_data is not None + assert self._rollout_data is not None + + gait = self._gait() + base_action, next_phase = _single_cpg_action( + gait, + observation, + self.phase, + self._dt, + ) + first_candidates = _candidate_actions( + np.clip(base_action, -1.0, 1.0), + self.rng, + include_random=True, + ) + initial_state = np.repeat( + _model_state(self._state_data)[None, :], + len(first_candidates), + axis=0, + ) + first_scores = _score_actions( + gait=gait, + model=self._model, + rollout_data=self._rollout_data, + initial_state=initial_state, + phase=np.full( + len(first_candidates), + self.phase, + dtype=np.float64, + ), + first_actions=first_candidates, + frame_skip=self._frame_skip, + dt=self._dt, + ) + top_indices = np.argsort(first_scores)[-TREE_WIDTH:] + top_first_actions = first_candidates[top_indices] + + top_count = len(top_first_actions) + top_initial_state = np.repeat( + _model_state(self._state_data)[None, :], + top_count, + axis=0, + ) + second_state, first_reward = _step_batch( + self._model, + self._rollout_data, + top_initial_state, + top_first_actions, + self._frame_skip, + self._dt, + ) + second_phase = np.full( + top_count, + next_phase, + dtype=np.float64, + ) + repeated_gait = np.repeat(gait[None, :], top_count, axis=0) + second_base_actions, _ = _cpg_actions( + repeated_gait, + _observations_from_states(self._model, second_state), + second_phase.copy(), + self._dt, + ) + + second_action_sets = [ + _candidate_actions( + second_base_actions[branch].astype(np.float64), + self.rng, + include_random=False, + ) + for branch in range(top_count) + ] + branch_indices = np.concatenate( + [ + np.full(len(actions), branch, dtype=np.int64) + for branch, actions in enumerate(second_action_sets) + ] + ) + second_actions = np.concatenate(second_action_sets, axis=0) + second_initial_state = np.concatenate( + [ + np.repeat( + second_state[branch : branch + 1], + len(actions), + axis=0, + ) + for branch, actions in enumerate(second_action_sets) + ], + axis=0, + ) + second_initial_phase = np.concatenate( + [ + np.full( + len(actions), + second_phase[branch], + dtype=np.float64, + ) + for branch, actions in enumerate(second_action_sets) + ] + ) + tree_scores = first_reward[branch_indices] + _score_actions( + gait=gait, + model=self._model, + rollout_data=self._rollout_data, + initial_state=second_initial_state, + phase=second_initial_phase, + first_actions=second_actions, + frame_skip=self._frame_skip, + dt=self._dt, + ) + + best_branch = branch_indices[int(np.argmax(tree_scores))] + action = top_first_actions[best_branch] + self.phase = next_phase + self.step += 1 + return np.clip(action, -1.0, 1.0) diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchor_policies/starter_policy.py b/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchor_policies/starter_policy.py new file mode 100644 index 0000000..13d1e73 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchor_policies/starter_policy.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Asymmetric two-rate CPG with joint-space PD feedback.""" + +from __future__ import annotations + +import math + +import numpy as np + + +GAIT = np.asarray( + [ + 2.0720429776207037, + 2.0890962589603244, + 0.7454172634828156, + -0.06577692937619559, + 0.6454089230776165, + -0.6746024815060896, + -0.5255276137795621, + -0.776911900431121, + 0.4784639889085752, + -1.5344226994990133, + 0.8642039688512475, + -0.6508749004237321, + -0.8139980805865409, + -0.43273458554686073, + 0.9311256504045836, + -0.3884941211535119, + 0.5058403771772146, + 0.29329785944557646, + -0.41630444646878295, + -0.17793318904003722, + 0.25287195521722133, + 0.1864602367775759, + -0.2174643453198326, + 0.32053306369231493, + 0.18857020059058224, + 0.2877837436012836, + -0.46494131953713835, + -0.23362679675599532, + -0.179250145837136, + -0.2538333727428432, + -0.19506084586961667, + -0.3065628153040692, + ], + dtype=np.float64, +) +DT = 0.05 +PHASE_SUBSTEPS = 10 +KP = 1.0 +KD = 0.02 + + +def _frequency(raw_value: np.ndarray) -> np.ndarray: + sigmoid = 1.0 / (1.0 + np.exp(-raw_value)) + return 0.5 + 4.5 * sigmoid + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = int(seed) + self.phase = 0.0 + + def act(self, observation: np.ndarray) -> np.ndarray: + observation = np.asarray(observation, dtype=np.float64) + stance_frequency = _frequency(GAIT[0:1]) + swing_frequency = _frequency(GAIT[1:2]) + phase = np.asarray([self.phase], dtype=np.float64) + + for _ in range(PHASE_SUBSTEPS): + frequency = np.where( + np.sin(phase) > 0.0, + swing_frequency, + stance_frequency, + ) + phase = (phase + 2.0 * math.pi * frequency * (DT / PHASE_SUBSTEPS)) % ( + 2.0 * math.pi + ) + self.phase = float(phase[0]) + + features = np.asarray( + [ + 1.0, + math.sin(self.phase), + math.cos(self.phase), + math.sin(2.0 * self.phase), + math.cos(2.0 * self.phase), + ], + dtype=np.float64, + ) + target = features @ GAIT[2:].reshape(5, 6) + action = KP * (target - observation[2:8]) - KD * observation[11:17] + return np.clip(action, -1.0, 1.0) diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchors.json b/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchors.json new file mode 100644 index 0000000..d433d7a --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/anchors.json @@ -0,0 +1,12 @@ +{ + "environment": "Gymnasium HalfCheetah-v5 with 17 observations and optional copied-model planning", + "max_steps": 1000, + "reference_policy": { + "description": "Article staged swing-amplitude top-K8 tree MPC policy", + "path": "anchor_policies/reference_policy.py" + }, + "starter_policy": { + "description": "Public asymmetric two-rate CPG/PD policy", + "path": "anchor_policies/starter_policy.py" + } +} diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/config.toml b/tasks/simulation_heuristics_halfcheetah_v1/verifier/config.toml new file mode 100644 index 0000000..93d9f5d --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/config.toml @@ -0,0 +1,29 @@ +version = "1.0" + +[evaluation] +max_steps = 1000 +failure_return = -1000.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.7 +seeds = [211, 307] + +[[evaluation.suites]] +name = "hidden_robustness" +weight = 0.3 +seeds = [503] + +[[evaluation.variants]] +name = "light_low_friction" +mass_scale = 0.92 +friction_scale = 0.85 +damping_scale = 0.95 +actuator_scale = 1.0 + +[[evaluation.variants]] +name = "heavy_weak_actuators" +mass_scale = 1.08 +friction_scale = 1.10 +damping_scale = 1.05 +actuator_scale = 0.92 diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_halfcheetah_v1/verifier/evaluate_hidden.py new file mode 100755 index 0000000..fd959f7 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/evaluate_hidden.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tomllib +from pathlib import Path + + +VERIFIER_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] +RUNTIME_CANDIDATES.extend(ancestor / "src" for ancestor in VERIFIER_DIR.parents) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.halfcheetah import ( # noqa: E402 + DynamicsVariant, + HalfCheetahEvaluation, + evaluate_halfcheetah_policy, +) + + +EvaluationBundle = tuple[ + HalfCheetahEvaluation, + HalfCheetahEvaluation, + float, +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run hidden HalfCheetah evaluation.") + parser.add_argument("policy", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument( + "--config", + type=Path, + default=Path(__file__).parent / "config.toml", + help="Evaluation suite config; production can inject a private file.", + ) + parser.add_argument( + "--anchors", + type=Path, + default=Path(__file__).parent / "anchors.json", + help="Normalization anchors matching the selected evaluation suite.", + ) + return parser.parse_args() + + +def _policy_path(path: Path) -> Path: + resolved = path.resolve() + return resolved / "policy.py" if resolved.is_dir() else resolved + + +def _policy_fingerprint(path: Path) -> str: + return hashlib.sha256(_policy_path(path).read_bytes()).hexdigest() + + +def _evaluate_raw( + policy: Path, + *, + evaluation: dict, +) -> EvaluationBundle: + suites = {suite["name"]: suite for suite in evaluation["suites"]} + nominal = evaluate_halfcheetah_policy( + policy, + seeds=suites["hidden_nominal"]["seeds"], + max_steps=evaluation["max_steps"], + variants=(DynamicsVariant(),), + failure_return=evaluation["failure_return"], + ) + variants = tuple(DynamicsVariant(**variant) for variant in evaluation["variants"]) + robustness = evaluate_halfcheetah_policy( + policy, + seeds=suites["hidden_robustness"]["seeds"], + max_steps=evaluation["max_steps"], + variants=variants, + failure_return=evaluation["failure_return"], + ) + score = ( + suites["hidden_nominal"]["weight"] * nominal.mean_return + + suites["hidden_robustness"]["weight"] * robustness.mean_return + ) + return nominal, robustness, score + + +def _evaluate_cached( + policy: Path, + *, + evaluation: dict, + cache: dict[str, EvaluationBundle], +) -> EvaluationBundle: + fingerprint = _policy_fingerprint(policy) + if fingerprint not in cache: + cache[fingerprint] = _evaluate_raw( + policy, + evaluation=evaluation, + ) + return cache[fingerprint] + + +def _anchor_score( + anchors: dict, + name: str, + *, + anchors_path: Path, + evaluation: dict, + cache: dict[str, EvaluationBundle], +) -> float: + anchor = anchors[name] + score = anchor.get("score") + if isinstance(score, int | float): + return float(score) + relative_path = anchor.get("path") + if not isinstance(relative_path, str) or not relative_path: + raise ValueError(f"{name} must declare score or path") + policy = anchors_path.parent / relative_path + _, _, calibrated_score = _evaluate_cached( + policy, + evaluation=evaluation, + cache=cache, + ) + return calibrated_score + + +def main() -> None: + args = parse_args() + config = tomllib.loads(args.config.read_text()) + anchors = json.loads(args.anchors.read_text()) + evaluation = config["evaluation"] + suites = {suite["name"]: suite for suite in evaluation["suites"]} + cache: dict[str, EvaluationBundle] = {} + + nominal, robustness, score = _evaluate_cached( + args.policy, + evaluation=evaluation, + cache=cache, + ) + starter_score = _anchor_score( + anchors, + "starter_policy", + anchors_path=args.anchors, + evaluation=evaluation, + cache=cache, + ) + reference_score = _anchor_score( + anchors, + "reference_policy", + anchors_path=args.anchors, + evaluation=evaluation, + cache=cache, + ) + if reference_score == starter_score: + raise ValueError("starter and reference anchors must have different scores") + normalized_score = round( + 100.0 * (score - starter_score) / (reference_score - starter_score), + 9, + ) + payload = { + "score": score, + "normalized_score": normalized_score, + "starter_score": starter_score, + "reference_score": reference_score, + "hidden_nominal_weight": suites["hidden_nominal"]["weight"], + "hidden_robustness_weight": suites["hidden_robustness"]["weight"], + "hidden_nominal": nominal.to_dict(), + "hidden_robustness": robustness.to_dict(), + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_halfcheetah_v1/verifier/rubrics/verifier.md new file mode 100644 index 0000000..c253eff --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/rubrics/verifier.md @@ -0,0 +1,5 @@ +# Simulation Heuristics HalfCheetah v1 Verifier Rubric + +- `task_success`: normalized hidden-suite HalfCheetah return, where the + asymmetric CPG/PD starter maps to `0` and the staged-tree article reference + maps to `1`. diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/rubrics/verifier.toml b/tasks/simulation_heuristics_halfcheetah_v1/verifier/rubrics/verifier.toml new file mode 100644 index 0000000..09523f1 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/rubrics/verifier.toml @@ -0,0 +1,9 @@ +version = "0.1" + +[[criteria]] +id = "task_success" +description = "Normalized hidden-suite HalfCheetah return." +weight = 1.0 + +[scoring] +method = "weighted_mean" diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/test.sh b/tasks/simulation_heuristics_halfcheetah_v1/verifier/test.sh new file mode 100755 index 0000000..bd98ee1 --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/test.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier +policy=/app/final_policy/policy.py + +if [ ! -f "$policy" ]; then + printf '0.0\n' > /logs/verifier/reward.txt + printf '{"reward": 0.0}\n' > /logs/verifier/reward.json + exit 0 +fi + +python /verifier/evaluate_hidden.py \ + "$policy" \ + --config /verifier/config.toml \ + --anchors /verifier/anchors.json \ + --output /logs/verifier/genesis-score.json + +python - <<'PY' +import json +from pathlib import Path + +score = json.loads(Path("/logs/verifier/genesis-score.json").read_text()) +reward = max(0.0, min(1.0, float(score["normalized_score"]) / 100.0)) +Path("/logs/verifier/reward.txt").write_text(f"{reward}\n") +Path("/logs/verifier/reward.json").write_text( + json.dumps({"reward": reward}, indent=2) + "\n" +) +PY diff --git a/tasks/simulation_heuristics_halfcheetah_v1/verifier/verifier.md b/tasks/simulation_heuristics_halfcheetah_v1/verifier/verifier.md new file mode 100644 index 0000000..f09068e --- /dev/null +++ b/tasks/simulation_heuristics_halfcheetah_v1/verifier/verifier.md @@ -0,0 +1,35 @@ +--- +document_version: "0.3" +verifier: + name: simulation_heuristics_halfcheetah_v1 + default_strategy: deterministic + strategies: + deterministic: + type: script + command: ./test.sh + rubric: + combine: weighted_mean + dimensions: + task_success: + weight: 1.0 + source: deterministic + outputs: + reward_text: /logs/verifier/reward.txt + reward_json: /logs/verifier/reward.json + details_json: /logs/verifier/genesis-score.json + aggregate_policy: + method: weighted_mean + metrics: + task_success: 1.0 +--- + +## verifier intent + +Run the submitted HalfCheetah policy on full-horizon hidden nominal seeds and +conservative hidden dynamics variants. BenchFlow's canonical reward is the +normalized GenesisBench score clamped to `[0, 1]`; full raw and normalized +metrics remain available in `genesis-score.json`. + +Normalization evaluates trusted local copies of the starter and staged-tree +reference on the same platform. Byte-identical policies are evaluated once per +verifier invocation so the oracle does not pay the reference MPC cost twice. diff --git a/tasks/simulation_heuristics_montezuma_v1/README.md b/tasks/simulation_heuristics_montezuma_v1/README.md new file mode 100644 index 0000000..8cd054c --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/README.md @@ -0,0 +1,82 @@ +# Simulation Heuristics Montezuma v1 — Boundary Reference Task + +`simulation_heuristics_montezuma_v1` turns the Montezuma boundary experiment +from *Learning Beyond Gradients* into a native GenesisBench task: + +```text +native RGB starter +→ public development evaluation +→ final_policy/policy.py +→ hidden seed/no-op/recovery suites +→ platform-local 0/100 anchors +``` + +## Historical result preserved by the task + +The article reports that an earlier graph search improved key distance +`72 → 28` without earning reward. A later native-image run replayed `400` +points in `1769` environment steps with `86` macro-actions, but was mostly +open-loop. The trusted reference reproduces that boundary result while using +image-based trajectory re-entry to survive the checked-in recovery suite. + +## Score + +Each episode contributes at most `400` raw points: + +```text +raw score = + 0.25 * reproduction + + 0.10 * hidden deterministic seeds + + 0.10 * startup no-op variants + + 0.55 * intermediate-state recovery +``` + +The verifier evaluates the checked-in starter and trusted reference on the +same platform as the submission. They normalize to exactly `0` and `100`. +A plain copied open-loop replay receives reproduction credit but fails the +recovery handoffs and therefore cannot reach the reference anchor. + +## Local commands + +Install EnvPool only for the command being run: + +```bash +uv run --with envpool==1.1.1 \ + python tasks/simulation_heuristics_montezuma_v1/evaluate.py \ + --policy tasks/simulation_heuristics_montezuma_v1/starter_policy/policy.py +``` + +Run the hidden evaluator: + +```bash +uv run --with envpool==1.1.1 \ + python tasks/simulation_heuristics_montezuma_v1/verifier/evaluate_hidden.py \ + tasks/simulation_heuristics_montezuma_v1/oracle/policy.py +``` + +Validate the native task package: + +```bash +uv run bench tasks check \ + tasks/simulation_heuristics_montezuma_v1 \ + --level publication-grade +``` + +Run the oracle through BenchFlow: + +```bash +uv run bench eval run \ + --tasks-dir tasks/simulation_heuristics_montezuma_v1 \ + --agent oracle \ + --sandbox docker \ + --context-root . +``` + +The prepared agent workspace excludes `oracle/`, `verifier/`, and `evidence/`. +Official leaderboard runs should inject a private suite and matching anchors. + +## Provenance + +The evaluator, starter, and recovery implementation are independent +GenesisBench code. The calibration trajectory is attributed in +`task_context/provenance.md`; retain that notice when redistributing the task. diff --git a/tasks/simulation_heuristics_montezuma_v1/environment/Dockerfile b/tasks/simulation_heuristics_montezuma_v1/environment/Dockerfile new file mode 100644 index 0000000..06a73f8 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/environment/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12-slim + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends bash ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir \ + "envpool==1.1.1" \ + "numpy>=1.26,<3" \ + "pyyaml>=6,<7" + +RUN useradd --create-home --uid 1000 agent \ + && mkdir -p /app /opt/genesisbench + +COPY src/genesisbench /opt/genesisbench/genesisbench +COPY tasks/simulation_heuristics_montezuma_v1/evaluate.py /app/evaluate.py +COPY tasks/simulation_heuristics_montezuma_v1/starter_policy /app/starter_policy +COPY tasks/simulation_heuristics_montezuma_v1/task_context /app/task_context + +RUN cp -R /app/starter_policy /app/final_policy \ + && chown -R root:root /opt/genesisbench \ + && chmod -R a-w /opt/genesisbench \ + && chown -R agent:agent /app + +ENV PYTHONPATH="/opt/genesisbench" + +WORKDIR /app + +RUN mkdir -p /logs/verifier /logs/agent /logs/artifacts diff --git a/tasks/simulation_heuristics_montezuma_v1/evaluate.py b/tasks/simulation_heuristics_montezuma_v1/evaluate.py new file mode 100644 index 0000000..7100279 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/evaluate.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +TASK_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = ( + TASK_DIR / "_runtime", + TASK_DIR.parent / "src", + TASK_DIR.parent.parent / "src", +) +for candidate in RUNTIME_CANDIDATES: + if candidate.is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.montezuma import ( # noqa: E402 + MontezumaVariant, + evaluate_montezuma_policy, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate a native-image Montezuma policy." + ) + parser.add_argument( + "--policy", + type=Path, + default=Path("final_policy/policy.py"), + ) + parser.add_argument("--episodes", type=int, default=1) + parser.add_argument("--max-steps", type=int, default=2600) + parser.add_argument("--seed", type=int, default=10001) + parser.add_argument("--initial-noops", type=int, default=0) + parser.add_argument("--bootstrap-steps", type=int, default=0) + parser.add_argument( + "--bootstrap-policy", + type=Path, + help="Trusted development prelude; defaults to --policy.", + ) + parser.add_argument("--json-output-file", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + variant = MontezumaVariant( + name="development", + bootstrap_steps=args.bootstrap_steps, + pre_policy_noops=args.initial_noops, + ) + bootstrap_policy = None + if args.bootstrap_steps: + bootstrap_policy = args.bootstrap_policy or args.policy + result = evaluate_montezuma_policy( + args.policy, + seeds=range(args.seed, args.seed + args.episodes), + max_steps=args.max_steps, + variants=(variant,), + bootstrap_policy_path=bootstrap_policy, + ) + rendered = result.to_json() + print(rendered) + if args.json_output_file is not None: + args.json_output_file.parent.mkdir(parents=True, exist_ok=True) + args.json_output_file.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_montezuma_v1/oracle/policy.py b/tasks/simulation_heuristics_montezuma_v1/oracle/policy.py new file mode 100644 index 0000000..3278d8a --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/oracle/policy.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Trusted image-synchronized reference for the 400-point boundary.""" + +from __future__ import annotations + +from collections import defaultdict +import hashlib +from pathlib import Path + +import numpy as np + + +DATA_PATH = Path(__file__).with_name("reference_trajectory.npz") +with np.load(DATA_PATH, allow_pickle=False) as trajectory: + ACTIONS = np.asarray(trajectory["actions"], dtype=np.int64) + FRAME_HASHES = np.asarray(trajectory["hashes"], dtype=np.uint64) + FEATURES = np.asarray(trajectory["features"], dtype=np.uint8) +FEATURES_I16 = FEATURES.astype(np.int16) + +HASH_TO_INDICES: dict[int, tuple[int, ...]] +_hash_to_indices: defaultdict[int, list[int]] = defaultdict(list) +for _index, _frame_hash in enumerate(FRAME_HASHES): + _hash_to_indices[int(_frame_hash)].append(_index) +HASH_TO_INDICES = { + frame_hash: tuple(indices) for frame_hash, indices in _hash_to_indices.items() +} + + +def _frame(observation: np.ndarray) -> np.ndarray: + frame = np.asarray(observation, dtype=np.uint8) + if frame.ndim != 3: + raise ValueError(f"Expected one RGB image, got shape {frame.shape}") + return frame + + +def _frame_hash(frame: np.ndarray) -> int: + digest = hashlib.blake2b(frame.tobytes(), digest_size=8).digest() + return int.from_bytes(digest, byteorder="little", signed=False) + + +def _feature(frame: np.ndarray) -> np.ndarray: + if frame.shape[0] in (1, 3, 4): + frame = np.moveaxis(frame, 0, -1) + rgb = frame[..., :3].astype(np.uint16) + grayscale = (77 * rgb[..., 0] + 150 * rgb[..., 1] + 29 * rgb[..., 2]) >> 8 + return grayscale.astype(np.uint8)[::7, ::7] + + +class Policy: + """Replay through image-state lookup, including fresh-policy re-entry.""" + + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.cursor = 0 + self.started = False + + def _nearest_index(self, frame: np.ndarray) -> int: + indices = HASH_TO_INDICES.get(_frame_hash(frame)) + if indices: + return min( + indices, + key=lambda index: (abs(index - self.cursor), index), + ) + + feature = _feature(frame).astype(np.int16) + if self.started: + lower = max(0, self.cursor - 128) + upper = min(len(ACTIONS), self.cursor + 129) + else: + lower = 0 + upper = len(ACTIONS) + distances = np.mean( + np.abs(FEATURES_I16[lower:upper] - feature), + axis=(1, 2), + ) + return lower + int(np.argmin(distances)) + + def act(self, observation: np.ndarray) -> int: + if self.cursor >= len(ACTIONS): + return 0 + index = self._nearest_index(_frame(observation)) + self.started = True + self.cursor = min(index + 1, len(ACTIONS)) + return int(ACTIONS[index]) diff --git a/tasks/simulation_heuristics_montezuma_v1/oracle/reference_trajectory.npz b/tasks/simulation_heuristics_montezuma_v1/oracle/reference_trajectory.npz new file mode 100644 index 0000000..f8d5191 Binary files /dev/null and b/tasks/simulation_heuristics_montezuma_v1/oracle/reference_trajectory.npz differ diff --git a/tasks/simulation_heuristics_montezuma_v1/oracle/solve.sh b/tasks/simulation_heuristics_montezuma_v1/oracle/solve.sh new file mode 100755 index 0000000..e43408c --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/oracle/solve.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /app/final_policy +cp /oracle/policy.py /app/final_policy/policy.py +cp /oracle/reference_trajectory.npz /app/final_policy/reference_trajectory.npz diff --git a/tasks/simulation_heuristics_montezuma_v1/starter_policy/policy.py b/tasks/simulation_heuristics_montezuma_v1/starter_policy/policy.py new file mode 100644 index 0000000..dd5d5a4 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/starter_policy/policy.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Weak native-image explorer for Montezuma's Revenge.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +RIGHTFIRE = 11 +LEFTFIRE = 12 + + +def _as_hwc(observation: np.ndarray) -> np.ndarray: + frame = np.asarray(observation, dtype=np.uint8) + if frame.ndim != 3: + raise ValueError(f"Expected an RGB image, got shape {frame.shape}") + if frame.shape[0] in (1, 3, 4): + frame = np.moveaxis(frame, 0, -1) + if frame.shape[-1] < 3: + raise ValueError(f"Expected RGB channels, got shape {frame.shape}") + return frame[..., :3] + + +class Policy: + """A valid but intentionally weak pixel-reactive starting point.""" + + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.step = 0 + + def act(self, observation: np.ndarray) -> int: + frame = _as_hwc(observation) + red = frame[..., 0].astype(np.int16) + green = frame[..., 1].astype(np.int16) + blue = frame[..., 2].astype(np.int16) + warm_pixels = (red > 120) & (red > green + 30) & (red > blue + 30) + + self.step += 1 + if self.step <= 20: + return NOOP + + locations = np.argwhere(warm_pixels) + if locations.size: + horizontal_center = float(np.mean(locations[:, 1])) + return RIGHTFIRE if horizontal_center < frame.shape[1] / 2 else LEFTFIRE + + return RIGHTFIRE if (self.step // 48) % 2 == 0 else LEFTFIRE diff --git a/tasks/simulation_heuristics_montezuma_v1/task.md b/tasks/simulation_heuristics_montezuma_v1/task.md new file mode 100644 index 0000000..7fac50a --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/task.md @@ -0,0 +1,146 @@ +--- +schema_version: "1.3" +task: + name: genesisbench/simulation_heuristics_montezuma_v1 + description: Improve a native-image Montezuma policy beyond a brittle open-loop replay. + authors: + - name: GenesisBench Contributors + keywords: + - simulation + - heuristics + - atari + - montezuma + - visual-control + - long-horizon-planning +metadata: + category: long-horizon-control + difficulty: hard + tags: + - atari + - native-image + - macro-actions + - recovery + - policy-search + reference_task: true + genesisbench: + starter: + path: starter_policy + submission: + directory: final_policy + entrypoint: policy.py + development: + episodes: 1 + max_steps: 2600 + seeds: [10001] + verifier: + reproduction_config: verifier/config.toml + anchors: verifier/anchors.json + supports_private_config: true +agent: + timeout_sec: 1800 + user: agent + network_mode: public +verifier: + timeout_sec: 600 + user: root + network_mode: no-network +environment: + build_timeout_sec: 1200 + cpus: 4 + memory_mb: 4096 + storage_mb: 10240 + network_mode: public + allow_internet: true + workdir: /app +benchflow: + document_version: "0.6" +--- +# Improve a recoverable Montezuma's Revenge policy + +You are given a valid but weak programmatic controller for EnvPool +`MontezumaRevenge-v5`. Improve it through autonomous research, coding, and +experimentation using only the native RGB observation. + +## Why this is a boundary task + +The published experiment behind this task documented two complementary facts: + +- an earlier state-graph search reduced the reported key distance from `72` to + `28`, while the game score remained `0`; +- a later unattended native-image run produced a real `400`-point replay using + `86` macro-actions and `1769` environment steps, but the route was mostly + open-loop. + +The goal is not merely to paste that timeline. Build a policy that can reproduce +the first `400` points and re-enter its plan after timing shifts or from an +intermediate visual state. + +## Development evaluation + +Run: + +```bash +python evaluate.py --policy final_policy/policy.py +``` + +Test a fresh-policy handoff after your own policy has reached an intermediate +state: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --bootstrap-steps 512 \ + --initial-noops 4 +``` + +The public evaluator exposes development seeds and perturbations. Final scoring +uses unseen seeds and a private mix of startup no-ops and recovery checkpoints. + +## Required submission + +Store the best policy at: + +```text +final_policy/policy.py +``` + +It must define `Policy` or `make_policy`: + +```python +class Policy: + def __init__(self, seed: int = 0): ... + def reset(self, seed: int = 0) -> None: ... + def act(self, observation) -> int: ... +``` + +`observation` is one native `uint8` RGB frame with shape `(3, 210, 160)`. +`act` must return one integer Atari action in `[0, 17]`. The policy may keep +recurrent state and load files stored below `final_policy/`. + +## Evaluation objective + +Each episode is capped at the boundary target of `400` points. Final scoring +combines: + +1. direct reproduction from reset; +2. unseen deterministic seeds; +3. controlled no-op timing variants; +4. fresh-policy recovery from hidden intermediate checkpoints. + +A copied action counter can receive credit for reproduction but cannot receive +full credit without using the image to identify and recover state. + +## Rules + +1. Work autonomously within the fixed wall-clock budget. +2. Do not modify `evaluate.py`, `_runtime/`, or other trusted task files. +3. Do not access `/oracle`, `/verifier`, hidden configurations, or hidden + trajectory data. +4. The policy receives native RGB images only. Do not use RAM, ALE state, + emulator objects, reward, `info`, lives, room identifiers, or hidden timing. +5. Do not fetch or copy a completed Montezuma solution. Research ideas are + allowed, but the submitted artifact must be your own implementation. +6. The final policy must run offline with the packages installed in the task + image. +7. Before finishing, evaluate `final_policy/policy.py` and leave the best + working artifact in place. diff --git a/tasks/simulation_heuristics_montezuma_v1/task_context/README.md b/tasks/simulation_heuristics_montezuma_v1/task_context/README.md new file mode 100644 index 0000000..2d04a64 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/task_context/README.md @@ -0,0 +1,10 @@ +# Task Context + +The stable public contract is split across: + +- `policy_api.md`: native RGB and action interface; +- `evaluation.md`: public and hidden score semantics; +- `provenance.md`: article history, source attribution, and licensing caveat. + +No RAM, ALE state, reward, `info`, lives, room identifiers, or hidden variant +metadata are supplied to the policy. diff --git a/tasks/simulation_heuristics_montezuma_v1/task_context/evaluation.md b/tasks/simulation_heuristics_montezuma_v1/task_context/evaluation.md new file mode 100644 index 0000000..d5c2ad4 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/task_context/evaluation.md @@ -0,0 +1,44 @@ +# Evaluation + +The runtime is pinned to EnvPool `1.1.1` with: + +```text +MontezumaRevenge-v5 +native 210x160 RGB +frame_skip = 1 +noop_max = 1 +use_fire_reset = true +episodic_life = false +reward_clip = false +repeat_action_probability = 0 +full_action_space = false +``` + +The boundary target is the first `400` game points. Per-episode scoring clips +the native return into `[0, 400]`; reaching more than `400` is useful research +evidence but does not raise this v1 boundary score. + +The checked-in final suite combines: + +- `25%` direct reproduction from reset; +- `10%` unseen deterministic seeds; +- `10%` startup no-op timing variants; +- `55%` fresh-policy recovery from intermediate trajectory checkpoints. + +For a recovery episode, a trusted policy first advances the environment to a +checkpoint. Optional no-ops are then applied. A new submitted policy instance +receives only the resulting RGB frame and must continue. Reward earned before +the checked-in checkpoints is zero. + +The public evaluator can exercise the same handoff mechanism using the +candidate policy itself as the trusted prelude: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --bootstrap-steps 512 \ + --initial-noops 4 +``` + +Official leaderboard evaluation can inject private seeds, checkpoints, no-op +counts, and matching platform-local anchors. diff --git a/tasks/simulation_heuristics_montezuma_v1/task_context/policy_api.md b/tasks/simulation_heuristics_montezuma_v1/task_context/policy_api.md new file mode 100644 index 0000000..57509db --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/task_context/policy_api.md @@ -0,0 +1,45 @@ +# Policy API + +The evaluator imports `final_policy/policy.py` in a clean Python process. +Define `Policy` or `make_policy`: + +```python +class Policy: + def __init__(self, seed: int = 0): + ... + + def reset(self, seed: int = 0) -> None: + ... + + def act(self, observation) -> int: + ... +``` + +The policy receives exactly one native EnvPool RGB observation: + +```text +shape: (3, 210, 160) +dtype: uint8 +layout: channel, height, width +``` + +The evaluator never passes reward, `info`, RAM, lives, ALE state, room ids, +emulator objects, bootstrap length, or no-op count. + +`act` returns one scalar integer in the full 18-action Atari space: + +| ID | Action | ID | Action | +| ---: | --- | ---: | --- | +| 0 | NOOP | 9 | DOWNLEFT | +| 1 | FIRE | 10 | UPFIRE | +| 2 | UP | 11 | RIGHTFIRE | +| 3 | RIGHT | 12 | LEFTFIRE | +| 4 | LEFT | 13 | DOWNFIRE | +| 5 | DOWN | 14 | UPRIGHTFIRE | +| 6 | UPRIGHT | 15 | UPLEFTFIRE | +| 7 | UPLEFT | 16 | DOWNRIGHTFIRE | +| 8 | DOWNRIGHT | 17 | DOWNLEFTFIRE | + +The policy may keep recurrent state and load assets stored below +`final_policy/`. Every episode, including a recovery handoff, uses a fresh +policy instance followed by `reset(seed=...)`. diff --git a/tasks/simulation_heuristics_montezuma_v1/task_context/provenance.md b/tasks/simulation_heuristics_montezuma_v1/task_context/provenance.md new file mode 100644 index 0000000..3bb0449 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/task_context/provenance.md @@ -0,0 +1,41 @@ +# History, provenance, and licensing + +This task preserves the boundary account published in *Learning Beyond +Gradients*: + +- the article reports an earlier state-graph search moving key distance from + `72` to `28` while score stayed `0`; +- the repaired replay `repair_replay_r1_t19734` scored `400`, used `86` + macro-actions and `1769` environment steps, and was described as mostly + open-loop. + +The calibration source inspected for this task was: + +```text +article: https://trinkle23897.github.io/learning-beyond-gradients/ +repository: https://github.com/Trinkle23897/learning-beyond-gradients +commit: 3555c2956c257d49a5015b782cbe485b14fd659e +artifact: atari/montezuma/heuristic_montezuma_400_macros.json +environment: EnvPool 1.1.1 MontezumaRevenge-v5 +seed: 10001 +``` + +GenesisBench independently implements the evaluator, starter, recovery +protocol, image matching, serialization, and verifier. No upstream Python +implementation is copied. + +The checked-in `reference_trajectory.npz` contains only the expanded action +ids plus native-image hashes and downsampled image features regenerated with +EnvPool `1.1.1`; it contains no RAM or reward trace. Its SHA-256 is: + +```text +72f7211be1c73d556c727b5ef1dc1fbd6aeddc7fe96d44ce99c764089f147aa1 +``` + +The inspected upstream snapshot did not contain a top-level license file, and +the replay artifact did not carry a file-level license grant. The expanded +action trace and regenerated native-image fingerprints are therefore retained +as provenance-tagged benchmark calibration data; this task does not assert a +general relicensing of the upstream artifact. Keep this notice and confirm any +additional permission needed before redistributing the calibration data outside +this benchmark. diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/reference_policy.py b/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/reference_policy.py new file mode 100644 index 0000000..3278d8a --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/reference_policy.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Trusted image-synchronized reference for the 400-point boundary.""" + +from __future__ import annotations + +from collections import defaultdict +import hashlib +from pathlib import Path + +import numpy as np + + +DATA_PATH = Path(__file__).with_name("reference_trajectory.npz") +with np.load(DATA_PATH, allow_pickle=False) as trajectory: + ACTIONS = np.asarray(trajectory["actions"], dtype=np.int64) + FRAME_HASHES = np.asarray(trajectory["hashes"], dtype=np.uint64) + FEATURES = np.asarray(trajectory["features"], dtype=np.uint8) +FEATURES_I16 = FEATURES.astype(np.int16) + +HASH_TO_INDICES: dict[int, tuple[int, ...]] +_hash_to_indices: defaultdict[int, list[int]] = defaultdict(list) +for _index, _frame_hash in enumerate(FRAME_HASHES): + _hash_to_indices[int(_frame_hash)].append(_index) +HASH_TO_INDICES = { + frame_hash: tuple(indices) for frame_hash, indices in _hash_to_indices.items() +} + + +def _frame(observation: np.ndarray) -> np.ndarray: + frame = np.asarray(observation, dtype=np.uint8) + if frame.ndim != 3: + raise ValueError(f"Expected one RGB image, got shape {frame.shape}") + return frame + + +def _frame_hash(frame: np.ndarray) -> int: + digest = hashlib.blake2b(frame.tobytes(), digest_size=8).digest() + return int.from_bytes(digest, byteorder="little", signed=False) + + +def _feature(frame: np.ndarray) -> np.ndarray: + if frame.shape[0] in (1, 3, 4): + frame = np.moveaxis(frame, 0, -1) + rgb = frame[..., :3].astype(np.uint16) + grayscale = (77 * rgb[..., 0] + 150 * rgb[..., 1] + 29 * rgb[..., 2]) >> 8 + return grayscale.astype(np.uint8)[::7, ::7] + + +class Policy: + """Replay through image-state lookup, including fresh-policy re-entry.""" + + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.cursor = 0 + self.started = False + + def _nearest_index(self, frame: np.ndarray) -> int: + indices = HASH_TO_INDICES.get(_frame_hash(frame)) + if indices: + return min( + indices, + key=lambda index: (abs(index - self.cursor), index), + ) + + feature = _feature(frame).astype(np.int16) + if self.started: + lower = max(0, self.cursor - 128) + upper = min(len(ACTIONS), self.cursor + 129) + else: + lower = 0 + upper = len(ACTIONS) + distances = np.mean( + np.abs(FEATURES_I16[lower:upper] - feature), + axis=(1, 2), + ) + return lower + int(np.argmin(distances)) + + def act(self, observation: np.ndarray) -> int: + if self.cursor >= len(ACTIONS): + return 0 + index = self._nearest_index(_frame(observation)) + self.started = True + self.cursor = min(index + 1, len(ACTIONS)) + return int(ACTIONS[index]) diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/reference_trajectory.npz b/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/reference_trajectory.npz new file mode 100644 index 0000000..f8d5191 Binary files /dev/null and b/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/reference_trajectory.npz differ diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/starter_policy.py b/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/starter_policy.py new file mode 100644 index 0000000..dd5d5a4 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/anchor_policies/starter_policy.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +"""Weak native-image explorer for Montezuma's Revenge.""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +RIGHTFIRE = 11 +LEFTFIRE = 12 + + +def _as_hwc(observation: np.ndarray) -> np.ndarray: + frame = np.asarray(observation, dtype=np.uint8) + if frame.ndim != 3: + raise ValueError(f"Expected an RGB image, got shape {frame.shape}") + if frame.shape[0] in (1, 3, 4): + frame = np.moveaxis(frame, 0, -1) + if frame.shape[-1] < 3: + raise ValueError(f"Expected RGB channels, got shape {frame.shape}") + return frame[..., :3] + + +class Policy: + """A valid but intentionally weak pixel-reactive starting point.""" + + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.seed = seed + self.step = 0 + + def act(self, observation: np.ndarray) -> int: + frame = _as_hwc(observation) + red = frame[..., 0].astype(np.int16) + green = frame[..., 1].astype(np.int16) + blue = frame[..., 2].astype(np.int16) + warm_pixels = (red > 120) & (red > green + 30) & (red > blue + 30) + + self.step += 1 + if self.step <= 20: + return NOOP + + locations = np.argwhere(warm_pixels) + if locations.size: + horizontal_center = float(np.mean(locations[:, 1])) + return RIGHTFIRE if horizontal_center < frame.shape[1] / 2 else LEFTFIRE + + return RIGHTFIRE if (self.step // 48) % 2 == 0 else LEFTFIRE diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/anchors.json b/tasks/simulation_heuristics_montezuma_v1/verifier/anchors.json new file mode 100644 index 0000000..91e2819 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/anchors.json @@ -0,0 +1,12 @@ +{ + "environment": "EnvPool 1.1.1 MontezumaRevenge-v5 native RGB, frame_skip=1", + "target_score": 400.0, + "reference_policy": { + "description": "Image-synchronized 400-point replay with checkpoint re-entry", + "path": "anchor_policies/reference_policy.py" + }, + "starter_policy": { + "description": "Public weak native-image explorer", + "path": "anchor_policies/starter_policy.py" + } +} diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/config.toml b/tasks/simulation_heuristics_montezuma_v1/verifier/config.toml new file mode 100644 index 0000000..061d932 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/config.toml @@ -0,0 +1,65 @@ +version = "1.0" + +[evaluation] +max_steps = 2600 +target_score = 400.0 +failure_score = 0.0 + +[[evaluation.variants]] +name = "canonical" + +[[evaluation.variants]] +name = "startup_noop_3" +pre_policy_noops = 3 + +[[evaluation.variants]] +name = "startup_noop_13" +pre_policy_noops = 13 + +[[evaluation.variants]] +name = "recover_512_noop_4" +bootstrap_steps = 512 +pre_policy_noops = 4 + +[[evaluation.variants]] +name = "recover_512_noop_8" +bootstrap_steps = 512 +pre_policy_noops = 8 + +[[evaluation.variants]] +name = "recover_768_noop_2" +bootstrap_steps = 768 +pre_policy_noops = 2 + +[[evaluation.variants]] +name = "recover_900" +bootstrap_steps = 900 + +[[evaluation.suites]] +name = "reproduction" +weight = 0.25 +seeds = [10001] +variants = ["canonical"] + +[[evaluation.suites]] +name = "hidden_seeds" +weight = 0.10 +seeds = [10007, 10103] +variants = ["canonical"] + +[[evaluation.suites]] +name = "noop_variants" +weight = 0.10 +seeds = [10001] +variants = ["startup_noop_3", "startup_noop_13"] + +[[evaluation.suites]] +name = "recovery" +weight = 0.55 +seeds = [10001] +variants = [ + "recover_512_noop_4", + "recover_512_noop_8", + "recover_768_noop_2", + "recover_900", +] diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_montezuma_v1/verifier/evaluate_hidden.py new file mode 100644 index 0000000..8642fe0 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/evaluate_hidden.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +import tomllib +from pathlib import Path + + +VERIFIER_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] +RUNTIME_CANDIDATES.extend(ancestor / "src" for ancestor in VERIFIER_DIR.parents) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.montezuma import ( # noqa: E402 + MontezumaEvaluation, + MontezumaVariant, + evaluate_montezuma_policy, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run hidden Montezuma boundary evaluation." + ) + parser.add_argument("policy", type=Path) + parser.add_argument( + "--config", + type=Path, + default=Path(__file__).parent / "config.toml", + help="Evaluation suite config; production can inject a private file.", + ) + parser.add_argument( + "--anchors", + type=Path, + default=Path(__file__).parent / "anchors.json", + help="Normalization anchors matching the selected evaluation suite.", + ) + parser.add_argument("--output", type=Path) + return parser.parse_args() + + +def _relative_policy( + anchors_path: Path, + anchor: dict, + *, + name: str, +) -> Path: + relative_path = anchor.get("path") + if not isinstance(relative_path, str) or not relative_path: + raise ValueError(f"{name} must declare a policy path") + path = anchors_path.parent / relative_path + if not path.is_file(): + raise FileNotFoundError(path) + return path + + +def _configured_variants(evaluation: dict) -> dict[str, MontezumaVariant]: + variants: dict[str, MontezumaVariant] = {} + for configured in evaluation["variants"]: + variant = MontezumaVariant( + name=str(configured["name"]), + bootstrap_steps=int(configured.get("bootstrap_steps", 0)), + pre_policy_noops=int(configured.get("pre_policy_noops", 0)), + ) + if variant.name in variants: + raise ValueError(f"Duplicate variant name: {variant.name}") + variants[variant.name] = variant + return variants + + +def _evaluate_raw( + policy: Path, + *, + evaluation: dict, + reference_policy: Path, +) -> tuple[dict[str, tuple[float, MontezumaEvaluation]], float]: + variants = _configured_variants(evaluation) + suites: dict[str, tuple[float, MontezumaEvaluation]] = {} + weighted_score = 0.0 + total_weight = 0.0 + + for suite in evaluation["suites"]: + suite_name = str(suite["name"]) + weight = float(suite["weight"]) + selected_variants = tuple(variants[str(name)] for name in suite["variants"]) + result = evaluate_montezuma_policy( + policy, + seeds=suite["seeds"], + max_steps=int(evaluation["max_steps"]), + variants=selected_variants, + bootstrap_policy_path=reference_policy, + target_score=float(evaluation["target_score"]), + failure_score=float(evaluation["failure_score"]), + ) + suites[suite_name] = (weight, result) + weighted_score += weight * result.capped_mean_score + total_weight += weight + + if total_weight <= 0: + raise ValueError("Evaluation suite weights must sum to a positive value") + return suites, weighted_score / total_weight + + +def _anchor_score( + anchors: dict, + name: str, + *, + anchors_path: Path, + evaluation: dict, + reference_policy: Path, +) -> float: + anchor = anchors[name] + score = anchor.get("score") + if isinstance(score, int | float): + return float(score) + policy = _relative_policy(anchors_path, anchor, name=name) + _, calibrated_score = _evaluate_raw( + policy, + evaluation=evaluation, + reference_policy=reference_policy, + ) + return calibrated_score + + +def main() -> None: + args = parse_args() + config = tomllib.loads(args.config.read_text()) + anchors = json.loads(args.anchors.read_text()) + evaluation = config["evaluation"] + reference_policy = _relative_policy( + args.anchors, + anchors["reference_policy"], + name="reference_policy", + ) + + suites, score = _evaluate_raw( + args.policy, + evaluation=evaluation, + reference_policy=reference_policy, + ) + starter_score = _anchor_score( + anchors, + "starter_policy", + anchors_path=args.anchors, + evaluation=evaluation, + reference_policy=reference_policy, + ) + reference_score = _anchor_score( + anchors, + "reference_policy", + anchors_path=args.anchors, + evaluation=evaluation, + reference_policy=reference_policy, + ) + if reference_score == starter_score: + raise ValueError("starter and reference anchors must have different scores") + normalized_score = round( + 100.0 * (score - starter_score) / (reference_score - starter_score), + 9, + ) + suite_payload = { + name: { + "weight": weight, + "score": result.capped_mean_score, + "evaluation": result.to_dict(), + } + for name, (weight, result) in suites.items() + } + payload = { + "score": score, + "normalized_score": normalized_score, + "starter_score": starter_score, + "reference_score": reference_score, + "target_score": float(evaluation["target_score"]), + "suites": suite_payload, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_montezuma_v1/verifier/rubrics/verifier.md new file mode 100644 index 0000000..07fada5 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/rubrics/verifier.md @@ -0,0 +1,8 @@ +# Montezuma's Revenge Boundary Policy Improvement Verifier Rubric + +- `task_success`: normalized weighted native-game score, with each episode + capped at `400`, the starter mapped to `0`, and the trusted image-synchronized + reference mapped to `1`. + +The checked-in weighting is `25%` reproduction, `10%` deterministic hidden +seeds, `10%` startup no-op variants, and `55%` intermediate-state recovery. diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/rubrics/verifier.toml b/tasks/simulation_heuristics_montezuma_v1/verifier/rubrics/verifier.toml new file mode 100644 index 0000000..1faf03b --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/rubrics/verifier.toml @@ -0,0 +1,9 @@ +version = "0.1" + +[[criteria]] +id = "task_success" +description = "Normalized 400-point reproduction and visual recovery score." +weight = 1.0 + +[scoring] +method = "weighted_mean" diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/test.sh b/tasks/simulation_heuristics_montezuma_v1/verifier/test.sh new file mode 100755 index 0000000..bd98ee1 --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/test.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier +policy=/app/final_policy/policy.py + +if [ ! -f "$policy" ]; then + printf '0.0\n' > /logs/verifier/reward.txt + printf '{"reward": 0.0}\n' > /logs/verifier/reward.json + exit 0 +fi + +python /verifier/evaluate_hidden.py \ + "$policy" \ + --config /verifier/config.toml \ + --anchors /verifier/anchors.json \ + --output /logs/verifier/genesis-score.json + +python - <<'PY' +import json +from pathlib import Path + +score = json.loads(Path("/logs/verifier/genesis-score.json").read_text()) +reward = max(0.0, min(1.0, float(score["normalized_score"]) / 100.0)) +Path("/logs/verifier/reward.txt").write_text(f"{reward}\n") +Path("/logs/verifier/reward.json").write_text( + json.dumps({"reward": reward}, indent=2) + "\n" +) +PY diff --git a/tasks/simulation_heuristics_montezuma_v1/verifier/verifier.md b/tasks/simulation_heuristics_montezuma_v1/verifier/verifier.md new file mode 100644 index 0000000..107577a --- /dev/null +++ b/tasks/simulation_heuristics_montezuma_v1/verifier/verifier.md @@ -0,0 +1,38 @@ +--- +document_version: "0.3" +verifier: + name: simulation_heuristics_montezuma_v1 + default_strategy: deterministic + strategies: + deterministic: + type: script + command: ./test.sh + rubric: + combine: weighted_mean + dimensions: + task_success: + weight: 1.0 + source: deterministic + outputs: + reward_text: /logs/verifier/reward.txt + reward_json: /logs/verifier/reward.json + details_json: /logs/verifier/genesis-score.json + aggregate_policy: + method: weighted_mean + metrics: + task_success: 1.0 +--- + +## verifier intent + +Run the submitted native-image policy on direct reproduction, deterministic +hidden seeds, startup no-op variants, and fresh-policy checkpoint recovery. + +Each episode is capped at the `400`-point boundary. BenchFlow's canonical +reward is the normalized GenesisBench score clamped to `[0, 1]`; full suite, +raw-score, recovery, latency, and anchor metrics remain available in +`genesis-score.json`. + +The checked-in suite is reproducible. Official evaluation should inject a +private config and matching anchors so checkpoint/no-op details are not +authoring targets. diff --git a/tasks/simulation_heuristics_pong_ram_v1/README.md b/tasks/simulation_heuristics_pong_ram_v1/README.md new file mode 100644 index 0000000..e0688b8 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/README.md @@ -0,0 +1,167 @@ +# Simulation Heuristics Pong RAM v1 + +`simulation_heuristics_pong_ram_v1` packages the Pong 21 experiment from +*Learning Beyond Gradients* as a reproducible GenesisBench policy-improvement +task. + +The benchmark loop is: + +```text +weak RAM paddle tracker +→ agent experiments and edits code +→ public development episodes +→ final_policy/policy.py +→ clean hidden evaluation +→ locally normalized score +``` + +## Task contract + +The public workspace contains: + +- `task.md`: BenchFlow-native metadata and complete agent instructions; +- `starter_policy/policy.py`: a weak late reactive controller; +- `evaluate.py`: a queryable development evaluator; +- `task_context/`: stable policy, scoring, and provenance documentation; +- `_runtime/`: the trusted GenesisBench runtime copied by task preparation. + +The required artifact is: + +```text +final_policy/ + policy.py +``` + +The policy sees only the current 128-byte Atari RAM vector. Reward, emulator +objects, hidden seeds, and hidden reset configuration remain inside the trusted +evaluator. + +## Environment semantics + +The package preserves the article experiment's EnvPool `Pong-v5` setup: + +- EnvPool `1.1.1`; +- frame skip `1`; +- minimal action space; +- automatic fire reset; +- unclipped reward; +- no sticky actions; +- full-game episodes rather than episodic lives. + +The native episode score is the point differential in `[-21, 21]`. A score of +`21` means winning 21–0 and is the experiment target. + +## Final score + +The checked-in reproduction suite computes: + +```text +raw score = + 0.70 * hidden nominal mean Pong score + + 0.30 * hidden randomized-reset mean Pong score +``` + +Both suites use unseen seeds. The reset-robustness suite widens EnvPool's +random no-op reset window while preserving frame skip, reward, action, and +sticky-action semantics. + +The normalized score maps: + +- the checked-in weak starter controller to `0`; +- the checked-in Pong-21 reference controller to `100`. + +Both anchors are rerun locally on the same runtime as the submission. Scores +above `100` are permitted by the formula, although `21` is the native Pong +maximum. + +## Run locally + +EnvPool is task-specific and intentionally is not added to the repository-wide +Python dependencies. Run an evaluator with an ephemeral dependency: + +```bash +PYTHONPATH=src uv run --with envpool==1.1.1 \ + python tasks/simulation_heuristics_pong_ram_v1/evaluate.py \ + --policy tasks/simulation_heuristics_pong_ram_v1/starter_policy/policy.py +``` + +Reproduce the article target: + +```bash +PYTHONPATH=src uv run --with envpool==1.1.1 \ + python tasks/simulation_heuristics_pong_ram_v1/evaluate.py \ + --policy tasks/simulation_heuristics_pong_ram_v1/oracle/policy.py \ + --episodes 1 \ + --seed 0 +``` + +The result should report a mean score of `21.0`. + +Run the checked-in hidden reproduction suite: + +```bash +PYTHONPATH=src uv run --with envpool==1.1.1 \ + python tasks/simulation_heuristics_pong_ram_v1/verifier/evaluate_hidden.py \ + tasks/simulation_heuristics_pong_ram_v1/oracle/policy.py +``` + +Validate the native task package: + +```bash +uv run bench tasks check \ + tasks/simulation_heuristics_pong_ram_v1 \ + --level publication-grade +``` + +Run the trusted oracle through BenchFlow: + +```bash +uv run bench eval run \ + --tasks-dir tasks/simulation_heuristics_pong_ram_v1 \ + --agent oracle \ + --sandbox docker \ + --context-root . +``` + +Prepare exactly the public workspace an agent receives: + +```bash +uv run python scripts/prepare_task.py \ + simulation_heuristics_pong_ram_v1 \ + /tmp/genesisbench-simulation-heuristics-pong-ram-v1 \ + --force +``` + +The prepared workspace must not contain `verifier/`, `oracle/`, or +`evidence/`. + +## Public versus private final suites + +The checked-in verifier is a transparent reproduction suite. A public +leaderboard should inject a private config and matching private anchors: + +```bash +python verifier/evaluate_hidden.py final_policy/policy.py \ + --config /private/final-suite.toml \ + --anchors /private/final-anchors.json +``` + +Private suites should retain the documented Atari semantics while changing +seeds and reset windows. They should not introduce sticky actions or a +different frame skip under the same benchmark name. + +## Provenance and licensing + +The experiment semantics, RAM coordinate decoding, and geometric reference +controller are derived from: + +- Jiayi Weng, *Learning Beyond Gradients* (2026); +- `Trinkle23897/learning-beyond-gradients`; +- source artifact `atari/pong/heuristic_pong.py`; +- source commit `3555c2956c257d49a5015b782cbe485b14fd659e`. + +The source artifact carries the Copyright 2021 Garena Online Private Limited +notice and Apache License 2.0 header. GenesisBench retains that notice on +policy files and reimplements the policy-only interface and evaluator cleanly. +No source videos, images, trial logs, or ROM files are vendored into this task. +See `task_context/provenance.md` for the detailed boundary. diff --git a/tasks/simulation_heuristics_pong_ram_v1/environment/Dockerfile b/tasks/simulation_heuristics_pong_ram_v1/environment/Dockerfile new file mode 100644 index 0000000..6ddd1c7 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/environment/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12-slim + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends bash ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir \ + "envpool==1.1.1" \ + "numpy>=1.26,<3" \ + "pyyaml>=6,<7" + +RUN useradd --create-home --uid 1000 agent \ + && mkdir -p /app /opt/genesisbench + +COPY src/genesisbench /opt/genesisbench/genesisbench +COPY tasks/simulation_heuristics_pong_ram_v1/evaluate.py /app/evaluate.py +COPY tasks/simulation_heuristics_pong_ram_v1/starter_policy /app/starter_policy +COPY tasks/simulation_heuristics_pong_ram_v1/task_context /app/task_context + +RUN cp -R /app/starter_policy /app/final_policy \ + && chown -R root:root /opt/genesisbench \ + && chmod -R a-w /opt/genesisbench \ + && chown -R agent:agent /app + +ENV PYTHONPATH="/opt/genesisbench" + +WORKDIR /app + +RUN mkdir -p /logs/verifier /logs/agent /logs/artifacts diff --git a/tasks/simulation_heuristics_pong_ram_v1/evaluate.py b/tasks/simulation_heuristics_pong_ram_v1/evaluate.py new file mode 100644 index 0000000..c7be089 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/evaluate.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +TASK_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = ( + TASK_DIR / "_runtime", + TASK_DIR.parent / "src", + TASK_DIR.parent.parent / "src", +) +for candidate in RUNTIME_CANDIDATES: + if candidate.is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.pong import PongVariant, evaluate_pong_policy # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate a Pong RAM policy.") + parser.add_argument( + "--policy", + type=Path, + default=Path("final_policy/policy.py"), + ) + parser.add_argument("--episodes", type=int, default=3) + parser.add_argument("--max-steps", type=int, default=27_000) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--noop-max", type=int, default=1) + parser.add_argument("--json-output-file", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + result = evaluate_pong_policy( + args.policy, + seeds=range(args.seed, args.seed + args.episodes), + max_steps=args.max_steps, + variants=( + PongVariant( + name="development", + noop_max=args.noop_max, + ), + ), + ) + rendered = result.to_json() + print(rendered) + if args.json_output_file is not None: + args.json_output_file.parent.mkdir(parents=True, exist_ok=True) + args.json_output_file.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_pong_ram_v1/oracle/policy.py b/tasks/simulation_heuristics_pong_ram_v1/oracle/policy.py new file mode 100644 index 0000000..07aabe1 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/oracle/policy.py @@ -0,0 +1,118 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Geometric Pong RAM controller reimplemented for GenesisBench. + +The RAM decoding constants and controller behavior are derived from +Trinkle23897/learning-beyond-gradients at commit +3555c2956c257d49a5015b782cbe485b14fd659e. This file is a clean policy-only +implementation of that Apache-2.0-licensed artifact. +""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +MOVE_UP = 2 +MOVE_DOWN = 3 +HOME_Y = 105.0 +PADDLE_X = 141.0 +FIELD_TOP = 34.0 +FIELD_BOTTOM = 194.0 +DEADBAND = 6.0 +SPIN_OFFSET = 8.0 +MAX_VELOCITY_JUMP = 24.0 + + +def _reflect(value: float) -> float: + span = FIELD_BOTTOM - FIELD_TOP + folded = (value - FIELD_TOP) % (2.0 * span) + if folded <= span: + return FIELD_TOP + folded + return FIELD_BOTTOM - (folded - span) + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.previous_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + + def _track_ball( + self, + ball: tuple[float, float] | None, + ) -> None: + if ball is None: + self.previous_ball = None + self.velocity = None + return + + if self.previous_ball is not None: + dx = ball[0] - self.previous_ball[0] + dy = ball[1] - self.previous_ball[1] + plausible = ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ) + if plausible: + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * (self.velocity[0] + dx), + 0.5 * (self.velocity[1] + dy), + ) + else: + self.velocity = None + self.previous_ball = ball + + def act(self, observation: np.ndarray) -> int: + ram = np.asarray(observation, dtype=np.uint8) + if ram.shape != (128,): + raise ValueError(f"Expected Pong RAM shape (128,), got {ram.shape}") + + ball = None + if int(ram[54]) != 0: + ball = (float(ram[49]) - 49.0, float(ram[54]) - 13.0) + self_y = 0.972157 * float(ram[51]) - 2.553996 + opponent_y = 0.981619 * float(ram[50]) - 5.492890 + self._track_ball(ball) + + target_y = HOME_Y + if ball is not None: + ball_x, ball_y = ball + if self.velocity is not None and self.velocity[0] > 0.05: + velocity_x, velocity_y = self.velocity + if ball_x <= PADDLE_X: + target_y = _reflect( + ball_y + velocity_y / velocity_x * (PADDLE_X - ball_x) + ) + else: + target_y = ball_y + outgoing_sign = 1.0 if opponent_y < HOME_Y else -1.0 + target_y -= outgoing_sign * SPIN_OFFSET + elif self.velocity is None and ball_x > 90.0: + target_y = ball_y + + error = target_y - self_y + if error < -DEADBAND: + return MOVE_UP + if error > DEADBAND: + return MOVE_DOWN + return NOOP diff --git a/tasks/simulation_heuristics_pong_ram_v1/oracle/solve.sh b/tasks/simulation_heuristics_pong_ram_v1/oracle/solve.sh new file mode 100755 index 0000000..1af9fd8 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/oracle/solve.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /app/final_policy +cp /oracle/policy.py /app/final_policy/policy.py diff --git a/tasks/simulation_heuristics_pong_ram_v1/starter_policy/policy.py b/tasks/simulation_heuristics_pong_ram_v1/starter_policy/policy.py new file mode 100644 index 0000000..f88865c --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/starter_policy/policy.py @@ -0,0 +1,56 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Weak late reactive Pong RAM controller for GenesisBench. + +The RAM decoding constants are derived from +Trinkle23897/learning-beyond-gradients at commit +3555c2956c257d49a5015b782cbe485b14fd659e. +""" + +from __future__ import annotations + +import numpy as np + + +HOME_Y = 105.0 +CHASE_AFTER_X = 122.0 +DEADBAND = 10.0 + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.seed = seed + + def reset(self, seed: int = 0) -> None: + self.seed = seed + + def act(self, observation: np.ndarray) -> int: + ram = np.asarray(observation, dtype=np.uint8) + if ram.shape != (128,): + raise ValueError(f"Expected Pong RAM shape (128,), got {ram.shape}") + + self_y = 0.972157 * float(ram[51]) - 2.553996 + target_y = HOME_Y + if int(ram[54]) != 0: + ball_x = float(ram[49]) - 49.0 + if ball_x > CHASE_AFTER_X: + target_y = float(ram[54]) - 13.0 + + error = target_y - self_y + if error < -DEADBAND: + return 2 + if error > DEADBAND: + return 3 + return 0 diff --git a/tasks/simulation_heuristics_pong_ram_v1/task.md b/tasks/simulation_heuristics_pong_ram_v1/task.md new file mode 100644 index 0000000..ff5aad7 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/task.md @@ -0,0 +1,169 @@ +--- +schema_version: "1.3" +task: + name: genesisbench/simulation_heuristics_pong_ram_v1 + description: Improve a programmatic Atari Pong controller that acts from the current 128-byte RAM state. + authors: + - name: GenesisBench Contributors + keywords: + - simulation + - heuristics + - atari + - pong + - ram +metadata: + category: game-control + difficulty: medium + tags: + - atari + - discrete-control + - policy-search + - programmatic-control + - ram + reference_task: false + genesisbench: + starter: + path: starter_policy + submission: + directory: final_policy + entrypoint: policy.py + development: + episodes: 3 + max_steps: 27000 + seeds: [0, 1, 2] + verifier: + reproduction_config: verifier/config.toml + anchors: verifier/anchors.json + supports_private_config: true +agent: + timeout_sec: 1800 + user: agent + network_mode: public +verifier: + timeout_sec: 600 + user: root + network_mode: no-network +environment: + build_timeout_sec: 1200 + cpus: 4 + memory_mb: 4096 + storage_mb: 10240 + network_mode: public + allow_internet: true + workdir: /app +benchflow: + document_version: "0.6" +--- +# Improve the Atari Pong RAM controller + +You are given a working but weak programmatic controller for EnvPool +`Pong-v5`. Improve it through autonomous coding and experimentation. + +## Objective + +Maximize the native Pong episode score: + +```text +points won - points lost +``` + +An episode ends when either side reaches 21 points. The target score is the +perfect result: + +```text ++21 +``` + +The evaluator uses the article experiment's Atari setup: + +- current 128-byte RAM state as the policy observation; +- minimal Pong discrete action space; +- frame skip `1`; +- unclipped rewards; +- no sticky actions; +- automatic fire reset; +- no episodic-life wrapper. + +## Starting point + +`starter_policy/policy.py` contains a late reactive paddle tracker. It decodes +the ball and controlled paddle from RAM, but begins chasing too late and does +not predict wall bounces or shape outgoing returns. + +You may replace it, tune it, add search scripts, or build a more capable +stateful controller. The submitted policy must remain a directly inspectable +programmatic controller; do not download or bundle a pretrained neural +network. + +## Development evaluation + +Run: + +```bash +python evaluate.py --policy starter_policy/policy.py +``` + +Evaluate your current submission: + +```bash +python evaluate.py --policy final_policy/policy.py +``` + +For a quick smoke test: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --episodes 1 \ + --max-steps 4000 +``` + +The public evaluator uses visible development seeds. The final evaluator uses +unseen seeds and multiple random no-op reset windows while keeping the article +environment semantics fixed. A controller that depends on one memorized serve +timing is unlikely to score well. + +## Required submission + +Store your best policy at: + +```text +final_policy/policy.py +``` + +It must define either: + +```python +class Policy: + def __init__(self, seed: int = 0): ... + def reset(self, seed: int = 0) -> None: ... + def act(self, observation) -> int: ... +``` + +or: + +```python +def make_policy(seed: int = 0): ... +``` + +The returned object must implement `reset` and `act`. + +`observation` is a NumPy-compatible vector with shape `(128,)` and dtype +`uint8`. `act` must return one finite integer in the environment's minimal +discrete action space. The standard controller actions are `0` for no-op, `2` +for paddle up, and `3` for paddle down. + +## Rules + +1. Work autonomously. Do not ask for user feedback. +2. Use the fixed wall-clock budget for iterative improvement. +3. Do not modify `evaluate.py` or trusted runtime files. +4. Do not access `/oracle`, `/verifier`, or reconstruct hidden seeds/config. +5. The policy receives RAM only. It does not receive reward, `info`, emulator + objects, screenshots, or hidden evaluator parameters. +6. The policy may keep recurrent state and load files stored below + `final_policy/`. +7. The final policy must run offline with packages installed in the task + environment. +8. Before finishing, evaluate `final_policy/policy.py` and leave the best + working version in place. diff --git a/tasks/simulation_heuristics_pong_ram_v1/task_context/evaluation.md b/tasks/simulation_heuristics_pong_ram_v1/task_context/evaluation.md new file mode 100644 index 0000000..d0cc520 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/task_context/evaluation.md @@ -0,0 +1,64 @@ +# Evaluation + +## Native environment + +The evaluator uses EnvPool `Pong-v5` with: + +```text +frame_skip = 1 +reward_clip = false +repeat_action_probability = 0.0 +use_fire_reset = true +episodic_life = false +full_action_space = false +``` + +Each environment step advances one Atari frame. The full episode is capped at +27,000 policy decisions as a safety limit. + +## Native score + +Pong emits: + +```text ++1 when the controlled right paddle wins a point +-1 when the opponent wins a point +``` + +The episode ends when either side reaches 21 points. Therefore: + +```text +episode score = points_for - points_against +range = [-21, 21] +target = 21 +``` + +## Public metrics + +The development evaluator reports: + +- mean, minimum, and maximum native score; +- win rate; +- perfect-score rate; +- invalid-policy rate; +- mean policy action latency; +- per-episode seeds, variants, point totals, lengths, and errors. + +## Final suite + +The reproduction verifier combines: + +```text +0.70 * unseen-seed nominal mean score ++ 0.30 * unseen-seed randomized-reset mean score +``` + +Both suites keep the article's frame skip and zero-sticky-action semantics. +The randomized-reset suite only widens the random no-op window before a game +begins. Official leaderboard runs may inject private seeds and reset windows. + +Invalid actions, import errors, constructor/reset errors, and policy exceptions +receive a native failure score of `-21` for the affected episode. + +The raw score is normalized against locally rerun starter and reference +policies. Starter maps to `0`; the Pong-21 reference maps to `100`. diff --git a/tasks/simulation_heuristics_pong_ram_v1/task_context/policy_api.md b/tasks/simulation_heuristics_pong_ram_v1/task_context/policy_api.md new file mode 100644 index 0000000..3801d43 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/task_context/policy_api.md @@ -0,0 +1,47 @@ +# Policy API + +The evaluator imports `final_policy/policy.py` in a clean Python process. + +Define `Policy` or `make_policy`. The policy is instantiated once per episode +and reset before the first action. + +```python +class Policy: + def __init__(self, seed: int = 0): + ... + + def reset(self, seed: int = 0) -> None: + ... + + def act(self, observation): + ... +``` + +`observation` is the current Atari 2600 RAM image: + +```text +shape: (128,) +dtype: uint8 +range: 0..255 +``` + +No frame stack is supplied. A policy may retain previous RAM observations, +velocity estimates, rally state, or other recurrent state internally. + +`act` must return one finite integer accepted by EnvPool's minimal Pong action +space. The standard paddle controller uses: + +```text +0 = no-op +2 = paddle up +3 = paddle down +``` + +Other in-range minimal actions are accepted, but the evaluator rejects arrays +with more than one value, fractional actions, NaN/infinity, and out-of-range +integers. Invalid policies receive the configured failure score for that +episode. + +The policy does not receive reward, score, `info`, screenshots, emulator +objects, hidden seeds, or reset-window parameters. It may load files stored +below `final_policy/`. diff --git a/tasks/simulation_heuristics_pong_ram_v1/task_context/provenance.md b/tasks/simulation_heuristics_pong_ram_v1/task_context/provenance.md new file mode 100644 index 0000000..3ebca56 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/task_context/provenance.md @@ -0,0 +1,55 @@ +# Provenance and licensing + +## Source experiment + +This task packages the Pong 21 reproduction described in: + +```text +Jiayi Weng, Learning Beyond Gradients, 2026 +https://github.com/Trinkle23897/learning-beyond-gradients +``` + +The implementation was derived from the source artifact: + +```text +atari/pong/heuristic_pong.py +commit 3555c2956c257d49a5015b782cbe485b14fd659e +``` + +That artifact documents the expected seed-0 result: + +```text +episode score = 21.0 +mean score = 21.000 +``` + +## Reimplementation boundary + +GenesisBench does not copy the source script wholesale. It reimplements: + +- the policy-only `Policy.reset` / `Policy.act` contract; +- the trusted RAM-only evaluator; +- action and error validation; +- multi-seed reset-robustness evaluation; +- local starter/reference normalization; +- BenchFlow task, verifier, Docker, and oracle packaging. + +The reference preserves the source experiment's observable controller +semantics: RAM coordinate decoding, recurrent velocity estimation, reflected +intercept prediction, paddle deadband, and a small return-angle bias. + +No source videos, images, reports, trial records, or ROM files are copied into +the repository. + +## License notice + +The source policy file carries: + +```text +Copyright 2021 Garena Online Private Limited +Licensed under the Apache License, Version 2.0 +``` + +The starter, oracle, and trusted anchor policy files retain that notice and +identify the exact upstream artifact and commit. The surrounding GenesisBench +task packaging is authored for this repository. diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/anchor_policies/reference_policy.py b/tasks/simulation_heuristics_pong_ram_v1/verifier/anchor_policies/reference_policy.py new file mode 100644 index 0000000..07aabe1 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/anchor_policies/reference_policy.py @@ -0,0 +1,118 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Geometric Pong RAM controller reimplemented for GenesisBench. + +The RAM decoding constants and controller behavior are derived from +Trinkle23897/learning-beyond-gradients at commit +3555c2956c257d49a5015b782cbe485b14fd659e. This file is a clean policy-only +implementation of that Apache-2.0-licensed artifact. +""" + +from __future__ import annotations + +import numpy as np + + +NOOP = 0 +MOVE_UP = 2 +MOVE_DOWN = 3 +HOME_Y = 105.0 +PADDLE_X = 141.0 +FIELD_TOP = 34.0 +FIELD_BOTTOM = 194.0 +DEADBAND = 6.0 +SPIN_OFFSET = 8.0 +MAX_VELOCITY_JUMP = 24.0 + + +def _reflect(value: float) -> float: + span = FIELD_BOTTOM - FIELD_TOP + folded = (value - FIELD_TOP) % (2.0 * span) + if folded <= span: + return FIELD_TOP + folded + return FIELD_BOTTOM - (folded - span) + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.reset(seed) + + def reset(self, seed: int = 0) -> None: + self.previous_ball: tuple[float, float] | None = None + self.velocity: tuple[float, float] | None = None + + def _track_ball( + self, + ball: tuple[float, float] | None, + ) -> None: + if ball is None: + self.previous_ball = None + self.velocity = None + return + + if self.previous_ball is not None: + dx = ball[0] - self.previous_ball[0] + dy = ball[1] - self.previous_ball[1] + plausible = ( + abs(dx) <= MAX_VELOCITY_JUMP + and abs(dy) <= MAX_VELOCITY_JUMP + and abs(dx) + abs(dy) > 0.25 + ) + if plausible: + if self.velocity is None: + self.velocity = (dx, dy) + else: + self.velocity = ( + 0.5 * (self.velocity[0] + dx), + 0.5 * (self.velocity[1] + dy), + ) + else: + self.velocity = None + self.previous_ball = ball + + def act(self, observation: np.ndarray) -> int: + ram = np.asarray(observation, dtype=np.uint8) + if ram.shape != (128,): + raise ValueError(f"Expected Pong RAM shape (128,), got {ram.shape}") + + ball = None + if int(ram[54]) != 0: + ball = (float(ram[49]) - 49.0, float(ram[54]) - 13.0) + self_y = 0.972157 * float(ram[51]) - 2.553996 + opponent_y = 0.981619 * float(ram[50]) - 5.492890 + self._track_ball(ball) + + target_y = HOME_Y + if ball is not None: + ball_x, ball_y = ball + if self.velocity is not None and self.velocity[0] > 0.05: + velocity_x, velocity_y = self.velocity + if ball_x <= PADDLE_X: + target_y = _reflect( + ball_y + velocity_y / velocity_x * (PADDLE_X - ball_x) + ) + else: + target_y = ball_y + outgoing_sign = 1.0 if opponent_y < HOME_Y else -1.0 + target_y -= outgoing_sign * SPIN_OFFSET + elif self.velocity is None and ball_x > 90.0: + target_y = ball_y + + error = target_y - self_y + if error < -DEADBAND: + return MOVE_UP + if error > DEADBAND: + return MOVE_DOWN + return NOOP diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/anchor_policies/starter_policy.py b/tasks/simulation_heuristics_pong_ram_v1/verifier/anchor_policies/starter_policy.py new file mode 100644 index 0000000..f88865c --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/anchor_policies/starter_policy.py @@ -0,0 +1,56 @@ +# Copyright 2021 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Weak late reactive Pong RAM controller for GenesisBench. + +The RAM decoding constants are derived from +Trinkle23897/learning-beyond-gradients at commit +3555c2956c257d49a5015b782cbe485b14fd659e. +""" + +from __future__ import annotations + +import numpy as np + + +HOME_Y = 105.0 +CHASE_AFTER_X = 122.0 +DEADBAND = 10.0 + + +class Policy: + def __init__(self, seed: int = 0) -> None: + self.seed = seed + + def reset(self, seed: int = 0) -> None: + self.seed = seed + + def act(self, observation: np.ndarray) -> int: + ram = np.asarray(observation, dtype=np.uint8) + if ram.shape != (128,): + raise ValueError(f"Expected Pong RAM shape (128,), got {ram.shape}") + + self_y = 0.972157 * float(ram[51]) - 2.553996 + target_y = HOME_Y + if int(ram[54]) != 0: + ball_x = float(ram[49]) - 49.0 + if ball_x > CHASE_AFTER_X: + target_y = float(ram[54]) - 13.0 + + error = target_y - self_y + if error < -DEADBAND: + return 2 + if error > DEADBAND: + return 3 + return 0 diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/anchors.json b/tasks/simulation_heuristics_pong_ram_v1/verifier/anchors.json new file mode 100644 index 0000000..17ce0d4 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/anchors.json @@ -0,0 +1,14 @@ +{ + "environment": "EnvPool 1.1.1 Pong-v5 RAM, frame_skip=1, unclipped reward, no sticky actions", + "max_steps": 27000, + "source_commit": "3555c2956c257d49a5015b782cbe485b14fd659e", + "target_score": 21, + "reference_policy": { + "description": "Frozen geometric intercept controller reproducing Pong 21", + "path": "anchor_policies/reference_policy.py" + }, + "starter_policy": { + "description": "Public weak late reactive RAM tracker", + "path": "anchor_policies/starter_policy.py" + } +} diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/config.toml b/tasks/simulation_heuristics_pong_ram_v1/verifier/config.toml new file mode 100644 index 0000000..d95633e --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/config.toml @@ -0,0 +1,31 @@ +version = "1.0" + +[evaluation] +max_steps = 27000 +failure_score = -21.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.7 +variant = "article_nominal" +seeds = [101, 211, 307] + +[[evaluation.suites]] +name = "hidden_reset_robustness" +weight = 0.3 +variant = "randomized_reset" +seeds = [503, 601, 701] + +[[evaluation.variants]] +name = "article_nominal" +noop_max = 1 +frame_skip = 1 +repeat_action_probability = 0.0 +use_fire_reset = true + +[[evaluation.variants]] +name = "randomized_reset" +noop_max = 30 +frame_skip = 1 +repeat_action_probability = 0.0 +use_fire_reset = true diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_pong_ram_v1/verifier/evaluate_hidden.py new file mode 100644 index 0000000..14bc3e0 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/evaluate_hidden.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import sys +import tomllib +from pathlib import Path + + +VERIFIER_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] +RUNTIME_CANDIDATES.extend(ancestor / "src" for ancestor in VERIFIER_DIR.parents) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.pong import ( # noqa: E402 + PONG_TARGET_SCORE, + PongEvaluation, + PongVariant, + evaluate_pong_policy, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run hidden Atari Pong RAM evaluation." + ) + parser.add_argument("policy", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument( + "--config", + type=Path, + default=Path(__file__).parent / "config.toml", + help="Evaluation suite config; production can inject a private file.", + ) + parser.add_argument( + "--anchors", + type=Path, + default=Path(__file__).parent / "anchors.json", + help="Normalization anchors matching the selected evaluation suite.", + ) + return parser.parse_args() + + +def _configured_variants(evaluation: dict) -> dict[str, PongVariant]: + variants = {} + for configured in evaluation["variants"]: + variant = PongVariant( + name=configured["name"], + noop_max=int(configured["noop_max"]), + frame_skip=int(configured["frame_skip"]), + repeat_action_probability=float(configured["repeat_action_probability"]), + use_fire_reset=bool(configured["use_fire_reset"]), + ) + variants[variant.name] = variant + return variants + + +def _evaluate_raw( + policy: Path, + *, + evaluation: dict, +) -> tuple[dict[str, PongEvaluation], float]: + variants = _configured_variants(evaluation) + suites: dict[str, PongEvaluation] = {} + weighted_score = 0.0 + total_weight = 0.0 + + for suite in evaluation["suites"]: + weight = float(suite["weight"]) + variant_name = suite["variant"] + if variant_name not in variants: + raise ValueError( + f"Suite {suite['name']!r} references unknown variant {variant_name!r}" + ) + result = evaluate_pong_policy( + policy, + seeds=suite["seeds"], + max_steps=int(evaluation["max_steps"]), + variants=(variants[variant_name],), + failure_score=float(evaluation["failure_score"]), + ) + suites[suite["name"]] = result + weighted_score += weight * result.mean_score + total_weight += weight + + if not math.isclose(total_weight, 1.0): + raise ValueError( + f"Evaluation suite weights must sum to 1.0, got {total_weight}" + ) + return suites, weighted_score + + +def _anchor_score( + anchors: dict, + name: str, + *, + anchors_path: Path, + evaluation: dict, + evaluated_policy: Path, + evaluated_score: float, +) -> float: + anchor = anchors[name] + score = anchor.get("score") + if isinstance(score, int | float): + return float(score) + relative_path = anchor.get("path") + if not isinstance(relative_path, str) or not relative_path: + raise ValueError(f"{name} must declare score or path") + policy = (anchors_path.parent / relative_path).resolve() + if policy == evaluated_policy: + return evaluated_score + _, calibrated_score = _evaluate_raw( + policy, + evaluation=evaluation, + ) + return calibrated_score + + +def main() -> None: + args = parse_args() + config = tomllib.loads(args.config.read_text()) + anchors = json.loads(args.anchors.read_text()) + evaluation = config["evaluation"] + suites, score = _evaluate_raw( + args.policy, + evaluation=evaluation, + ) + evaluated_policy = args.policy.resolve() + if evaluated_policy.is_dir(): + evaluated_policy = evaluated_policy / "policy.py" + starter_score = _anchor_score( + anchors, + "starter_policy", + anchors_path=args.anchors, + evaluation=evaluation, + evaluated_policy=evaluated_policy, + evaluated_score=score, + ) + reference_score = _anchor_score( + anchors, + "reference_policy", + anchors_path=args.anchors, + evaluation=evaluation, + evaluated_policy=evaluated_policy, + evaluated_score=score, + ) + if reference_score == starter_score: + raise ValueError("starter and reference anchors must have different scores") + + normalized_score = round( + 100.0 * (score - starter_score) / (reference_score - starter_score), + 9, + ) + suite_weights = { + suite["name"]: float(suite["weight"]) for suite in evaluation["suites"] + } + payload = { + "score": score, + "normalized_score": normalized_score, + "starter_score": starter_score, + "reference_score": reference_score, + "target_score": PONG_TARGET_SCORE, + "suite_weights": suite_weights, + "suites": {name: result.to_dict() for name, result in suites.items()}, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_pong_ram_v1/verifier/rubrics/verifier.md new file mode 100644 index 0000000..cb5e19b --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/rubrics/verifier.md @@ -0,0 +1,13 @@ +# Simulation Heuristics Pong RAM v1 Verifier Rubric + +- `task_success`: weighted hidden-suite native Pong score, normalized so the + weak starter maps to `0` and the frozen Pong-21 reference maps to `1`. + +The hidden score combines: + +- `70%` unseen nominal seeds; +- `30%` unseen seeds with a wider random no-op reset window. + +Malformed or failing policies receive `-21` for each affected episode. The +BenchFlow reward is clamped to `[0, 1]`, while the detailed output retains raw +scores and unclamped normalization. diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/rubrics/verifier.toml b/tasks/simulation_heuristics_pong_ram_v1/verifier/rubrics/verifier.toml new file mode 100644 index 0000000..6e10add --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/rubrics/verifier.toml @@ -0,0 +1,9 @@ +version = "0.1" + +[[criteria]] +id = "task_success" +description = "Normalized hidden-suite Atari Pong RAM score." +weight = 1.0 + +[scoring] +method = "weighted_mean" diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/test.sh b/tasks/simulation_heuristics_pong_ram_v1/verifier/test.sh new file mode 100755 index 0000000..bd98ee1 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/test.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier +policy=/app/final_policy/policy.py + +if [ ! -f "$policy" ]; then + printf '0.0\n' > /logs/verifier/reward.txt + printf '{"reward": 0.0}\n' > /logs/verifier/reward.json + exit 0 +fi + +python /verifier/evaluate_hidden.py \ + "$policy" \ + --config /verifier/config.toml \ + --anchors /verifier/anchors.json \ + --output /logs/verifier/genesis-score.json + +python - <<'PY' +import json +from pathlib import Path + +score = json.loads(Path("/logs/verifier/genesis-score.json").read_text()) +reward = max(0.0, min(1.0, float(score["normalized_score"]) / 100.0)) +Path("/logs/verifier/reward.txt").write_text(f"{reward}\n") +Path("/logs/verifier/reward.json").write_text( + json.dumps({"reward": reward}, indent=2) + "\n" +) +PY diff --git a/tasks/simulation_heuristics_pong_ram_v1/verifier/verifier.md b/tasks/simulation_heuristics_pong_ram_v1/verifier/verifier.md new file mode 100644 index 0000000..54721b3 --- /dev/null +++ b/tasks/simulation_heuristics_pong_ram_v1/verifier/verifier.md @@ -0,0 +1,34 @@ +--- +document_version: "0.3" +verifier: + name: simulation_heuristics_pong_ram_v1 + default_strategy: deterministic + strategies: + deterministic: + type: script + command: ./test.sh + rubric: + combine: weighted_mean + dimensions: + task_success: + weight: 1.0 + source: deterministic + outputs: + reward_text: /logs/verifier/reward.txt + reward_json: /logs/verifier/reward.json + details_json: /logs/verifier/genesis-score.json + aggregate_policy: + method: weighted_mean + metrics: + task_success: 1.0 +--- + +## verifier intent + +Run the submitted Atari Pong RAM policy on unseen nominal seeds and unseen +randomized-reset seeds while preserving the article's frame-skip-1, +unclipped-reward, zero-sticky-action semantics. + +BenchFlow's canonical reward is the locally normalized GenesisBench score +clamped to `[0, 1]`. Full native Pong scores, point totals, errors, suite +weights, and unclamped normalization remain in `genesis-score.json`. diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/README.md b/tasks/simulation_heuristics_vizdoom_d1_v1/README.md new file mode 100644 index 0000000..9224eb7 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/README.md @@ -0,0 +1,100 @@ +# Simulation Heuristics VizDoom D1 v1 + +This task turns the D1 Basic experiment from *Learning Beyond Gradients* into +a clean GenesisBench policy-improvement package. + +The public policy contract is deliberately narrow: + +- input image: the `240 x 180` RGB frame returned by EnvPool `render()`; +- public variable: `HEALTH`; +- output: one of the six combined D1 actions. + +The policy never receives the EnvPool environment, WAD/config paths, map +geometry, labels, object coordinates, the base seed, or the vector lane id. +The evaluator also audits policy source for direct simulator or privileged-map +access. + +## Article reproduction target + +With EnvPool `1.1.1`, batch seed `0`, ten lanes, frame skip `1`, and 2,100 +steps, the frozen reference policy reproduces: + +```text +mean = 0.9440999741666019 +min = 0.28999998047947884 +rewards = [ + 1.0799999684095383, + 1.0399999842047691, + 0.28999998047947884, + 0.9279999658465385, + 1.0799999684095383, + 1.0099999718368053, + 1.0349999852478504, + 1.004999976605177, + 0.9439999675378203, + 1.0289999730885029, +] +``` + +## Task flow + +```text +starter_policy/policy.py +→ public evaluate.py +→ prepared final_policy/policy.py +→ hidden unseen-seed evaluation +→ local starter/reference normalization +``` + +`final_policy/` is created in the prepared agent workspace from +`starter_policy/`; it is intentionally not checked into the source package +because the shared task preparer owns that copy step. + +## Hidden verifier isolation + +EnvPool `1.1.1` cannot safely create multiple D1 VizDoom instances +sequentially in one Linux verifier process and working directory. The hidden +evaluator therefore runs each seed suite in a fresh subprocess with a unique +working directory. + +Before each suite, the root verifier terminates only stale task-owned +`/app/evaluate.py` and VizDoom processes, then removes exact `ViZDoom*` +artifacts owned by the task from `/dev/shm` and +`/tmp/boost_interprocess`. This releases shared memory left by timed-out agent +development evaluations without touching unrelated resources. + +The checked-in starter/reference scores were calibrated in clean, separate +EnvPool `1.1.1` runs. D1 screen-CV returns differ between the supported +`darwin-arm64` and `linux-x86_64` runtimes, so each platform has an explicit +profile. Per-suite means and the exact hidden config are stored in +`verifier/anchors.json`; missing or stale platform calibration fails closed. + +## Run locally + +```bash +python tasks/simulation_heuristics_vizdoom_d1_v1/evaluate.py \ + --policy tasks/simulation_heuristics_vizdoom_d1_v1/oracle/policy.py +``` + +Validate the native task package: + +```bash +uv run bench tasks check \ + tasks/simulation_heuristics_vizdoom_d1_v1 \ + --level publication-grade +``` + +## Provenance and licensing + +The behavioral contract and published target come from +`Trinkle23897/learning-beyond-gradients` commit +`3555c2956c257d49a5015b782cbe485b14fd659e`. +The source repository's runtime notes explicitly pin the experiments to +EnvPool `1.1.1`; the task Docker image and evaluator enforce the same version. + +That repository did not declare a source license at the pinned revision. +Accordingly, this package does not vendor its policy source. The evaluator, +task API, starter, and compact reference policy are independently implemented +for GenesisBench from the public experiment description and measured +behavior. EnvPool is installed from its published package and retains its own +license. diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/environment/Dockerfile b/tasks/simulation_heuristics_vizdoom_d1_v1/environment/Dockerfile new file mode 100644 index 0000000..83c32a1 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/environment/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.12-slim + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends \ + bash \ + ca-certificates \ + libgl1 \ + libglib2.0-0 \ + libx11-6 \ + libxext6 \ + libxrender1 \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir \ + "envpool==1.1.1" \ + "numpy>=1.26,<2.3" \ + "opencv-python-headless==4.12.0.88" \ + "pyyaml>=6,<7" + +RUN useradd --create-home --uid 1000 agent \ + && mkdir -p /app /opt/genesisbench + +COPY src/genesisbench /opt/genesisbench/genesisbench +COPY tasks/simulation_heuristics_vizdoom_d1_v1/evaluate.py /app/evaluate.py +COPY tasks/simulation_heuristics_vizdoom_d1_v1/starter_policy /app/starter_policy +COPY tasks/simulation_heuristics_vizdoom_d1_v1/task_context /app/task_context + +RUN cp -R /app/starter_policy /app/final_policy \ + && chown -R root:root /opt/genesisbench \ + && chmod -R a-w /opt/genesisbench \ + && chown -R agent:agent /app + +ENV PYTHONPATH="/opt/genesisbench" +ENV SDL_VIDEODRIVER="dummy" + +WORKDIR /app + +RUN mkdir -p /logs/verifier /logs/agent /logs/artifacts diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/evaluate.py b/tasks/simulation_heuristics_vizdoom_d1_v1/evaluate.py new file mode 100755 index 0000000..28181a0 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/evaluate.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +TASK_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = ( + TASK_DIR / "_runtime", + TASK_DIR.parent / "src", + TASK_DIR.parent.parent / "src", +) +for candidate in RUNTIME_CANDIDATES: + if candidate.is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.vizdoom import evaluate_vizdoom_policy # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate a rendered-pixel VizDoom D1 policy." + ) + parser.add_argument( + "--policy", + type=Path, + default=Path("final_policy/policy.py"), + ) + parser.add_argument("--episodes", type=int, default=10) + parser.add_argument("--max-steps", type=int, default=2100) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--render-width", type=int, default=240) + parser.add_argument("--render-height", type=int, default=180) + parser.add_argument("--json-output-file", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + result = evaluate_vizdoom_policy( + args.policy, + scenario="d1", + seed=args.seed, + episodes=args.episodes, + max_steps=args.max_steps, + frame_skip=1, + render_width=args.render_width, + render_height=args.render_height, + failure_return=-1.0, + ) + rendered = result.to_json() + print(rendered) + if args.json_output_file is not None: + args.json_output_file.parent.mkdir(parents=True, exist_ok=True) + args.json_output_file.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() + diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/oracle/policy.py b/tasks/simulation_heuristics_vizdoom_d1_v1/oracle/policy.py new file mode 100644 index 0000000..83d7f96 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/oracle/policy.py @@ -0,0 +1,129 @@ +"""Compact screen-CV reference for the VizDoom D1 article experiment. + +This is an independent GenesisBench implementation of the published behavior. +No upstream source file is vendored. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +import cv2 +import numpy as np + + +NONE = 0 +TURN_RIGHT = 1 +TURN_LEFT = 2 +FORWARD = 3 +FORWARD_RIGHT = 4 +FORWARD_LEFT = 5 + + +def _medikit_box( + frame: np.ndarray, +) -> tuple[float, float, float, float, float] | None: + channel = np.asarray(frame)[..., 0] + height, width = channel.shape + mask = np.where(channel > 150, 255, 0).astype(np.uint8) + mask[: int(height * 0.38), :] = 0 + mask = cv2.morphologyEx( + mask, + cv2.MORPH_CLOSE, + np.ones((3, 5), dtype=np.uint8), + ) + mask = cv2.dilate( + mask, + np.ones((2, 3), dtype=np.uint8), + iterations=1, + ) + count, _, stats, centroids = cv2.connectedComponentsWithStats(mask) + + best: tuple[float, float, float, float, float, float] | None = None + for component in range(1, count): + x, y, box_width, box_height, area = stats[component] + center_x, center_y = centroids[component] + aspect = box_width / max(1, box_height) + if area < 10 or area > 3500: + continue + if y < height * 0.38 or box_height > height * 0.50: + continue + if box_width > width * 0.75 or not 0.35 <= aspect <= 10.0: + continue + if box_width > width * 0.40 and box_height < height * 0.05: + continue + score = float(area) + 0.9 * float(center_y) + if 0.5 < aspect < 6.0: + score += 30.0 + candidate = ( + score, + float(area), + float(center_x), + float(center_y), + float(box_width), + float(box_height), + ) + if best is None or candidate > best: + best = candidate + + if best is None: + return None + _, area, center_x, center_y, box_width, box_height = best + return area, center_x, center_y, box_width, box_height + + +class Policy: + _next_lane = 0 + + def __init__(self) -> None: + self._lane_phase = type(self)._next_lane + type(self)._next_lane += 1 + self.reset() + + def reset(self) -> None: + self.step = 0 + self.last_seen = -999 + self.last_direction = -1 + + def act( + self, + frame: np.ndarray, + variables: Mapping[str, float], + ) -> int: + image = np.asarray(frame) + height, width = image.shape[:2] + health = float(variables["HEALTH"]) + detected = _medikit_box(image) + step = self.step + self.step += 1 + + if detected is None: + if step - self.last_seen < 40: + return TURN_LEFT if self.last_direction < 0 else TURN_RIGHT + phase = (step // 35 + self._lane_phase) % 2 + return TURN_LEFT if phase == 0 else TURN_RIGHT + + area, center_x, center_y, box_width, box_height = detected + offset = center_x - width / 2.0 + self.last_seen = step + self.last_direction = -1 if center_x < width / 2.0 else 1 + close = ( + area > 180.0 + or center_y > height * 0.80 + or box_height > height * 0.12 + or box_width > width * 0.20 + ) + + if close and health > 68.0: + if offset < -16.0: + return TURN_LEFT + if offset > 16.0: + return TURN_RIGHT + return NONE + + margin = 18.0 if close else 12.0 + if offset < -margin: + return FORWARD_LEFT + if offset > margin: + return FORWARD_RIGHT + return FORWARD diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/oracle/solve.sh b/tasks/simulation_heuristics_vizdoom_d1_v1/oracle/solve.sh new file mode 100755 index 0000000..d4a3448 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/oracle/solve.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /app/final_policy +cp /oracle/policy.py /app/final_policy/policy.py + diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/starter_policy/policy.py b/tasks/simulation_heuristics_vizdoom_d1_v1/starter_policy/policy.py new file mode 100644 index 0000000..eda1128 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/starter_policy/policy.py @@ -0,0 +1,50 @@ +"""Weak rendered-pixel starter for VizDoom D1 Basic.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import numpy as np + + +NONE = 0 +TURN_RIGHT = 1 +TURN_LEFT = 2 +FORWARD = 3 +FORWARD_RIGHT = 4 +FORWARD_LEFT = 5 + + +class Policy: + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.step = 0 + + def act( + self, + frame: np.ndarray, + variables: Mapping[str, float], + ) -> int: + self.step += 1 + health = float(variables["HEALTH"]) + if health > 25.0: + return NONE + + image = np.asarray(frame) + height, width = image.shape[:2] + bright = image[..., 0] > 180 + bright[: height // 2] = False + _, columns = np.nonzero(bright) + + if len(columns) < 8: + return TURN_LEFT if (self.step // 45) % 2 == 0 else TURN_RIGHT + + center_x = float(columns.mean()) + offset = center_x - width / 2.0 + if offset < -14.0: + return FORWARD_LEFT + if offset > 14.0: + return FORWARD_RIGHT + return FORWARD diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/task.md b/tasks/simulation_heuristics_vizdoom_d1_v1/task.md new file mode 100644 index 0000000..3f53b4c --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/task.md @@ -0,0 +1,151 @@ +--- +schema_version: "1.3" +task: + name: genesisbench/simulation_heuristics_vizdoom_d1_v1 + description: Improve a rendered-pixel VizDoom D1 medikit policy under a fixed autonomous research budget. + authors: + - name: GenesisBench Contributors + keywords: + - simulation + - heuristics + - vizdoom + - computer-vision + - visual-control +metadata: + category: visual-control + difficulty: medium + tags: + - vizdoom + - screen-cv + - programmatic-policy + - sparse-reward + reference_task: false + genesisbench: + starter: + path: starter_policy + submission: + directory: final_policy + entrypoint: policy.py + development: + episodes: 10 + max_steps: 2100 + seeds: [0] + frame_skip: 1 + render_size: [240, 180] + verifier: + reproduction_config: verifier/config.toml + anchors: verifier/anchors.json + supports_private_config: true +agent: + timeout_sec: 1800 + user: agent + network_mode: public +verifier: + timeout_sec: 300 + user: root + network_mode: no-network +environment: + build_timeout_sec: 1200 + cpus: 4 + memory_mb: 4096 + storage_mb: 10240 + network_mode: public + allow_internet: true + workdir: /app +benchflow: + document_version: "0.6" +--- +# Improve the VizDoom D1 screen policy + +You are given a working but weak programmatic controller for EnvPool +`D1Basic-v1`. Improve it through autonomous coding and repeated development +evaluation. + +## Objective + +Maximize mean episode reward over ten D1 lanes. The useful behavior is to find +the medikit from pixels, stage near it while health is high, and collect it +after health decay makes the pickup valuable. + +The published reference target for the visible seed batch is approximately: + +```text +10-seed mean = 0.9441 +10-seed min = 0.2900 +``` + +The task runtime is pinned to the article's declared EnvPool `1.1.1`. + +## Observable inputs + +At each step your policy receives exactly: + +1. an immutable `uint8` rendered frame with shape `(180, 240, 3)`; +2. an immutable mapping containing only public `HEALTH`. + +It does not receive reward, environment objects, map data, labels, object +coordinates, the EnvPool base seed, or a vector lane identifier. + +## Development evaluation + +Run: + +```bash +python evaluate.py --policy final_policy/policy.py +``` + +For a shorter smoke test: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --episodes 2 \ + --max-steps 200 +``` + +The public evaluator uses batch seed `0`. Final evaluation uses different, +unseen seed batches. + +## Required submission + +Store the best policy at: + +```text +final_policy/policy.py +``` + +It must define `Policy` or `make_policy`: + +```python +class Policy: + def reset(self) -> None: + ... + + def act(self, frame, variables) -> int: + ... +``` + +The returned action must be an integer in `[0, 5]`: + +```text +0 NONE +1 TURN_RIGHT +2 TURN_LEFT +3 FORWARD +4 FORWARD_RIGHT +5 FORWARD_LEFT +``` + +## Rules + +1. Work autonomously and use the fixed wall-clock budget well. +2. Do not modify `evaluate.py`, `_runtime/`, or trusted task files. +3. Do not access `/oracle`, `/verifier`, hidden configs, or hidden seeds. +4. Do not import EnvPool or VizDoom from the policy. +5. Do not read WAD/config files, map geometry, labels, object coordinates, + automaps, or simulator state. +6. Do not hard-code routes or behavior for the public seed batch. +7. The final controller must use rendered pixels plus public `HEALTH` only. +8. The final policy must run offline with the installed packages. +9. Before finishing, evaluate `final_policy/policy.py` and leave the best + working version in place. diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/evaluation.md b/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/evaluation.md new file mode 100644 index 0000000..c0037df --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/evaluation.md @@ -0,0 +1,54 @@ +# Evaluation + +## Public development suite + +The default public command runs one ten-lane EnvPool batch: + +```text +scenario: D1Basic-v1 +EnvPool: 1.1.1 +batch seed: 0 +episodes: 10 +frame skip: 1 +maximum steps: 2,100 +render size: 240 x 180 +``` + +EnvPool's native D1 reward is accumulated without reshaping. The evaluator +reports mean, minimum, and maximum return, per-lane returns and lengths, +invalid-policy rate, final public variables, and action latency. + +## Hidden suite + +The checked-in reproducibility verifier uses seed batches that do not overlap +the public batch. A hosted leaderboard can inject a private config and matching +private anchors. + +Raw score is the weighted mean return across hidden suites. The normalized +GenesisBench score maps: + +- the checked-in starter to `0`; +- the frozen screen-CV reference to `100`. + +Scores above `100` are valid. The checked-in reproducibility anchors are +numeric scores generated from clean, separate EnvPool `1.1.1` processes. The +verifier selects an explicit `darwin-arm64` or `linux-x86_64` profile and +validates its per-suite means against the exact selected config before using +it. Missing platform calibration fails closed. Candidate seed suites run in +fresh subprocesses and unique working directories because the EnvPool `1.1.1` +D1 runtime is not safely re-entrant in one Linux process. Private path-based +anchors remain supported and use the same subprocess isolation. + +The verifier also clears stale D1 state before every suite. Cleanup is limited +to task-owned `/app/evaluate.py`/VizDoom processes and exact `ViZDoom*` +shared-memory artifacts owned by `root` or `agent`; unrelated shared-memory +files are preserved. + +## Failure handling + +A policy receives the configured failure return for an episode if it: + +- fails to import or instantiate; +- returns an invalid action; +- raises during `act`; +- violates the privileged-state source audit. diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/policy_api.md b/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/policy_api.md new file mode 100644 index 0000000..98ae755 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/policy_api.md @@ -0,0 +1,59 @@ +# Policy API + +The evaluator imports `final_policy/policy.py` in a clean Python process. + +Define `Policy` or `make_policy`: + +```python +class Policy: + def __init__(self): + ... + + def reset(self) -> None: + ... + + def act(self, frame, variables) -> int: + ... +``` + +The evaluator creates one policy instance per active EnvPool lane and calls +`reset()` before the first action. + +## Frame + +`frame` is an immutable, C-contiguous NumPy `uint8` array: + +```text +shape = (render_height, render_width, 3) +public default = (180, 240, 3) +``` + +It is the RGB image returned by EnvPool `render()`. Copy it before attempting +in-place edits. + +## Variables + +`variables` is an immutable mapping with exactly one key: + +```text +HEALTH: float +``` + +No reward, seed, lane id, map state, labels, or object coordinates are +included. + +## Actions + +Return a scalar integer: + +| Value | Action | +| ---: | --- | +| 0 | no-op | +| 1 | turn right | +| 2 | turn left | +| 3 | move forward | +| 4 | move forward and right | +| 5 | move forward and left | + +Non-integral, out-of-range, or non-finite actions invalidate the episode. + diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/provenance.md b/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/provenance.md new file mode 100644 index 0000000..31a6e8c --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/task_context/provenance.md @@ -0,0 +1,32 @@ +# Provenance and licensing + +## Behavioral source + +The experiment definition and target metrics come from: + +```text +repository: https://github.com/Trinkle23897/learning-beyond-gradients +revision: 3555c2956c257d49a5015b782cbe485b14fd659e +article section: VizDoom D1 Basic CV Policy +declared runtime: EnvPool 1.1.1 +``` + +The public article describes brightness thresholding, morphology, connected +components, visual alignment, and delayed medikit collection using rendered +pixels plus public `HEALTH`. + +## Reimplementation boundary + +The pinned source repository did not contain a declared license. GenesisBench +therefore does not copy or redistribute its implementation. This task's API, +runtime, starter policy, evaluator, verifier, and compact reference policy are +new code written for this repository from the public behavioral description +and independently reproduced output. + +EnvPool, OpenCV, NumPy, VizDoom runtime assets, and their transitive components +remain governed by their respective upstream licenses. + +The source README explicitly states that the experiments were written against +EnvPool `1.1.1`. Both the source policy and the GenesisBench oracle reproduce +the published ten-seed vector on that version, so no compatibility exception +is used. diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchor_policies/reference_policy.py b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchor_policies/reference_policy.py new file mode 100644 index 0000000..8783d4c --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchor_policies/reference_policy.py @@ -0,0 +1,125 @@ +"""Frozen screen-CV reference anchor for the VizDoom D1 task.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import cv2 +import numpy as np + + +NONE = 0 +TURN_RIGHT = 1 +TURN_LEFT = 2 +FORWARD = 3 +FORWARD_RIGHT = 4 +FORWARD_LEFT = 5 + + +def _medikit_box( + frame: np.ndarray, +) -> tuple[float, float, float, float, float] | None: + channel = np.asarray(frame)[..., 0] + height, width = channel.shape + mask = np.where(channel > 150, 255, 0).astype(np.uint8) + mask[: int(height * 0.38), :] = 0 + mask = cv2.morphologyEx( + mask, + cv2.MORPH_CLOSE, + np.ones((3, 5), dtype=np.uint8), + ) + mask = cv2.dilate( + mask, + np.ones((2, 3), dtype=np.uint8), + iterations=1, + ) + count, _, stats, centroids = cv2.connectedComponentsWithStats(mask) + + best: tuple[float, float, float, float, float, float] | None = None + for component in range(1, count): + x, y, box_width, box_height, area = stats[component] + center_x, center_y = centroids[component] + aspect = box_width / max(1, box_height) + if area < 10 or area > 3500: + continue + if y < height * 0.38 or box_height > height * 0.50: + continue + if box_width > width * 0.75 or not 0.35 <= aspect <= 10.0: + continue + if box_width > width * 0.40 and box_height < height * 0.05: + continue + score = float(area) + 0.9 * float(center_y) + if 0.5 < aspect < 6.0: + score += 30.0 + candidate = ( + score, + float(area), + float(center_x), + float(center_y), + float(box_width), + float(box_height), + ) + if best is None or candidate > best: + best = candidate + + if best is None: + return None + _, area, center_x, center_y, box_width, box_height = best + return area, center_x, center_y, box_width, box_height + + +class Policy: + _next_lane = 0 + + def __init__(self) -> None: + self._lane_phase = type(self)._next_lane + type(self)._next_lane += 1 + self.reset() + + def reset(self) -> None: + self.step = 0 + self.last_seen = -999 + self.last_direction = -1 + + def act( + self, + frame: np.ndarray, + variables: Mapping[str, float], + ) -> int: + image = np.asarray(frame) + height, width = image.shape[:2] + health = float(variables["HEALTH"]) + detected = _medikit_box(image) + step = self.step + self.step += 1 + + if detected is None: + if step - self.last_seen < 40: + return TURN_LEFT if self.last_direction < 0 else TURN_RIGHT + phase = (step // 35 + self._lane_phase) % 2 + return TURN_LEFT if phase == 0 else TURN_RIGHT + + area, center_x, center_y, box_width, box_height = detected + offset = center_x - width / 2.0 + self.last_seen = step + self.last_direction = -1 if center_x < width / 2.0 else 1 + close = ( + area > 180.0 + or center_y > height * 0.80 + or box_height > height * 0.12 + or box_width > width * 0.20 + ) + + if close and health > 68.0: + if offset < -16.0: + return TURN_LEFT + if offset > 16.0: + return TURN_RIGHT + return NONE + + margin = 18.0 if close else 12.0 + if offset < -margin: + return FORWARD_LEFT + if offset > margin: + return FORWARD_RIGHT + return FORWARD diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchor_policies/starter_policy.py b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchor_policies/starter_policy.py new file mode 100644 index 0000000..af863e4 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchor_policies/starter_policy.py @@ -0,0 +1,50 @@ +"""Frozen weak rendered-pixel anchor for VizDoom D1 Basic.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import numpy as np + + +NONE = 0 +TURN_RIGHT = 1 +TURN_LEFT = 2 +FORWARD = 3 +FORWARD_RIGHT = 4 +FORWARD_LEFT = 5 + + +class Policy: + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.step = 0 + + def act( + self, + frame: np.ndarray, + variables: Mapping[str, float], + ) -> int: + self.step += 1 + health = float(variables["HEALTH"]) + if health > 25.0: + return NONE + + image = np.asarray(frame) + height, width = image.shape[:2] + bright = image[..., 0] > 180 + bright[: height // 2] = False + _, columns = np.nonzero(bright) + + if len(columns) < 8: + return TURN_LEFT if (self.step // 45) % 2 == 0 else TURN_RIGHT + + center_x = float(columns.mean()) + offset = center_x - width / 2.0 + if offset < -14.0: + return FORWARD_LEFT + if offset > 14.0: + return FORWARD_RIGHT + return FORWARD diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchors.json b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchors.json new file mode 100644 index 0000000..2dbf2ff --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/anchors.json @@ -0,0 +1,98 @@ +{ + "environment": "EnvPool 1.1.1 D1Basic-v1, rendered RGB pixels plus HEALTH only", + "calibrations": { + "darwin-arm64": { + "envpool_version": "1.1.1", + "evaluation": { + "scenario": "d1", + "max_steps": 2100, + "frame_skip": 1, + "render_width": 240, + "render_height": 180, + "failure_return": -1.0, + "suites": [ + { + "name": "hidden_nominal", + "weight": 0.7, + "seed": 101, + "episodes": 5 + }, + { + "name": "hidden_secondary", + "weight": 0.3, + "seed": 607, + "episodes": 5 + } + ] + }, + "reference_policy": { + "score": 0.6982399759441613, + "suite_mean_returns": { + "hidden_nominal": 0.8401999741792678, + "hidden_secondary": 0.3669999800622463 + } + }, + "starter_policy": { + "score": 0.44161998964846133, + "suite_mean_returns": { + "hidden_nominal": 0.5395999886095524, + "hidden_secondary": 0.21299999207258224 + } + } + }, + "linux-x86_64": { + "envpool_version": "1.1.1", + "evaluation": { + "scenario": "d1", + "max_steps": 2100, + "frame_skip": 1, + "render_width": 240, + "render_height": 180, + "failure_return": -1.0, + "suites": [ + { + "name": "hidden_nominal", + "weight": 0.7, + "seed": 101, + "episodes": 5 + }, + { + "name": "hidden_secondary", + "weight": 0.3, + "seed": 607, + "episodes": 5 + } + ] + }, + "reference_policy": { + "score": 0.8178199748694897, + "suite_mean_returns": { + "hidden_nominal": 0.9847999721765518, + "hidden_secondary": 0.42819998115301133 + } + }, + "starter_policy": { + "score": 0.44161998964846133, + "suite_mean_returns": { + "hidden_nominal": 0.5395999886095524, + "hidden_secondary": 0.21299999207258224 + } + } + } + }, + "public_article_target": { + "envpool_version": "1.1.1", + "batch_seed": 0, + "episodes": 10, + "mean_return": 0.9440999741666019, + "min_return": 0.28999998047947884 + }, + "reference_policy": { + "description": "Frozen morphology and connected-components medikit policy", + "path": "anchor_policies/reference_policy.py" + }, + "starter_policy": { + "description": "Public weak brightness-centroid starter", + "path": "anchor_policies/starter_policy.py" + } +} diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/config.toml b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/config.toml new file mode 100644 index 0000000..08f1e86 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/config.toml @@ -0,0 +1,22 @@ +version = "1.0" + +[evaluation] +scenario = "d1" +max_steps = 2100 +frame_skip = 1 +render_width = 240 +render_height = 180 +failure_return = -1.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.7 +seed = 101 +episodes = 5 + +[[evaluation.suites]] +name = "hidden_secondary" +weight = 0.3 +seed = 607 +episodes = 5 + diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/evaluate_hidden.py new file mode 100755 index 0000000..6a8d0ac --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/evaluate_hidden.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import os +import platform +import pwd +import signal +import subprocess +import sys +import tempfile +import time +import tomllib +from pathlib import Path +from typing import Any + + +VERIFIER_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] +RUNTIME_CANDIDATES.extend( + ancestor / "src" for ancestor in VERIFIER_DIR.parents +) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.vizdoom import ( # noqa: E402 + VIZDOOM_ARTICLE_ENVPOOL_VERSION, + evaluate_vizdoom_policy, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run hidden VizDoom D1 evaluation." + ) + parser.add_argument("policy", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument( + "--config", + type=Path, + default=VERIFIER_DIR / "config.toml", + help="Evaluation suite config; production can inject a private file.", + ) + parser.add_argument( + "--anchors", + type=Path, + default=VERIFIER_DIR / "anchors.json", + help="Normalization anchors matching the selected evaluation suite.", + ) + parser.add_argument( + "--worker-request", + type=Path, + help=argparse.SUPPRESS, + ) + return parser.parse_args() + + +def _worker_main( + policy: Path, + *, + request_path: Path, + output_path: Path | None, +) -> None: + if output_path is None: + raise ValueError("--output is required in worker mode") + request = json.loads(request_path.read_text()) + evaluation = request["evaluation"] + suite = request["suite"] + result = evaluate_vizdoom_policy( + policy, + scenario=evaluation["scenario"], + seed=suite["seed"], + episodes=suite["episodes"], + max_steps=evaluation["max_steps"], + frame_skip=evaluation["frame_skip"], + render_width=evaluation["render_width"], + render_height=evaluation["render_height"], + failure_return=evaluation["failure_return"], + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(result.to_json() + "\n") + + +def _safe_name(value: str) -> str: + rendered = "".join( + character if character.isalnum() or character in "-_" else "-" + for character in value + ).strip("-") + return rendered or "suite" + + +def _resource_roots() -> tuple[Path, ...]: + override = os.environ.get("GENESISBENCH_VIZDOOM_RESOURCE_ROOTS") + if override: + return tuple( + Path(value) + for value in override.split(os.pathsep) + if value + ) + return (Path("/dev/shm"), Path("/tmp/boost_interprocess")) + + +def _is_vizdoom_resource(name: str) -> bool: + lowered = name.lower() + return lowered.startswith("vizdoom") or lowered.startswith("sem.vizdoom") + + +def _task_owned_uids() -> set[int]: + owners = {os.geteuid()} + try: + owners.add(pwd.getpwnam("agent").pw_uid) + except KeyError: + pass + return owners + + +def _task_owned_stale_processes() -> list[int]: + if os.geteuid() != 0 or not Path("/app").is_dir(): + return [] + stale: list[int] = [] + current_pid = os.getpid() + for process_dir in Path("/proc").iterdir(): + if not process_dir.name.isdigit(): + continue + pid = int(process_dir.name) + if pid == current_pid: + continue + try: + command = ( + (process_dir / "cmdline") + .read_bytes() + .replace(b"\0", b" ") + .decode(errors="replace") + ) + cwd = Path(os.readlink(process_dir / "cwd")) + comm = (process_dir / "comm").read_text().strip().lower() + except (FileNotFoundError, PermissionError, ProcessLookupError): + continue + if not cwd.is_relative_to("/app"): + continue + is_vizdoom = comm == "vizdoom" or "/vizdoom/bin/vizdoom" in command + is_public_evaluator = ( + "python" in command + and ( + "/app/evaluate.py" in command + or " evaluate.py " in f" {command} " + ) + ) + if is_vizdoom or is_public_evaluator: + stale.append(pid) + return stale + + +def _terminate_processes(pids: list[int]) -> int: + if not pids: + return 0 + for pid in pids: + try: + os.kill(pid, signal.SIGTERM) + except (PermissionError, ProcessLookupError): + pass + deadline = time.monotonic() + 1.0 + remaining = set(pids) + while remaining and time.monotonic() < deadline: + remaining = {pid for pid in remaining if Path(f"/proc/{pid}").exists()} + if remaining: + time.sleep(0.05) + for pid in remaining: + try: + os.kill(pid, signal.SIGKILL) + except (PermissionError, ProcessLookupError): + pass + return len(pids) + + +def _remove_vizdoom_resources() -> tuple[int, int]: + removed = 0 + removed_bytes = 0 + allowed_uids = _task_owned_uids() + overridden = "GENESISBENCH_VIZDOOM_RESOURCE_ROOTS" in os.environ + for root in _resource_roots(): + if not root.is_dir(): + continue + for resource in root.iterdir(): + if not _is_vizdoom_resource(resource.name): + continue + try: + metadata = resource.lstat() + except FileNotFoundError: + continue + if not overridden and metadata.st_uid not in allowed_uids: + continue + try: + if resource.is_dir() and not resource.is_symlink(): + resource.rmdir() + else: + resource.unlink() + except (FileNotFoundError, OSError): + continue + removed += 1 + removed_bytes += metadata.st_size + return removed, removed_bytes + + +def _cleanup_stale_vizdoom_state() -> dict[str, int]: + stale_processes = _task_owned_stale_processes() + terminated = _terminate_processes(stale_processes) + removed, removed_bytes = _remove_vizdoom_resources() + return { + "terminated_processes": terminated, + "removed_resources": removed, + "removed_bytes": removed_bytes, + } + + +def _evaluate_suite_isolated( + policy: Path, + *, + evaluation: dict[str, Any], + suite: dict[str, Any], + workspace_root: Path, + run_label: str, + suite_index: int, +) -> tuple[dict[str, Any], dict[str, int]]: + cleanup = _cleanup_stale_vizdoom_state() + workspace = ( + workspace_root + / f"{_safe_name(run_label)}-{suite_index:02d}-{_safe_name(suite['name'])}" + ) + workspace.mkdir(parents=True, exist_ok=False) + request_path = workspace / "request.json" + output_path = workspace / "result.json" + request_path.write_text( + json.dumps( + { + "evaluation": evaluation, + "suite": suite, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + completed = subprocess.run( + [ + sys.executable, + str(Path(__file__).resolve()), + str(policy.resolve()), + "--worker-request", + str(request_path), + "--output", + str(output_path), + ], + cwd=workspace, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + "Isolated VizDoom suite evaluation failed " + f"for {run_label}/{suite['name']} with rc={completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + if not output_path.is_file(): + raise RuntimeError( + f"Isolated VizDoom suite did not write {output_path}" + ) + return json.loads(output_path.read_text()), cleanup + + +def _evaluate_raw( + policy: Path, + *, + evaluation: dict[str, Any], + workspace_root: Path, + run_label: str, +) -> tuple[dict[str, dict[str, Any]], float, list[dict[str, int]]]: + results: dict[str, dict[str, Any]] = {} + cleanup_runs: list[dict[str, int]] = [] + score = 0.0 + for suite_index, suite in enumerate(evaluation["suites"]): + result, cleanup = _evaluate_suite_isolated( + policy, + evaluation=evaluation, + suite=suite, + workspace_root=workspace_root, + run_label=run_label, + suite_index=suite_index, + ) + cleanup_runs.append(cleanup) + results[suite["name"]] = result + score += float(suite["weight"]) * float(result["mean_return"]) + return results, score, cleanup_runs + + +def _validate_numeric_anchor_calibration( + anchors: dict[str, Any], + *, + evaluation: dict[str, Any], +) -> tuple[str, dict[str, Any] | None]: + machine = platform.machine().lower() + machine = { + "amd64": "x86_64", + "aarch64": "arm64", + }.get(machine, machine) + platform_key = f"{sys.platform}-{machine}" + calibrations = anchors.get("calibrations") + if calibrations is None: + return platform_key, None + if not isinstance(calibrations, dict): + raise ValueError("anchor calibrations must be a mapping") + calibration = calibrations.get(platform_key) + if not isinstance(calibration, dict): + raise ValueError( + f"no numeric anchor calibration for platform {platform_key}" + ) + if ( + calibration.get("envpool_version") + != VIZDOOM_ARTICLE_ENVPOOL_VERSION + ): + raise ValueError("numeric anchors use the wrong EnvPool version") + if calibration.get("evaluation") != evaluation: + raise ValueError( + "numeric anchors do not match the selected hidden evaluation" + ) + for name in ("starter_policy", "reference_policy"): + numeric_anchor = calibration.get(name) + if not isinstance(numeric_anchor, dict) or not isinstance( + numeric_anchor.get("score"), + int | float, + ): + raise ValueError( + f"platform calibration must declare numeric {name}" + ) + return platform_key, calibration + + +def _anchor_score( + anchors: dict[str, Any], + name: str, + *, + anchors_path: Path, + evaluation: dict[str, Any], + workspace_root: Path, + calibration: dict[str, Any] | None, +) -> float: + anchor = anchors[name] + numeric_anchor = calibration.get(name) if calibration is not None else None + score = ( + numeric_anchor.get("score") + if isinstance(numeric_anchor, dict) + else None + ) + if isinstance(score, int | float): + suite_means = numeric_anchor.get("suite_mean_returns") + if not isinstance(suite_means, dict): + raise ValueError( + f"numeric anchor {name} must declare suite_mean_returns" + ) + suite_names = {suite["name"] for suite in evaluation["suites"]} + if set(suite_means) != suite_names: + raise ValueError( + f"numeric anchor {name} has mismatched hidden suites" + ) + derived_score = sum( + float(suite["weight"]) * float(suite_means[suite["name"]]) + for suite in evaluation["suites"] + ) + if not math.isclose( + derived_score, + float(score), + rel_tol=0.0, + abs_tol=1e-9, + ): + raise ValueError( + f"numeric anchor {name} score does not match suite means" + ) + return float(score) + relative_path = anchor.get("path") + if not isinstance(relative_path, str) or not relative_path: + raise ValueError(f"{name} must declare score or path") + policy = anchors_path.parent / relative_path + _, calibrated_score, _ = _evaluate_raw( + policy, + evaluation=evaluation, + workspace_root=workspace_root, + run_label=f"anchor-{name}", + ) + return calibrated_score + + +def main() -> None: + args = parse_args() + if args.worker_request is not None: + _worker_main( + args.policy, + request_path=args.worker_request, + output_path=args.output, + ) + return + + config = tomllib.loads(args.config.read_text()) + anchors = json.loads(args.anchors.read_text()) + evaluation = config["evaluation"] + calibration_key, calibration = _validate_numeric_anchor_calibration( + anchors, + evaluation=evaluation, + ) + with tempfile.TemporaryDirectory( + prefix="genesisbench-vizdoom-d1-hidden-" + ) as temporary_directory: + workspace_root = Path(temporary_directory) + suites, score, cleanup_runs = _evaluate_raw( + args.policy, + evaluation=evaluation, + workspace_root=workspace_root, + run_label="candidate", + ) + starter_score = _anchor_score( + anchors, + "starter_policy", + anchors_path=args.anchors, + evaluation=evaluation, + workspace_root=workspace_root, + calibration=calibration, + ) + reference_score = _anchor_score( + anchors, + "reference_policy", + anchors_path=args.anchors, + evaluation=evaluation, + workspace_root=workspace_root, + calibration=calibration, + ) + if reference_score == starter_score: + raise ValueError("starter and reference anchors must have different scores") + normalized_score = round( + 100.0 * (score - starter_score) / (reference_score - starter_score), + 9, + ) + payload = { + "score": score, + "normalized_score": normalized_score, + "starter_score": starter_score, + "reference_score": reference_score, + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": len(evaluation["suites"]), + "unique_working_directories": True, + }, + "anchor_calibration": { + "mode": "numeric" if calibration is not None else "path", + "platform": calibration_key, + "envpool_version": VIZDOOM_ARTICLE_ENVPOOL_VERSION, + }, + "resource_cleanup": { + "runs": cleanup_runs, + "terminated_processes": sum( + run["terminated_processes"] for run in cleanup_runs + ), + "removed_resources": sum( + run["removed_resources"] for run in cleanup_runs + ), + "removed_bytes": sum( + run["removed_bytes"] for run in cleanup_runs + ), + }, + "suites": suites, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/rubrics/verifier.md new file mode 100644 index 0000000..ad187d4 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/rubrics/verifier.md @@ -0,0 +1,5 @@ +# Simulation Heuristics VizDoom D1 v1 Verifier Rubric + +- `task_success`: normalized hidden-suite D1 return, where the public weak + starter maps to `0` and the frozen screen-CV reference maps to `1`. + diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/rubrics/verifier.toml b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/rubrics/verifier.toml new file mode 100644 index 0000000..dadf0e2 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/rubrics/verifier.toml @@ -0,0 +1,10 @@ +version = "0.1" + +[[criteria]] +id = "task_success" +description = "Normalized hidden-suite VizDoom D1 return." +weight = 1.0 + +[scoring] +method = "weighted_mean" + diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/test.sh b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/test.sh new file mode 100755 index 0000000..0d01f3d --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/test.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier +policy=/app/final_policy/policy.py + +if [ ! -f "$policy" ]; then + printf '0.0\n' > /logs/verifier/reward.txt + printf '{"reward": 0.0}\n' > /logs/verifier/reward.json + exit 0 +fi + +python /verifier/evaluate_hidden.py \ + "$policy" \ + --config /verifier/config.toml \ + --anchors /verifier/anchors.json \ + --output /logs/verifier/genesis-score.json + +python - <<'PY' +import json +from pathlib import Path + +score = json.loads(Path("/logs/verifier/genesis-score.json").read_text()) +reward = max(0.0, min(1.0, float(score["normalized_score"]) / 100.0)) +Path("/logs/verifier/reward.txt").write_text(f"{reward}\n") +Path("/logs/verifier/reward.json").write_text( + json.dumps({"reward": reward}, indent=2) + "\n" +) +PY + diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/verifier.md b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/verifier.md new file mode 100644 index 0000000..acad186 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/verifier/verifier.md @@ -0,0 +1,44 @@ +--- +document_version: "0.3" +verifier: + name: simulation_heuristics_vizdoom_d1_v1 + default_strategy: deterministic + strategies: + deterministic: + type: script + command: ./test.sh + rubric: + combine: weighted_mean + dimensions: + task_success: + weight: 1.0 + source: deterministic + outputs: + reward_text: /logs/verifier/reward.txt + reward_json: /logs/verifier/reward.json + details_json: /logs/verifier/genesis-score.json + aggregate_policy: + method: weighted_mean + metrics: + task_success: 1.0 +--- + +## Verifier intent + +Run the submitted rendered-pixel D1 policy on hidden EnvPool seed batches. +Only the frame and public `HEALTH` value cross the trusted policy boundary. + +BenchFlow's canonical reward is the locally normalized GenesisBench score +clamped to `[0, 1]`. Raw returns, anchor scores, invalid-policy details, and +per-suite metrics remain available in `genesis-score.json`. + +Each candidate suite runs in a fresh subprocess and unique working directory. +This prevents EnvPool `1.1.1` from reusing its process-global D1 runtime or +colliding on `./_vizdoom`. Checked-in numeric anchors include exact config and +per-suite calibration metadata for `darwin-arm64` and `linux-x86_64`; +unsupported or mismatched platform calibration fails closed. + +Before each suite, the root verifier terminates stale task-owned public +evaluator/VizDoom processes and removes only exact task-owned `ViZDoom*` +shared-memory artifacts. Cleanup counts and reclaimed bytes are recorded in +`genesis-score.json`. diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/README.md b/tasks/simulation_heuristics_vizdoom_d3_v1/README.md new file mode 100644 index 0000000..a64c498 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/README.md @@ -0,0 +1,82 @@ +# Simulation Heuristics VizDoom D3 v1 + +This task packages the D3 Battle experiment from *Learning Beyond Gradients* +as a native GenesisBench policy-improvement benchmark. + +The public controller boundary contains only: + +- the current three-channel screen image; +- public `HEALTH`, `AMMO2`, `HITCOUNT`, `DAMAGECOUNT`, and `KILLCOUNT`; +- an eight-value action vector for attack, movement, and turning. + +Map files, object labels, coordinates, automaps, environment objects, seeds, +and vector lane ids never cross the boundary. Policy source is audited for +direct simulator and privileged-map access. + +## Article reproduction target + +With EnvPool `1.1.1`, batch seed `0`, ten lanes, frame skip `2`, `640 x 480` +screen input, and reward `DAMAGECOUNT + 10 * KILLCOUNT`, the frozen reference +policy reproduces: + +```text +mean = 557.0 +min = 440.0 +rewards = [545, 475, 480, 440, 690, 500, 600, 595, 530, 715] +``` + +## Task flow + +```text +starter_policy/policy.py +→ public evaluate.py +→ prepared final_policy/policy.py +→ hidden unseen-seed evaluation +→ local starter/reference normalization +``` + +The shared task preparer creates `final_policy/policy.py` from the starter in +the agent workspace. A checked-in `final_policy/` would conflict with that +native preparation step, so the source package keeps only the starter and +oracle copies. + +## Hidden verifier isolation + +EnvPool `1.1.1` cannot safely create multiple D3 VizDoom instances +sequentially in one Linux verifier process and working directory. The hidden +evaluator therefore runs each seed suite in a fresh subprocess with a unique +working directory. + +The checked-in starter/reference scores were calibrated in clean, separate +EnvPool `1.1.1` runs. Their per-suite means and the exact hidden config are +stored in `verifier/anchors.json`; the verifier rejects numeric anchors when +that calibration metadata drifts. + +## Run locally + +```bash +python tasks/simulation_heuristics_vizdoom_d3_v1/evaluate.py \ + --policy tasks/simulation_heuristics_vizdoom_d3_v1/oracle/policy.py +``` + +Validate the native task package: + +```bash +uv run bench tasks check \ + tasks/simulation_heuristics_vizdoom_d3_v1 \ + --level publication-grade +``` + +## Provenance and licensing + +The behavioral contract and target vector come from +`Trinkle23897/learning-beyond-gradients` commit +`3555c2956c257d49a5015b782cbe485b14fd659e`. +The source repository's runtime notes explicitly pin the experiments to +EnvPool `1.1.1`; the task Docker image and evaluator enforce the same version. + +The pinned source repository did not declare a license. This package therefore +does not vendor its implementation. The GenesisBench runtime, API, starter, +and minimized reference controller are independently written from the public +experiment description and reproduced behavior. Third-party runtime packages +retain their own licenses. diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/environment/Dockerfile b/tasks/simulation_heuristics_vizdoom_d3_v1/environment/Dockerfile new file mode 100644 index 0000000..81dfbb1 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/environment/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.12-slim + +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends \ + bash \ + ca-certificates \ + libgl1 \ + libglib2.0-0 \ + libx11-6 \ + libxext6 \ + libxrender1 \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir \ + "envpool==1.1.1" \ + "numpy>=1.26,<2.3" \ + "opencv-python-headless==4.12.0.88" \ + "pyyaml>=6,<7" + +RUN useradd --create-home --uid 1000 agent \ + && mkdir -p /app /opt/genesisbench + +COPY src/genesisbench /opt/genesisbench/genesisbench +COPY tasks/simulation_heuristics_vizdoom_d3_v1/evaluate.py /app/evaluate.py +COPY tasks/simulation_heuristics_vizdoom_d3_v1/starter_policy /app/starter_policy +COPY tasks/simulation_heuristics_vizdoom_d3_v1/task_context /app/task_context + +RUN cp -R /app/starter_policy /app/final_policy \ + && chown -R root:root /opt/genesisbench \ + && chmod -R a-w /opt/genesisbench \ + && chown -R agent:agent /app + +ENV PYTHONPATH="/opt/genesisbench" +ENV SDL_VIDEODRIVER="dummy" + +WORKDIR /app + +RUN mkdir -p /logs/verifier /logs/agent /logs/artifacts diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/evaluate.py b/tasks/simulation_heuristics_vizdoom_d3_v1/evaluate.py new file mode 100755 index 0000000..5513a5a --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/evaluate.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +TASK_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = ( + TASK_DIR / "_runtime", + TASK_DIR.parent / "src", + TASK_DIR.parent.parent / "src", +) +for candidate in RUNTIME_CANDIDATES: + if candidate.is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.vizdoom import evaluate_vizdoom_policy # noqa: E402 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Evaluate a screen-CV VizDoom D3 battle policy." + ) + parser.add_argument( + "--policy", + type=Path, + default=Path("final_policy/policy.py"), + ) + parser.add_argument("--episodes", type=int, default=10) + parser.add_argument("--max-steps", type=int, default=1050) + parser.add_argument("--frame-skip", type=int, default=2) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--render-width", type=int, default=640) + parser.add_argument("--render-height", type=int, default=480) + parser.add_argument("--json-output-file", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + result = evaluate_vizdoom_policy( + args.policy, + scenario="d3", + seed=args.seed, + episodes=args.episodes, + max_steps=args.max_steps, + frame_skip=args.frame_skip, + render_width=args.render_width, + render_height=args.render_height, + failure_return=0.0, + ) + rendered = result.to_json() + print(rendered) + if args.json_output_file is not None: + args.json_output_file.parent.mkdir(parents=True, exist_ok=True) + args.json_output_file.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/oracle/policy.py b/tasks/simulation_heuristics_vizdoom_d3_v1/oracle/policy.py new file mode 100644 index 0000000..ec12c50 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/oracle/policy.py @@ -0,0 +1,471 @@ +"""Minimized screen-CV reference for the VizDoom D3 article experiment. + +This is an independent GenesisBench implementation of the published behavior. +No upstream source file is vendored. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +import cv2 +import numpy as np + + +def _move( + *, + attack: float = 0.0, + speed: float = 0.0, + forward: float = 0.0, + backward: float = 0.0, + right: float = 0.0, + left: float = 0.0, + turn180: float = 0.0, + turn: float = 0.0, +) -> np.ndarray: + return np.asarray( + [attack, speed, forward, backward, right, left, turn180, turn], + dtype=np.float64, + ) + + +def _enemy_box( + image: np.ndarray, +) -> tuple[float, float, float, float, float] | None: + height, width = image.shape[:2] + scale_x = width / 320.0 + scale_y = height / 240.0 + center = width / 2.0 + channels = [ + image[..., channel].astype(np.int16) for channel in range(3) + ] + primary, secondary, tertiary = channels + enemy_pixels = ( + (primary > 50) + & ((primary - np.maximum(secondary, tertiary)) > 25) + & (secondary < 55) + & (tertiary < 55) + ) + mask = enemy_pixels.astype(np.uint8) * 255 + mask[: int(40 * scale_y), :] = 0 + mask[int(205 * scale_y) :, :] = 0 + closing_size = max(2, int(round(4 * scale_x))) + mask = cv2.morphologyEx( + mask, + cv2.MORPH_CLOSE, + np.ones((closing_size, closing_size), dtype=np.uint8), + ) + mask = cv2.dilate( + mask, + np.ones( + ( + max(2, int(4 * scale_y)), + max(2, int(3 * scale_x)), + ), + dtype=np.uint8, + ), + iterations=1, + ) + count, _, stats, centroids = cv2.connectedComponentsWithStats(mask, 8) + best: tuple[float, float, float, float, float] | None = None + best_score = -1.0 + for component in range(1, count): + _, _, box_width, box_height, area = stats[component] + center_x, center_y = centroids[component] + if area < 12 * scale_x * scale_y: + continue + if box_height < 7 * scale_y: + continue + if box_width > 95 * scale_x or box_height > 115 * scale_y: + continue + score = float(area) * ( + float(box_height) / (16 * scale_y) + ) ** 1.25 + score *= 1.0 + ( + float(center_y) - 90 * scale_y + ) / (180 * scale_y) + score /= 1.0 + abs(float(center_x) - center) / (80 * scale_x) + if score > best_score: + best_score = score + best = ( + float(center_x), + float(center_y), + float(box_width), + float(box_height), + float(area), + ) + return best + + +def _supply_box( + image: np.ndarray, + *, + ammo: float, + health: float, +) -> tuple[float, float, float, float, float] | None: + height, width = image.shape[:2] + scale_x = width / 320.0 + scale_y = height / 240.0 + center = width / 2.0 + first = image[..., 0].astype(np.int16) + second = image[..., 1].astype(np.int16) + third = image[..., 2].astype(np.int16) + enemy = ( + (first > 50) + & ((first - np.maximum(second, third)) > 25) + & (second < 55) + & (third < 55) + ) + + medikit_threshold = 95 if ammo <= 5 or health <= -1 else 115 + medikit = ( + (first > medikit_threshold) + & (second > 75) + & (third > 70) + ) + clip = (first > 90) & (second > 60) & ((first - third) > 25) + bright_mask = ((medikit | clip) & (~enemy)).astype(np.uint8) * 255 + bright_mask[: int(50 * scale_y), :] = 0 + bright_mask[int(218 * scale_y) :, :] = 0 + bright_mask = cv2.morphologyEx( + bright_mask, + cv2.MORPH_CLOSE, + np.ones( + ( + max(2, int(3 * scale_y)), + max(2, int(3 * scale_x)), + ), + dtype=np.uint8, + ), + ) + count, _, stats, centroids = cv2.connectedComponentsWithStats( + bright_mask, + 8, + ) + best: tuple[float, float, float, float, float] | None = None + best_score = -1.0 + for component in range(1, count): + _, _, box_width, box_height, area = stats[component] + center_x, center_y = centroids[component] + if area < 10 * scale_x * scale_y: + continue + if area > 3000 * scale_x * scale_y: + continue + if box_width > 60 * scale_x or box_height > 52 * scale_y: + continue + if box_width < 2 * scale_x or box_height < 2 * scale_y: + continue + score = float(area) * ( + 1.0 + (float(center_y) - 85 * scale_y) / (130 * scale_y) + ) + score /= 1.0 + abs(float(center_x) - center) / (85 * scale_x) + if score > best_score: + best_score = score + best = ( + float(center_x), + float(center_y), + float(box_width), + float(box_height), + float(area), + ) + if best is not None: + return best + + color_difference = np.maximum.reduce( + [ + np.abs(first - second), + np.abs(first - third), + np.abs(second - third), + ] + ) + fallback_mask = ( + (color_difference > 24) & (~enemy) + ).astype(np.uint8) * 255 + fallback_mask[: int(55 * scale_y), :] = 0 + fallback_mask[int(205 * scale_y) :, :] = 0 + fallback_mask = cv2.morphologyEx( + fallback_mask, + cv2.MORPH_CLOSE, + np.ones( + ( + max(2, int(3 * scale_y)), + max(2, int(3 * scale_x)), + ), + dtype=np.uint8, + ), + ) + count, _, stats, centroids = cv2.connectedComponentsWithStats( + fallback_mask, + 8, + ) + best = None + best_score = -1.0 + for component in range(1, count): + _, _, box_width, box_height, area = stats[component] + center_x, center_y = centroids[component] + if area < 8 * scale_x * scale_y: + continue + if area > 700 * scale_x * scale_y: + continue + if box_width > 52 * scale_x or box_height > 42 * scale_y: + continue + score = float(area) * ( + 1.0 + (float(center_y) - 80 * scale_y) / (140 * scale_y) + ) + score /= 1.0 + abs(float(center_x) - center) / (90 * scale_x) + if score > best_score: + best_score = score + best = ( + float(center_x), + float(center_y), + float(box_width), + float(box_height), + float(area), + ) + return best + + +def _view_key(image: np.ndarray) -> bytes: + gray = image.astype(np.uint8).mean(axis=2).astype(np.uint8) + small = cv2.resize(gray, (8, 6), interpolation=cv2.INTER_AREA) + return (small > small.mean()).astype(np.uint8).tobytes() + + +class Policy: + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.step = 0 + self.damage = 0.0 + self.hit = 0.0 + self.health = 100.0 + self.lock = 0 + self.bad_lock_steps = 0 + self.last_horizontal_error = 0.0 + self.search_direction = 1.0 + self.panic_steps = 0 + self.previous_frame: np.ndarray | None = None + self.stuck_steps = 0 + self.last_progress_step = 0 + self.adapted = False + self.adapted_arc: float | None = None + self.adapted_bored_after: int | None = None + self.adapted_bored_turn: int | None = None + self.escape_index = 0 + self.recent_views: list[bytes] = [] + + def _novelty_escape(self) -> np.ndarray: + self.escape_index += 1 + fraction = (self.escape_index * 0.6180339887498949) % 1.0 + magnitude = 3.0 + fraction * 7.0 + self.search_direction *= -1.0 + return _move( + speed=1.0, + backward=1.0, + turn=self.search_direction * magnitude, + ) + + def act( + self, + frame: np.ndarray, + variables: Mapping[str, float], + ) -> np.ndarray: + image = np.asarray(frame) + height, width = image.shape[:2] + scale_x = width / 320.0 + scale_y = height / 240.0 + screen_center = width / 2.0 + ammo = float(variables["AMMO2"]) + health = float(variables["HEALTH"]) + damage = float(variables["DAMAGECOUNT"]) + hit = float(variables["HITCOUNT"]) + step = self.step + self.step += 1 + + if damage > self.damage or hit > self.hit: + self.lock = 10 + self.bad_lock_steps = 0 + self.last_progress_step = step + elif self.lock > 0: + self.bad_lock_steps += 1 + if health < self.health - 0.1: + self.panic_steps = 12 + self.search_direction *= -1.0 + self.damage = damage + self.hit = hit + self.health = health + + enemy = _enemy_box(image) + if enemy is not None and ammo > 0: + center_x, _, _, box_height, area = enemy + horizontal_error = (center_x - screen_center) / scale_x + self.last_horizontal_error = horizontal_error + self.lock = max(self.lock, 4) + turn = float( + np.clip(horizontal_error * 0.18, -8.0, 8.0) + ) + if abs(horizontal_error) > 4.0: + danger = ( + area > 220 * scale_x * scale_y + or box_height > 29 * scale_y + ) + if danger: + if horizontal_error < 0: + return _move( + speed=1.0, + backward=1.0, + right=1.0, + turn=turn, + ) + return _move( + speed=1.0, + backward=1.0, + left=1.0, + turn=turn, + ) + return _move(speed=1.0, forward=1.0, turn=turn) + + strafe_right = (step // 8) % 2 == 1 + if ( + area > 380 * scale_x * scale_y + or box_height > 42 * scale_y + or self.panic_steps > 0 + ): + self.panic_steps = max(0, self.panic_steps - 1) + return _move( + attack=1.0, + speed=1.0, + backward=1.0, + right=1.0 if strafe_right else 0.0, + left=0.0 if strafe_right else 1.0, + turn=turn, + ) + return _move( + attack=1.0, + speed=1.0, + right=1.0 if strafe_right else 0.0, + left=0.0 if strafe_right else 1.0, + turn=turn, + ) + + if self.lock > 0 and ammo > 0: + self.lock -= 1 + turn = float( + np.clip( + self.last_horizontal_error * 0.18, + -8.0, + 8.0, + ) + ) + if ( + abs(self.last_horizontal_error) <= 4.0 + and self.bad_lock_steps < 4 + ): + return _move(attack=1.0, turn=turn) + if self.bad_lock_steps >= 4: + self.search_direction *= -1.0 + self.bad_lock_steps = 0 + if abs(self.last_horizontal_error) > 4.0: + return _move(speed=1.0, turn=turn) + return _move( + speed=1.0, + turn=self.search_direction * 6.0, + ) + + if self.panic_steps > 0: + self.panic_steps -= 1 + return _move( + speed=1.0, + backward=1.0, + turn=self.search_direction * 6.0, + ) + + if ammo <= 10: + supply = _supply_box(image, ammo=ammo, health=health) + if supply is not None: + center_x = supply[0] + horizontal_error = (center_x - screen_center) / scale_x + turn = float( + np.clip(horizontal_error * 0.18, -8.0, 8.0) + ) + return _move(speed=1.0, forward=1.0, turn=turn) + + key = _view_key(image) + repeated = key in self.recent_views + self.recent_views.append(key) + if len(self.recent_views) > 80: + self.recent_views.pop(0) + if repeated and step - self.last_progress_step > 160: + return self._novelty_escape() + + if self.previous_frame is not None: + difference = float( + np.mean( + np.abs( + image.astype(np.int16) + - self.previous_frame.astype(np.int16) + ) + ) + ) + if difference < 0.9: + self.stuck_steps += 1 + else: + self.stuck_steps = max(0, self.stuck_steps - 1) + self.previous_frame = image.copy() + if self.stuck_steps > 10: + self.stuck_steps = 0 + self.search_direction *= -1.0 + return _move( + speed=1.0, + backward=1.0, + turn=self.search_direction * 6.0, + ) + + if not self.adapted and step >= 250: + low_progress = damage <= 15 and health >= 90 + danger_progress = health <= 75 and damage >= 60 + if low_progress or danger_progress: + self.adapted_arc = 1.4 + self.adapted_bored_after = 180 + self.adapted_bored_turn = 20 + self.adapted = True + + arc_turn = 1.8 if self.adapted_arc is None else self.adapted_arc + bored_after = ( + 190 + if self.adapted_bored_after is None + else self.adapted_bored_after + ) + bored_turn = ( + 20 + if self.adapted_bored_turn is None + else self.adapted_bored_turn + ) + bored_arc = arc_turn + if damage >= 80 and ammo >= 20 and health >= 85: + bored_arc = 0.8 + + if step - self.last_progress_step > bored_after: + bored_phase = ( + step - self.last_progress_step - bored_after + ) % (bored_turn + 90) + if bored_phase < bored_turn: + if bored_phase == 0: + self.search_direction *= -1.0 + return _move(speed=1.0, turn180=1.0) + return _move( + speed=1.0, + turn=self.search_direction * 6.0, + ) + return _move( + speed=1.0, + forward=1.0, + turn=self.search_direction * bored_arc, + ) + + return _move( + speed=1.0, + forward=1.0, + turn=self.search_direction * arc_turn, + ) diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/oracle/solve.sh b/tasks/simulation_heuristics_vizdoom_d3_v1/oracle/solve.sh new file mode 100755 index 0000000..d4a3448 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/oracle/solve.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /app/final_policy +cp /oracle/policy.py /app/final_policy/policy.py + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/starter_policy/policy.py b/tasks/simulation_heuristics_vizdoom_d3_v1/starter_policy/policy.py new file mode 100644 index 0000000..1bc31fd --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/starter_policy/policy.py @@ -0,0 +1,58 @@ +"""Weak sweep-and-fire starter for VizDoom D3 Battle.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import numpy as np + + +def _action( + *, + attack: float = 0.0, + speed: float = 0.0, + forward: float = 0.0, + backward: float = 0.0, + right: float = 0.0, + left: float = 0.0, + turn180: float = 0.0, + turn: float = 0.0, +) -> np.ndarray: + return np.asarray( + [attack, speed, forward, backward, right, left, turn180, turn], + dtype=np.float64, + ) + + +class Policy: + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.step = 0 + self.direction = 1.0 + + def act( + self, + frame: np.ndarray, + variables: Mapping[str, float], + ) -> np.ndarray: + del frame + ammo = float(variables["AMMO2"]) + phase = self.step % 120 + self.step += 1 + if phase == 90: + self.direction *= -1.0 + if phase < 90: + return _action( + attack=1.0 if ammo > 0 else 0.0, + speed=1.0, + forward=1.0, + turn=self.direction * 2.0, + ) + return _action( + attack=1.0 if ammo > 0 else 0.0, + speed=1.0, + turn=self.direction * 6.0, + ) + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/task.md b/tasks/simulation_heuristics_vizdoom_d3_v1/task.md new file mode 100644 index 0000000..d2f35fa --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/task.md @@ -0,0 +1,158 @@ +--- +schema_version: "1.3" +task: + name: genesisbench/simulation_heuristics_vizdoom_d3_v1 + description: Improve a screen-CV VizDoom D3 battle policy under a fixed autonomous research budget. + authors: + - name: GenesisBench Contributors + keywords: + - simulation + - heuristics + - vizdoom + - computer-vision + - first-person-control +metadata: + category: visual-control + difficulty: hard + tags: + - vizdoom + - screen-cv + - programmatic-policy + - combat + reference_task: false + genesisbench: + starter: + path: starter_policy + submission: + directory: final_policy + entrypoint: policy.py + development: + episodes: 10 + max_steps: 1050 + seeds: [0] + frame_skip: 2 + render_size: [640, 480] + verifier: + reproduction_config: verifier/config.toml + anchors: verifier/anchors.json + supports_private_config: true +agent: + timeout_sec: 1800 + user: agent + network_mode: public +verifier: + timeout_sec: 600 + user: root + network_mode: no-network +environment: + build_timeout_sec: 1200 + cpus: 4 + memory_mb: 4096 + storage_mb: 10240 + network_mode: public + allow_internet: true + workdir: /app +benchflow: + document_version: "0.6" +--- +# Improve the VizDoom D3 screen-CV battle policy + +You are given a working but weak programmatic controller for EnvPool +`D3Battle-v1`. Improve it through autonomous coding and repeated development +evaluation. + +## Objective + +Maximize: + +```text +episode reward = DAMAGECOUNT + 10 * KILLCOUNT +``` + +The controller must close three loops from screen pixels: + +1. detect, aim at, and attack visible enemies; +2. seek health or ammunition when resources are low; +3. explore and recover from walls or repeated views when no target is visible. + +The visible reference target is: + +```text +10-seed mean = 557 +10-seed min = 440 +``` + +The task runtime is pinned to the article's declared EnvPool `1.1.1`. + +## Observable inputs + +At each step your policy receives exactly: + +1. an immutable `uint8` screen frame with shape `(480, 640, 3)`; +2. an immutable mapping containing public `HEALTH`, `AMMO2`, `HITCOUNT`, + `DAMAGECOUNT`, and `KILLCOUNT`. + +It does not receive reward, environment objects, map data, labels, object +coordinates, seeds, or a vector lane identifier. + +## Development evaluation + +Run: + +```bash +python evaluate.py --policy final_policy/policy.py +``` + +For a shorter smoke test: + +```bash +python evaluate.py \ + --policy final_policy/policy.py \ + --episodes 2 \ + --max-steps 100 +``` + +The public evaluator uses batch seed `0`. Final evaluation uses different, +unseen seed batches. + +## Required submission + +Store the best policy at: + +```text +final_policy/policy.py +``` + +It must define `Policy` or `make_policy`: + +```python +class Policy: + def reset(self) -> None: + ... + + def act(self, frame, variables): + ... +``` + +Return a finite NumPy-compatible array with shape `(8,)`: + +```text +[ATTACK, SPEED, FORWARD, BACKWARD, RIGHT, LEFT, TURN180, TURN_DELTA] +``` + +The first seven channels must be in `[0, 1]`; turn delta must be in +`[-12, 12]`. + +## Rules + +1. Work autonomously and use the fixed wall-clock budget well. +2. Do not modify `evaluate.py`, `_runtime/`, or trusted task files. +3. Do not access `/oracle`, `/verifier`, hidden configs, or hidden seeds. +4. Do not import EnvPool or VizDoom from the policy. +5. Do not read WAD/config files, map geometry, labels, object coordinates, + automaps, or simulator state. +6. Do not hard-code routes or behavior for the public seed batch. +7. Use screen CV plus the five allowlisted public variables only. +8. The final policy must run offline with the installed packages. +9. Before finishing, evaluate `final_policy/policy.py` and leave the best + working version in place. diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/evaluation.md b/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/evaluation.md new file mode 100644 index 0000000..1ed67a3 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/evaluation.md @@ -0,0 +1,53 @@ +# Evaluation + +## Public development suite + +The default public command runs one ten-lane EnvPool batch: + +```text +scenario: D3Battle-v1 +EnvPool: 1.1.1 +batch seed: 0 +episodes: 10 +frame skip: 2 +maximum steps: 1,050 +screen size: 640 x 480 +reward: DAMAGECOUNT + 10 * KILLCOUNT +``` + +Weapon and crosshair rendering are disabled. The configured action channels +are attack, speed, forward/backward movement, strafing, 180-degree turn, and +horizontal turn delta. + +The evaluator reports mean, minimum, and maximum return, per-lane returns and +lengths, invalid-policy rate, final allowlisted variables, and action latency. + +## Hidden suite + +The checked-in reproducibility verifier uses seed batches that do not overlap +the public batch. A hosted leaderboard can inject a private config and matching +private anchors. + +Raw score is the weighted mean return across hidden suites. The normalized +GenesisBench score maps: + +- the checked-in sweep-and-fire starter to `0`; +- the frozen screen-CV battle reference to `100`. + +Scores above `100` are valid. The checked-in reproducibility anchors are +numeric scores generated +from clean, separate EnvPool `1.1.1` processes. The verifier validates their +per-suite means against the exact selected config before using them. Candidate +seed suites run in fresh subprocesses and unique working directories because +the EnvPool `1.1.1` D3 runtime is not safely re-entrant in one Linux process. +Private path-based anchors remain supported and use the same subprocess +isolation. + +## Failure handling + +A policy receives the configured failure return for an episode if it: + +- fails to import or instantiate; +- returns an invalid action; +- raises during `act`; +- violates the privileged-state source audit. diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/policy_api.md b/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/policy_api.md new file mode 100644 index 0000000..a784a58 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/policy_api.md @@ -0,0 +1,67 @@ +# Policy API + +The evaluator imports `final_policy/policy.py` in a clean Python process. + +Define `Policy` or `make_policy`: + +```python +class Policy: + def __init__(self): + ... + + def reset(self) -> None: + ... + + def act(self, frame, variables): + ... +``` + +One policy instance is created per active EnvPool lane and reset before the +first action. + +## Frame + +`frame` is an immutable, C-contiguous NumPy `uint8` array: + +```text +shape = (render_height, render_width, 3) +public default = (480, 640, 3) +``` + +The three channels are the raw `CRCGCB` screen format configured for the +article experiment. Treat them as ordered image channels rather than assuming +standard sRGB colorimetry. + +## Variables + +`variables` is an immutable mapping with exactly: + +```text +HEALTH: float +AMMO2: float +HITCOUNT: float +DAMAGECOUNT: float +KILLCOUNT: float +``` + +No reward, seed, lane id, map state, labels, or object coordinates are +included. + +## Actions + +Return a NumPy-compatible vector with shape `(8,)`: + +| Index | Channel | Valid range | +| ---: | --- | --- | +| 0 | attack | `[0, 1]` | +| 1 | speed | `[0, 1]` | +| 2 | move forward | `[0, 1]` | +| 3 | move backward | `[0, 1]` | +| 4 | move right | `[0, 1]` | +| 5 | move left | `[0, 1]` | +| 6 | turn 180 degrees | `[0, 1]` | +| 7 | horizontal turn delta | `[-12, 12]` | + +Non-finite, out-of-range, or incorrectly shaped actions invalidate the +episode. + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/provenance.md b/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/provenance.md new file mode 100644 index 0000000..90b2248 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/task_context/provenance.md @@ -0,0 +1,32 @@ +# Provenance and licensing + +## Behavioral source + +The experiment definition and target metrics come from: + +```text +repository: https://github.com/Trinkle23897/learning-beyond-gradients +revision: 3555c2956c257d49a5015b782cbe485b14fd659e +article section: VizDoom D3 Battle CV Policy +declared runtime: EnvPool 1.1.1 +``` + +The public article describes a screen-CV controller that detects enemies and +supplies, uses public health/ammunition/combat counters, and switches among +combat, resource seeking, and exploration loops. + +## Reimplementation boundary + +The pinned source repository did not contain a declared license. GenesisBench +therefore does not copy or redistribute its implementation. This task's API, +runtime, starter policy, evaluator, verifier, and minimized reference policy +are new code written for this repository from the public behavioral +description and independently reproduced output. + +EnvPool, OpenCV, NumPy, VizDoom runtime assets, and their transitive components +remain governed by their respective upstream licenses. + +The source README explicitly states that the experiments were written against +EnvPool `1.1.1`. Both the source policy and the GenesisBench oracle reproduce +the published ten-seed vector on that version, so no compatibility exception +is used. diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchor_policies/reference_policy.py b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchor_policies/reference_policy.py new file mode 100644 index 0000000..5d51668 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchor_policies/reference_policy.py @@ -0,0 +1,467 @@ +"""Frozen minimized screen-CV reference anchor for VizDoom D3.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import cv2 +import numpy as np + + +def _move( + *, + attack: float = 0.0, + speed: float = 0.0, + forward: float = 0.0, + backward: float = 0.0, + right: float = 0.0, + left: float = 0.0, + turn180: float = 0.0, + turn: float = 0.0, +) -> np.ndarray: + return np.asarray( + [attack, speed, forward, backward, right, left, turn180, turn], + dtype=np.float64, + ) + + +def _enemy_box( + image: np.ndarray, +) -> tuple[float, float, float, float, float] | None: + height, width = image.shape[:2] + scale_x = width / 320.0 + scale_y = height / 240.0 + center = width / 2.0 + channels = [ + image[..., channel].astype(np.int16) for channel in range(3) + ] + primary, secondary, tertiary = channels + enemy_pixels = ( + (primary > 50) + & ((primary - np.maximum(secondary, tertiary)) > 25) + & (secondary < 55) + & (tertiary < 55) + ) + mask = enemy_pixels.astype(np.uint8) * 255 + mask[: int(40 * scale_y), :] = 0 + mask[int(205 * scale_y) :, :] = 0 + closing_size = max(2, int(round(4 * scale_x))) + mask = cv2.morphologyEx( + mask, + cv2.MORPH_CLOSE, + np.ones((closing_size, closing_size), dtype=np.uint8), + ) + mask = cv2.dilate( + mask, + np.ones( + ( + max(2, int(4 * scale_y)), + max(2, int(3 * scale_x)), + ), + dtype=np.uint8, + ), + iterations=1, + ) + count, _, stats, centroids = cv2.connectedComponentsWithStats(mask, 8) + best: tuple[float, float, float, float, float] | None = None + best_score = -1.0 + for component in range(1, count): + _, _, box_width, box_height, area = stats[component] + center_x, center_y = centroids[component] + if area < 12 * scale_x * scale_y: + continue + if box_height < 7 * scale_y: + continue + if box_width > 95 * scale_x or box_height > 115 * scale_y: + continue + score = float(area) * ( + float(box_height) / (16 * scale_y) + ) ** 1.25 + score *= 1.0 + ( + float(center_y) - 90 * scale_y + ) / (180 * scale_y) + score /= 1.0 + abs(float(center_x) - center) / (80 * scale_x) + if score > best_score: + best_score = score + best = ( + float(center_x), + float(center_y), + float(box_width), + float(box_height), + float(area), + ) + return best + + +def _supply_box( + image: np.ndarray, + *, + ammo: float, + health: float, +) -> tuple[float, float, float, float, float] | None: + height, width = image.shape[:2] + scale_x = width / 320.0 + scale_y = height / 240.0 + center = width / 2.0 + first = image[..., 0].astype(np.int16) + second = image[..., 1].astype(np.int16) + third = image[..., 2].astype(np.int16) + enemy = ( + (first > 50) + & ((first - np.maximum(second, third)) > 25) + & (second < 55) + & (third < 55) + ) + + medikit_threshold = 95 if ammo <= 5 or health <= -1 else 115 + medikit = ( + (first > medikit_threshold) + & (second > 75) + & (third > 70) + ) + clip = (first > 90) & (second > 60) & ((first - third) > 25) + bright_mask = ((medikit | clip) & (~enemy)).astype(np.uint8) * 255 + bright_mask[: int(50 * scale_y), :] = 0 + bright_mask[int(218 * scale_y) :, :] = 0 + bright_mask = cv2.morphologyEx( + bright_mask, + cv2.MORPH_CLOSE, + np.ones( + ( + max(2, int(3 * scale_y)), + max(2, int(3 * scale_x)), + ), + dtype=np.uint8, + ), + ) + count, _, stats, centroids = cv2.connectedComponentsWithStats( + bright_mask, + 8, + ) + best: tuple[float, float, float, float, float] | None = None + best_score = -1.0 + for component in range(1, count): + _, _, box_width, box_height, area = stats[component] + center_x, center_y = centroids[component] + if area < 10 * scale_x * scale_y: + continue + if area > 3000 * scale_x * scale_y: + continue + if box_width > 60 * scale_x or box_height > 52 * scale_y: + continue + if box_width < 2 * scale_x or box_height < 2 * scale_y: + continue + score = float(area) * ( + 1.0 + (float(center_y) - 85 * scale_y) / (130 * scale_y) + ) + score /= 1.0 + abs(float(center_x) - center) / (85 * scale_x) + if score > best_score: + best_score = score + best = ( + float(center_x), + float(center_y), + float(box_width), + float(box_height), + float(area), + ) + if best is not None: + return best + + color_difference = np.maximum.reduce( + [ + np.abs(first - second), + np.abs(first - third), + np.abs(second - third), + ] + ) + fallback_mask = ( + (color_difference > 24) & (~enemy) + ).astype(np.uint8) * 255 + fallback_mask[: int(55 * scale_y), :] = 0 + fallback_mask[int(205 * scale_y) :, :] = 0 + fallback_mask = cv2.morphologyEx( + fallback_mask, + cv2.MORPH_CLOSE, + np.ones( + ( + max(2, int(3 * scale_y)), + max(2, int(3 * scale_x)), + ), + dtype=np.uint8, + ), + ) + count, _, stats, centroids = cv2.connectedComponentsWithStats( + fallback_mask, + 8, + ) + best = None + best_score = -1.0 + for component in range(1, count): + _, _, box_width, box_height, area = stats[component] + center_x, center_y = centroids[component] + if area < 8 * scale_x * scale_y: + continue + if area > 700 * scale_x * scale_y: + continue + if box_width > 52 * scale_x or box_height > 42 * scale_y: + continue + score = float(area) * ( + 1.0 + (float(center_y) - 80 * scale_y) / (140 * scale_y) + ) + score /= 1.0 + abs(float(center_x) - center) / (90 * scale_x) + if score > best_score: + best_score = score + best = ( + float(center_x), + float(center_y), + float(box_width), + float(box_height), + float(area), + ) + return best + + +def _view_key(image: np.ndarray) -> bytes: + gray = image.astype(np.uint8).mean(axis=2).astype(np.uint8) + small = cv2.resize(gray, (8, 6), interpolation=cv2.INTER_AREA) + return (small > small.mean()).astype(np.uint8).tobytes() + + +class Policy: + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.step = 0 + self.damage = 0.0 + self.hit = 0.0 + self.health = 100.0 + self.lock = 0 + self.bad_lock_steps = 0 + self.last_horizontal_error = 0.0 + self.search_direction = 1.0 + self.panic_steps = 0 + self.previous_frame: np.ndarray | None = None + self.stuck_steps = 0 + self.last_progress_step = 0 + self.adapted = False + self.adapted_arc: float | None = None + self.adapted_bored_after: int | None = None + self.adapted_bored_turn: int | None = None + self.escape_index = 0 + self.recent_views: list[bytes] = [] + + def _novelty_escape(self) -> np.ndarray: + self.escape_index += 1 + fraction = (self.escape_index * 0.6180339887498949) % 1.0 + magnitude = 3.0 + fraction * 7.0 + self.search_direction *= -1.0 + return _move( + speed=1.0, + backward=1.0, + turn=self.search_direction * magnitude, + ) + + def act( + self, + frame: np.ndarray, + variables: Mapping[str, float], + ) -> np.ndarray: + image = np.asarray(frame) + height, width = image.shape[:2] + scale_x = width / 320.0 + scale_y = height / 240.0 + screen_center = width / 2.0 + ammo = float(variables["AMMO2"]) + health = float(variables["HEALTH"]) + damage = float(variables["DAMAGECOUNT"]) + hit = float(variables["HITCOUNT"]) + step = self.step + self.step += 1 + + if damage > self.damage or hit > self.hit: + self.lock = 10 + self.bad_lock_steps = 0 + self.last_progress_step = step + elif self.lock > 0: + self.bad_lock_steps += 1 + if health < self.health - 0.1: + self.panic_steps = 12 + self.search_direction *= -1.0 + self.damage = damage + self.hit = hit + self.health = health + + enemy = _enemy_box(image) + if enemy is not None and ammo > 0: + center_x, _, _, box_height, area = enemy + horizontal_error = (center_x - screen_center) / scale_x + self.last_horizontal_error = horizontal_error + self.lock = max(self.lock, 4) + turn = float( + np.clip(horizontal_error * 0.18, -8.0, 8.0) + ) + if abs(horizontal_error) > 4.0: + danger = ( + area > 220 * scale_x * scale_y + or box_height > 29 * scale_y + ) + if danger: + if horizontal_error < 0: + return _move( + speed=1.0, + backward=1.0, + right=1.0, + turn=turn, + ) + return _move( + speed=1.0, + backward=1.0, + left=1.0, + turn=turn, + ) + return _move(speed=1.0, forward=1.0, turn=turn) + + strafe_right = (step // 8) % 2 == 1 + if ( + area > 380 * scale_x * scale_y + or box_height > 42 * scale_y + or self.panic_steps > 0 + ): + self.panic_steps = max(0, self.panic_steps - 1) + return _move( + attack=1.0, + speed=1.0, + backward=1.0, + right=1.0 if strafe_right else 0.0, + left=0.0 if strafe_right else 1.0, + turn=turn, + ) + return _move( + attack=1.0, + speed=1.0, + right=1.0 if strafe_right else 0.0, + left=0.0 if strafe_right else 1.0, + turn=turn, + ) + + if self.lock > 0 and ammo > 0: + self.lock -= 1 + turn = float( + np.clip( + self.last_horizontal_error * 0.18, + -8.0, + 8.0, + ) + ) + if ( + abs(self.last_horizontal_error) <= 4.0 + and self.bad_lock_steps < 4 + ): + return _move(attack=1.0, turn=turn) + if self.bad_lock_steps >= 4: + self.search_direction *= -1.0 + self.bad_lock_steps = 0 + if abs(self.last_horizontal_error) > 4.0: + return _move(speed=1.0, turn=turn) + return _move( + speed=1.0, + turn=self.search_direction * 6.0, + ) + + if self.panic_steps > 0: + self.panic_steps -= 1 + return _move( + speed=1.0, + backward=1.0, + turn=self.search_direction * 6.0, + ) + + if ammo <= 10: + supply = _supply_box(image, ammo=ammo, health=health) + if supply is not None: + center_x = supply[0] + horizontal_error = (center_x - screen_center) / scale_x + turn = float( + np.clip(horizontal_error * 0.18, -8.0, 8.0) + ) + return _move(speed=1.0, forward=1.0, turn=turn) + + key = _view_key(image) + repeated = key in self.recent_views + self.recent_views.append(key) + if len(self.recent_views) > 80: + self.recent_views.pop(0) + if repeated and step - self.last_progress_step > 160: + return self._novelty_escape() + + if self.previous_frame is not None: + difference = float( + np.mean( + np.abs( + image.astype(np.int16) + - self.previous_frame.astype(np.int16) + ) + ) + ) + if difference < 0.9: + self.stuck_steps += 1 + else: + self.stuck_steps = max(0, self.stuck_steps - 1) + self.previous_frame = image.copy() + if self.stuck_steps > 10: + self.stuck_steps = 0 + self.search_direction *= -1.0 + return _move( + speed=1.0, + backward=1.0, + turn=self.search_direction * 6.0, + ) + + if not self.adapted and step >= 250: + low_progress = damage <= 15 and health >= 90 + danger_progress = health <= 75 and damage >= 60 + if low_progress or danger_progress: + self.adapted_arc = 1.4 + self.adapted_bored_after = 180 + self.adapted_bored_turn = 20 + self.adapted = True + + arc_turn = 1.8 if self.adapted_arc is None else self.adapted_arc + bored_after = ( + 190 + if self.adapted_bored_after is None + else self.adapted_bored_after + ) + bored_turn = ( + 20 + if self.adapted_bored_turn is None + else self.adapted_bored_turn + ) + bored_arc = arc_turn + if damage >= 80 and ammo >= 20 and health >= 85: + bored_arc = 0.8 + + if step - self.last_progress_step > bored_after: + bored_phase = ( + step - self.last_progress_step - bored_after + ) % (bored_turn + 90) + if bored_phase < bored_turn: + if bored_phase == 0: + self.search_direction *= -1.0 + return _move(speed=1.0, turn180=1.0) + return _move( + speed=1.0, + turn=self.search_direction * 6.0, + ) + return _move( + speed=1.0, + forward=1.0, + turn=self.search_direction * bored_arc, + ) + + return _move( + speed=1.0, + forward=1.0, + turn=self.search_direction * arc_turn, + ) diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchor_policies/starter_policy.py b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchor_policies/starter_policy.py new file mode 100644 index 0000000..9a7ccfa --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchor_policies/starter_policy.py @@ -0,0 +1,58 @@ +"""Frozen weak sweep-and-fire anchor for VizDoom D3 Battle.""" + +from __future__ import annotations + +from collections.abc import Mapping + +import numpy as np + + +def _action( + *, + attack: float = 0.0, + speed: float = 0.0, + forward: float = 0.0, + backward: float = 0.0, + right: float = 0.0, + left: float = 0.0, + turn180: float = 0.0, + turn: float = 0.0, +) -> np.ndarray: + return np.asarray( + [attack, speed, forward, backward, right, left, turn180, turn], + dtype=np.float64, + ) + + +class Policy: + def __init__(self) -> None: + self.reset() + + def reset(self) -> None: + self.step = 0 + self.direction = 1.0 + + def act( + self, + frame: np.ndarray, + variables: Mapping[str, float], + ) -> np.ndarray: + del frame + ammo = float(variables["AMMO2"]) + phase = self.step % 120 + self.step += 1 + if phase == 90: + self.direction *= -1.0 + if phase < 90: + return _action( + attack=1.0 if ammo > 0 else 0.0, + speed=1.0, + forward=1.0, + turn=self.direction * 2.0, + ) + return _action( + attack=1.0 if ammo > 0 else 0.0, + speed=1.0, + turn=self.direction * 6.0, + ) + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchors.json b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchors.json new file mode 100644 index 0000000..f03524e --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/anchors.json @@ -0,0 +1,54 @@ +{ + "environment": "EnvPool 1.1.1 D3Battle-v1, screen CV plus five public variables", + "calibration": { + "envpool_version": "1.1.1", + "evaluation": { + "scenario": "d3", + "max_steps": 1050, + "frame_skip": 2, + "render_width": 640, + "render_height": 480, + "failure_return": 0.0, + "suites": [ + { + "name": "hidden_nominal", + "weight": 0.7, + "seed": 101, + "episodes": 5 + }, + { + "name": "hidden_secondary", + "weight": 0.3, + "seed": 607, + "episodes": 5 + } + ] + } + }, + "public_article_target": { + "envpool_version": "1.1.1", + "batch_seed": 0, + "episodes": 10, + "mean_return": 557.0, + "min_return": 440.0, + "returns": [545, 475, 480, 440, 690, 500, 600, 595, 530, 715] + }, + "reference_policy": { + "description": "Frozen screen-CV combat, supply, and exploration policy", + "path": "anchor_policies/reference_policy.py", + "score": 331.1, + "suite_mean_returns": { + "hidden_nominal": 311.0, + "hidden_secondary": 378.0 + } + }, + "starter_policy": { + "description": "Public weak sweep-and-fire starter", + "path": "anchor_policies/starter_policy.py", + "score": 31.9, + "suite_mean_returns": { + "hidden_nominal": 28.0, + "hidden_secondary": 41.0 + } + } +} diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/config.toml b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/config.toml new file mode 100644 index 0000000..207ac9b --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/config.toml @@ -0,0 +1,22 @@ +version = "1.0" + +[evaluation] +scenario = "d3" +max_steps = 1050 +frame_skip = 2 +render_width = 640 +render_height = 480 +failure_return = 0.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.7 +seed = 101 +episodes = 5 + +[[evaluation.suites]] +name = "hidden_secondary" +weight = 0.3 +seed = 607 +episodes = 5 + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/evaluate_hidden.py b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/evaluate_hidden.py new file mode 100755 index 0000000..6989b88 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/evaluate_hidden.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import subprocess +import sys +import tempfile +import tomllib +from pathlib import Path +from typing import Any + + +VERIFIER_DIR = Path(__file__).resolve().parent +RUNTIME_CANDIDATES = [Path("/opt/genesisbench")] +RUNTIME_CANDIDATES.extend( + ancestor / "src" for ancestor in VERIFIER_DIR.parents +) +for candidate in RUNTIME_CANDIDATES: + if (candidate / "genesisbench").is_dir(): + sys.path.insert(0, str(candidate)) + break + +from genesisbench.vizdoom import ( # noqa: E402 + VIZDOOM_ARTICLE_ENVPOOL_VERSION, + evaluate_vizdoom_policy, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run hidden VizDoom D3 evaluation." + ) + parser.add_argument("policy", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument( + "--config", + type=Path, + default=VERIFIER_DIR / "config.toml", + help="Evaluation suite config; production can inject a private file.", + ) + parser.add_argument( + "--anchors", + type=Path, + default=VERIFIER_DIR / "anchors.json", + help="Normalization anchors matching the selected evaluation suite.", + ) + parser.add_argument( + "--worker-request", + type=Path, + help=argparse.SUPPRESS, + ) + return parser.parse_args() + + +def _worker_main( + policy: Path, + *, + request_path: Path, + output_path: Path | None, +) -> None: + if output_path is None: + raise ValueError("--output is required in worker mode") + request = json.loads(request_path.read_text()) + evaluation = request["evaluation"] + suite = request["suite"] + result = evaluate_vizdoom_policy( + policy, + scenario=evaluation["scenario"], + seed=suite["seed"], + episodes=suite["episodes"], + max_steps=evaluation["max_steps"], + frame_skip=evaluation["frame_skip"], + render_width=evaluation["render_width"], + render_height=evaluation["render_height"], + failure_return=evaluation["failure_return"], + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(result.to_json() + "\n") + + +def _safe_name(value: str) -> str: + rendered = "".join( + character if character.isalnum() or character in "-_" else "-" + for character in value + ).strip("-") + return rendered or "suite" + + +def _evaluate_suite_isolated( + policy: Path, + *, + evaluation: dict[str, Any], + suite: dict[str, Any], + workspace_root: Path, + run_label: str, + suite_index: int, +) -> dict[str, Any]: + workspace = ( + workspace_root + / f"{_safe_name(run_label)}-{suite_index:02d}-{_safe_name(suite['name'])}" + ) + workspace.mkdir(parents=True, exist_ok=False) + request_path = workspace / "request.json" + output_path = workspace / "result.json" + request_path.write_text( + json.dumps( + { + "evaluation": evaluation, + "suite": suite, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + completed = subprocess.run( + [ + sys.executable, + str(Path(__file__).resolve()), + str(policy.resolve()), + "--worker-request", + str(request_path), + "--output", + str(output_path), + ], + cwd=workspace, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + "Isolated VizDoom suite evaluation failed " + f"for {run_label}/{suite['name']} with rc={completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + if not output_path.is_file(): + raise RuntimeError( + f"Isolated VizDoom suite did not write {output_path}" + ) + return json.loads(output_path.read_text()) + + +def _evaluate_raw( + policy: Path, + *, + evaluation: dict[str, Any], + workspace_root: Path, + run_label: str, +) -> tuple[dict[str, dict[str, Any]], float]: + results: dict[str, dict[str, Any]] = {} + score = 0.0 + for suite_index, suite in enumerate(evaluation["suites"]): + result = _evaluate_suite_isolated( + policy, + evaluation=evaluation, + suite=suite, + workspace_root=workspace_root, + run_label=run_label, + suite_index=suite_index, + ) + results[suite["name"]] = result + score += float(suite["weight"]) * float(result["mean_return"]) + return results, score + + +def _validate_numeric_anchor_calibration( + anchors: dict[str, Any], + *, + evaluation: dict[str, Any], +) -> None: + numeric_anchors = [ + anchor + for name in ("starter_policy", "reference_policy") + if isinstance((anchor := anchors[name]).get("score"), int | float) + ] + if not numeric_anchors: + return + calibration = anchors.get("calibration") + if not isinstance(calibration, dict): + raise ValueError("numeric anchors require calibration metadata") + if ( + calibration.get("envpool_version") + != VIZDOOM_ARTICLE_ENVPOOL_VERSION + ): + raise ValueError("numeric anchors use the wrong EnvPool version") + if calibration.get("evaluation") != evaluation: + raise ValueError( + "numeric anchors do not match the selected hidden evaluation" + ) + + +def _anchor_score( + anchors: dict[str, Any], + name: str, + *, + anchors_path: Path, + evaluation: dict[str, Any], + workspace_root: Path, +) -> float: + anchor = anchors[name] + score = anchor.get("score") + if isinstance(score, int | float): + suite_means = anchor.get("suite_mean_returns") + if not isinstance(suite_means, dict): + raise ValueError( + f"numeric anchor {name} must declare suite_mean_returns" + ) + suite_names = {suite["name"] for suite in evaluation["suites"]} + if set(suite_means) != suite_names: + raise ValueError( + f"numeric anchor {name} has mismatched hidden suites" + ) + derived_score = sum( + float(suite["weight"]) * float(suite_means[suite["name"]]) + for suite in evaluation["suites"] + ) + if not math.isclose( + derived_score, + float(score), + rel_tol=0.0, + abs_tol=1e-9, + ): + raise ValueError( + f"numeric anchor {name} score does not match suite means" + ) + return float(score) + relative_path = anchor.get("path") + if not isinstance(relative_path, str) or not relative_path: + raise ValueError(f"{name} must declare score or path") + policy = anchors_path.parent / relative_path + _, calibrated_score = _evaluate_raw( + policy, + evaluation=evaluation, + workspace_root=workspace_root, + run_label=f"anchor-{name}", + ) + return calibrated_score + + +def main() -> None: + args = parse_args() + if args.worker_request is not None: + _worker_main( + args.policy, + request_path=args.worker_request, + output_path=args.output, + ) + return + + config = tomllib.loads(args.config.read_text()) + anchors = json.loads(args.anchors.read_text()) + evaluation = config["evaluation"] + _validate_numeric_anchor_calibration(anchors, evaluation=evaluation) + with tempfile.TemporaryDirectory( + prefix="genesisbench-vizdoom-d3-hidden-" + ) as temporary_directory: + workspace_root = Path(temporary_directory) + suites, score = _evaluate_raw( + args.policy, + evaluation=evaluation, + workspace_root=workspace_root, + run_label="candidate", + ) + starter_score = _anchor_score( + anchors, + "starter_policy", + anchors_path=args.anchors, + evaluation=evaluation, + workspace_root=workspace_root, + ) + reference_score = _anchor_score( + anchors, + "reference_policy", + anchors_path=args.anchors, + evaluation=evaluation, + workspace_root=workspace_root, + ) + if reference_score == starter_score: + raise ValueError("starter and reference anchors must have different scores") + normalized_score = round( + 100.0 * (score - starter_score) / (reference_score - starter_score), + 9, + ) + payload = { + "score": score, + "normalized_score": normalized_score, + "starter_score": starter_score, + "reference_score": reference_score, + "isolation": { + "mode": "subprocess_per_suite", + "suite_process_count": len(evaluation["suites"]), + "unique_working_directories": True, + }, + "suites": suites, + } + rendered = json.dumps(payload, indent=2, sort_keys=True) + print(rendered) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + + +if __name__ == "__main__": + main() diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/rubrics/verifier.md b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/rubrics/verifier.md new file mode 100644 index 0000000..2e4c401 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/rubrics/verifier.md @@ -0,0 +1,6 @@ +# Simulation Heuristics VizDoom D3 v1 Verifier Rubric + +- `task_success`: normalized hidden-suite D3 battle return, where the public + sweep-and-fire starter maps to `0` and the frozen screen-CV reference maps + to `1`. + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/rubrics/verifier.toml b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/rubrics/verifier.toml new file mode 100644 index 0000000..037420a --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/rubrics/verifier.toml @@ -0,0 +1,10 @@ +version = "0.1" + +[[criteria]] +id = "task_success" +description = "Normalized hidden-suite VizDoom D3 battle return." +weight = 1.0 + +[scoring] +method = "weighted_mean" + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/test.sh b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/test.sh new file mode 100755 index 0000000..0d01f3d --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/test.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /logs/verifier +policy=/app/final_policy/policy.py + +if [ ! -f "$policy" ]; then + printf '0.0\n' > /logs/verifier/reward.txt + printf '{"reward": 0.0}\n' > /logs/verifier/reward.json + exit 0 +fi + +python /verifier/evaluate_hidden.py \ + "$policy" \ + --config /verifier/config.toml \ + --anchors /verifier/anchors.json \ + --output /logs/verifier/genesis-score.json + +python - <<'PY' +import json +from pathlib import Path + +score = json.loads(Path("/logs/verifier/genesis-score.json").read_text()) +reward = max(0.0, min(1.0, float(score["normalized_score"]) / 100.0)) +Path("/logs/verifier/reward.txt").write_text(f"{reward}\n") +Path("/logs/verifier/reward.json").write_text( + json.dumps({"reward": reward}, indent=2) + "\n" +) +PY + diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/verifier.md b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/verifier.md new file mode 100644 index 0000000..03d3f82 --- /dev/null +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/verifier/verifier.md @@ -0,0 +1,39 @@ +--- +document_version: "0.3" +verifier: + name: simulation_heuristics_vizdoom_d3_v1 + default_strategy: deterministic + strategies: + deterministic: + type: script + command: ./test.sh + rubric: + combine: weighted_mean + dimensions: + task_success: + weight: 1.0 + source: deterministic + outputs: + reward_text: /logs/verifier/reward.txt + reward_json: /logs/verifier/reward.json + details_json: /logs/verifier/genesis-score.json + aggregate_policy: + method: weighted_mean + metrics: + task_success: 1.0 +--- + +## Verifier intent + +Run the submitted D3 screen-CV policy on hidden EnvPool seed batches. Only the +screen and five allowlisted public game variables cross the trusted policy +boundary. + +BenchFlow's canonical reward is the locally normalized GenesisBench score +clamped to `[0, 1]`. Raw combat returns, anchor scores, invalid-policy details, +and per-suite metrics remain available in `genesis-score.json`. + +Each candidate suite runs in a fresh subprocess and unique working directory. +This prevents EnvPool `1.1.1` from reusing its process-global D3 runtime or +colliding on `./_vizdoom`. Checked-in numeric anchors include exact config and +per-suite calibration metadata; mismatched calibration fails closed. diff --git a/tests/test_article_suite_tooling.py b/tests/test_article_suite_tooling.py new file mode 100644 index 0000000..2b76d64 --- /dev/null +++ b/tests/test_article_suite_tooling.py @@ -0,0 +1,421 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts import build_article_suite_leaderboard as leaderboard +from scripts import bench_opencode +from scripts import run_article_suite as runner + + +def test_article_suite_declares_exactly_nine_unique_tasks() -> None: + assert len(runner.TASKS) == 9 + assert len(set(runner.TASKS)) == 9 + assert runner.TASKS == leaderboard.TASKS + assert "simulation_heuristics_ant_v1" in runner.TASKS + + +def test_command_uses_opencode_and_fail_closed_artifacts(tmp_path: Path) -> None: + model = { + "id": "gpt-5.6-sol", + "display_name": "GPT-5.6 Sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max", + } + command = runner.build_command( + model=model, + jobs_dir=tmp_path / "jobs", + sandbox="docker", + concurrency=1, + artifact_dir=tmp_path, + tasks=runner.TASKS, + azure_resource_name="example-resource", + ) + + assert command[command.index("--agent") + 1] == "opencode" + assert command[command.index("--model") + 1] == "azure/gpt-5.6-sol" + assert command[command.index("--usage-tracking") + 1] == "off" + assert command[command.index("--expected-tasks") + 1] == "9" + assert command.count("--include") == 9 + assert "--reasoning-effort" not in command + assert "scripts/bench_opencode.py" in command + assert "AZURE_RESOURCE_NAME=example-resource" in command + + +def test_benchflow_opencode_shim_uses_native_provider_mode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + original = bench_opencode.litellm_runtime._NATIVE_PROTOCOL_AGENTS + monkeypatch.setattr( + bench_opencode.litellm_runtime, + "_NATIVE_PROTOCOL_AGENTS", + original, + ) + + bench_opencode.apply_opencode_direct_provider_mode() + + assert "opencode" in ( + bench_opencode.litellm_runtime._NATIVE_PROTOCOL_AGENTS + ) + + +def test_benchflow_opencode_shim_writes_claude_oauth_credentials( + monkeypatch: pytest.MonkeyPatch, +) -> None: + original = bench_opencode.AGENTS["opencode"] + monkeypatch.setitem(bench_opencode.AGENTS, "opencode", original) + + bench_opencode.apply_opencode_claude_oauth_support() + + credentials = bench_opencode.AGENTS["opencode"].credential_files + oauth = [ + item + for item in credentials + if item.path == "{home}/.claude/.credentials.json" + ] + assert len(oauth) == 1 + assert oauth[0].env_source == "CLAUDE_CODE_OAUTH_TOKEN" + assert "claudeAiOauth" in oauth[0].template + + +def test_opencode_config_pins_gpt_5_6_sol_to_max() -> None: + model = { + "id": "gpt-5.6-sol", + "display_name": "GPT-5.6 Sol", + "model": "azure/gpt-5.6-sol", + "provider": "azure", + "provider_reasoning_effort": "max", + } + + config = runner._opencode_config(model) + definition = config["provider"]["azure"]["models"]["gpt-5.6-sol"] + + assert definition["options"]["reasoningEffort"] == "max" + assert definition["variants"]["max"]["reasoningEffort"] == "max" + + +def test_opencode_config_pins_claude_oauth_plugin() -> None: + model = { + "id": "claude-opus-4.8", + "display_name": "Claude Opus 4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max", + } + + config = runner._opencode_config(model) + definition = config["provider"]["anthropic"]["models"][ + "claude-opus-4-8" + ] + + assert config["plugin"] == ["opencode-claude-auth@2.0.0"] + assert definition["variants"]["max"]["thinking"]["budgetTokens"] == 31_999 + + +def test_claude_command_can_disable_idle_watchdog(tmp_path: Path) -> None: + model = { + "id": "claude-opus-4.8", + "display_name": "Claude Opus 4.8", + "model": "anthropic/claude-opus-4-8", + "provider": "claude_oauth", + "provider_reasoning_effort": "max", + "agent_idle_timeout_sec": 0, + "daytona_pty_readline_timeout_sec": 3600, + } + + command = runner.build_command( + model=model, + jobs_dir=tmp_path / "jobs", + sandbox="daytona", + concurrency=1, + artifact_dir=tmp_path, + tasks=(runner.TASKS[0],), + ) + + assert command[command.index("--agent-idle-timeout") + 1] == "0" + + +def test_claude_model_declares_long_daytona_pty_timeout() -> None: + models = runner._models() + claude = next(model for model in models if model["id"] == "claude-opus-4.8") + + assert claude["daytona_pty_readline_timeout_sec"] == 3600 + + +def test_azure_resource_name_is_derived_without_exposing_endpoint() -> None: + assert ( + runner._azure_resource_name( + { + "AZURE_API_ENDPOINT": ( + "https://example-resource.openai.azure.com/" + ) + } + ) + == "example-resource" + ) + + +def test_provider_environment_is_least_privilege() -> None: + shared = { + "AZURE_API_ENDPOINT": "https://example.openai.azure.com/", + "AZURE_API_KEY": "azure-secret", + "ANTHROPIC_API_KEY": "anthropic-or-azure-secret", + "CLAUDE_CODE_OAUTH_TOKEN": "oauth-secret", + } + azure = runner._scoped_provider_env( + {"provider": "azure"}, + shared, + ) + claude = runner._scoped_provider_env( + {"provider": "claude_oauth"}, + shared, + ) + + assert azure["AZURE_API_KEY"] == "azure-secret" + assert "CLAUDE_CODE_OAUTH_TOKEN" not in azure + assert "ANTHROPIC_API_KEY" not in azure + assert claude["CLAUDE_CODE_OAUTH_TOKEN"] == "oauth-secret" + assert claude["ANTHROPIC_API_KEY"] == "oauth-plugin" + assert "AZURE_API_KEY" not in claude + assert "AZURE_API_ENDPOINT" not in claude + + +def test_aggregate_loader_requires_every_task(tmp_path: Path) -> None: + model_root = tmp_path / "gpt-5.6-sol" + job = model_root / "jobs" / "run" + job.mkdir(parents=True) + rows = [] + for task in leaderboard.TASKS[:-1]: + rows.append( + { + "info": {"task_name": task}, + "reward": 1.0, + "error": None, + } + ) + (job / "results.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n" + ) + + with pytest.raises(RuntimeError, match="missing article-suite tasks"): + leaderboard._load_results(model_root) + + +def test_aggregate_loader_rejects_task_errors(tmp_path: Path) -> None: + model_root = tmp_path / "gpt-5.6-sol" + job = model_root / "jobs" / "run" + job.mkdir(parents=True) + rows = [ + { + "info": {"task_name": task}, + "reward": 1.0, + "error": {"error": "agent_error"} if index == 0 else None, + } + for index, task in enumerate(leaderboard.TASKS) + ] + (job / "results.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n" + ) + + with pytest.raises(RuntimeError, match="contains an error"): + leaderboard._load_results(model_root) + + partial = leaderboard._load_results( + model_root, + allow_partial_errors=True, + ) + assert len(partial) == 8 + + +def test_aggregate_loader_accepts_direct_provider_training_export_warning( + tmp_path: Path, +) -> None: + model_root = tmp_path / "gpt-5.6-sol" + job = model_root / "jobs" / "run" + job.mkdir(parents=True) + rows = [ + { + "info": {"task_name": task}, + "reward": 0.5, + "error": { + "error": "missing_llm_trajectory", + "error_chain_str": "not training-ready", + }, + } + for task in leaderboard.TASKS + ] + (job / "results.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n" + ) + + loaded = leaderboard._load_results(model_root) + + assert len(loaded) == 9 + + +def test_normalized_score_prefers_unbounded_verifier_details( + tmp_path: Path, +) -> None: + rollout = tmp_path / "rollout" + verifier = rollout / "verifier" + verifier.mkdir(parents=True) + (verifier / "genesis-score.json").write_text( + json.dumps({"normalized_score": 103.811802037}) + ) + row = { + "info": {"rollout_dir": str(rollout)}, + "reward": 1.0, + } + + assert leaderboard._normalized_task_score(row) == 103.811802037 + + +def test_article_score_packaging_sanitizes_absolute_artifact_paths() -> None: + payload = { + "policy_path": "/app/final_policy/policy.py", + "nested": { + "artifact_path": "/app/final_artifact", + "score": 12.0, + }, + "episodes": [{"policy_path": "/private/path/policy.py"}], + } + + sanitized = leaderboard._sanitize_score_paths(payload) + + assert sanitized["policy_path"] == "submitted_artifact" + assert sanitized["nested"]["artifact_path"] == "submitted_artifact" + assert sanitized["nested"]["score"] == 12.0 + assert sanitized["episodes"][0]["policy_path"] == "submitted_artifact" + + +def test_ant_timeout_only_digest_change_is_score_compatible() -> None: + old_digest = next( + iter( + leaderboard.TASK_DIGEST_COMPATIBILITY[ + "simulation_heuristics_ant_v1" + ] + ) + ) + + note = leaderboard._digest_compatibility_note( + "simulation_heuristics_ant_v1", + old_digest, + "sha256:new", + ) + + assert note is not None + assert "fail-closed internal timeout" in note + + +def test_scored_agent_timeout_is_a_valid_leaderboard_result( + tmp_path: Path, +) -> None: + model_root = tmp_path / "claude-opus-4.8" + job = model_root / "jobs" / "run" + job.mkdir(parents=True) + rows = [] + for index, task in enumerate(leaderboard.TASKS): + rollout = job / f"task-{index}" + verifier = rollout / "verifier" + verifier.mkdir(parents=True) + (verifier / "genesis-score.json").write_text( + json.dumps({"normalized_score": 0.0}) + ) + rows.append( + { + "info": { + "task_name": task, + "rollout_dir": str(rollout), + }, + "reward": 0.0, + "error": ( + {"error": "agent_timeout"} + if index == 0 + else {"error": "missing_llm_trajectory"} + ), + } + ) + (job / "results.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n" + ) + + loaded = leaderboard._load_results(model_root) + + assert len(loaded) == 9 + + +def test_latest_model_runs_can_resume_across_batches(tmp_path: Path) -> None: + for batch, model_id, finished_at in ( + ("batch-a", "gpt-5.6-sol", 1.0), + ("batch-b", "gpt-5.6-sol", 2.0), + ("batch-c", "gpt-5.5", 3.0), + ): + model_root = tmp_path / batch / model_id + model_root.mkdir(parents=True) + (model_root / "run_metadata.json").write_text( + json.dumps( + { + "model": {"id": model_id}, + "return_code": 0, + "dry_run": False, + "finished_at": finished_at, + } + ) + ) + selected = leaderboard._latest_model_runs(tmp_path) + + assert selected["gpt-5.6-sol"] == ( + tmp_path / "batch-b" / "gpt-5.6-sol" + ) + assert selected["gpt-5.5"] == tmp_path / "batch-c" / "gpt-5.5" + + +def test_latest_task_results_merges_partial_runs(tmp_path: Path) -> None: + model_id = "gpt-5.6-sol" + for index, task in enumerate(leaderboard.TASKS[:2], start=1): + model_root = tmp_path / f"batch-{index}" / model_id + job = model_root / "jobs" / "run" + job.mkdir(parents=True) + (model_root / "run_metadata.json").write_text( + json.dumps( + { + "model": {"id": model_id}, + "return_code": 1 if index == 1 else 0, + "dry_run": False, + "finished_at": float(index), + "tasks": [task], + } + ) + ) + (model_root / "task_manifest.json").write_text( + json.dumps( + { + "tasks": [ + { + "task_id": task, + "digest": f"sha256:digest-{index}", + } + ] + } + ) + ) + (job / "results.jsonl").write_text( + json.dumps( + { + "info": {"task_name": task}, + "reward": index / 10, + "error": None, + } + ) + + "\n" + ) + + selected = leaderboard._latest_task_results(tmp_path) + + assert set(selected[model_id]) == set(leaderboard.TASKS[:2]) + assert selected[model_id][leaderboard.TASKS[0]][1]["reward"] == 0.1 + assert selected[model_id][leaderboard.TASKS[1]][1]["reward"] == 0.2 + assert selected[model_id][leaderboard.TASKS[1]][3] == "sha256:digest-2" diff --git a/tests/test_leaderboard_artifacts.py b/tests/test_leaderboard_artifacts.py index e558bd4..a0a6737 100644 --- a/tests/test_leaderboard_artifacts.py +++ b/tests/test_leaderboard_artifacts.py @@ -1,11 +1,13 @@ from __future__ import annotations import json +import math from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] LEADERBOARD = REPO_ROOT / "leaderboard" / "simulation_heuristics_ant_v1.json" +ARTICLE_SUITE = REPO_ROOT / "leaderboard" / "article_suite.json" def test_packaged_leaderboard_is_self_contained() -> None: @@ -29,3 +31,52 @@ def test_packaged_leaderboard_is_self_contained() -> None: policy_path = score[suite]["policy_path"] assert policy_path == row["submission_path"] assert not Path(policy_path).is_absolute() + + +def test_article_suite_leaderboard_is_complete_and_self_contained() -> None: + payload = json.loads(ARTICLE_SUITE.read_text()) + + assert payload["benchmark"] == "learning_beyond_gradients_article_suite" + assert payload["task_count"] == 9 + assert len(payload["tasks"]) == 9 + assert len(payload["rows"]) == 4 + assert [row["rank"] for row in payload["rows"]] == [1, 2, 3, 4] + + averages = [ + row["average_normalized_score"] for row in payload["rows"] + ] + assert averages == sorted(averages, reverse=True) + + for row in payload["rows"]: + assert row["harness"] == "opencode" + assert set(row["task_scores"]) == set(payload["tasks"]) + assert set(row["submission_details"]) == set(payload["tasks"]) + assert math.isclose( + row["average_normalized_score"], + sum(row["task_scores"].values()) / payload["task_count"], + ) + for task, relative_score_path in row["submission_details"].items(): + score_path = REPO_ROOT / relative_score_path + metadata_path = score_path.with_name("metadata.json") + assert not Path(relative_score_path).is_absolute() + assert score_path.is_file() + assert metadata_path.is_file() + + score = json.loads(score_path.read_text()) + metadata = json.loads(metadata_path.read_text()) + assert metadata["task"] == task + assert metadata["harness"] == "opencode" + assert metadata["normalized_score"] == row["task_scores"][task] + assert score["normalized_score"] == row["task_scores"][task] + + def assert_no_absolute_artifact_paths(value: object) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key in {"policy_path", "artifact_path"}: + assert item == "submitted_artifact" + assert_no_absolute_artifact_paths(item) + elif isinstance(value, list): + for item in value: + assert_no_absolute_artifact_paths(item) + + assert_no_absolute_artifact_paths(score) diff --git a/tests/test_policy_module_loading.py b/tests/test_policy_module_loading.py new file mode 100644 index 0000000..a75acb2 --- /dev/null +++ b/tests/test_policy_module_loading.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from pathlib import Path +import sys +from typing import Callable + +import pytest + +from genesisbench.ant import _load_policy_module as load_ant_policy +from genesisbench.atari57 import _load_policy_module as load_atari57_policy +from genesisbench.breakout import _load_policy_module as load_breakout_policy +from genesisbench.halfcheetah import _load_policy_module as load_halfcheetah_policy +from genesisbench.montezuma import _load_policy_module as load_montezuma_policy +from genesisbench.pong import _load_policy_module as load_pong_policy +from genesisbench.vizdoom import _load_policy_module as load_vizdoom_policy + + +PolicyLoader = Callable[[Path], object] + + +@pytest.mark.parametrize( + "loader", + ( + load_ant_policy, + load_atari57_policy, + load_breakout_policy, + load_halfcheetah_policy, + load_montezuma_policy, + load_pong_policy, + load_vizdoom_policy, + ), +) +def test_policy_loader_supports_postponed_dataclass_annotations( + tmp_path: Path, + loader: PolicyLoader, +) -> None: + policy_path = tmp_path / f"{loader.__module__.replace('.', '_')}.py" + policy_path.write_text( + """ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class Policy: + gain: float = 1.0 +""".lstrip() + ) + + module = loader(policy_path) + try: + assert sys.modules[module.__name__] is module + assert module.Policy().gain == 1.0 + finally: + sys.modules.pop(module.__name__, None) diff --git a/tests/test_simulation_heuristics_ant_v1.py b/tests/test_simulation_heuristics_ant_v1.py index c05b8ea..a0fe590 100644 --- a/tests/test_simulation_heuristics_ant_v1.py +++ b/tests/test_simulation_heuristics_ant_v1.py @@ -1,29 +1,27 @@ from __future__ import annotations import json +import os from pathlib import Path import subprocess import sys +import tomllib + +import pytest from genesisbench.ant import DynamicsVariant, evaluate_ant_policy +from scripts.validate_tasks import validate_task REPO_ROOT = Path(__file__).resolve().parents[1] -STARTER = REPO_ROOT / "tasks" / "simulation_heuristics_ant_v1" / "starter_policy" / "policy.py" -REFERENCE = ( - REPO_ROOT - / "tasks" - / "simulation_heuristics_ant_v1" - / "oracle" - / "policy.py" -) -HIDDEN_EVALUATOR = ( - REPO_ROOT - / "tasks" - / "simulation_heuristics_ant_v1" - / "verifier" - / "evaluate_hidden.py" -) +TASK_DIR = REPO_ROOT / "tasks" / "simulation_heuristics_ant_v1" +STARTER = TASK_DIR / "starter_policy" / "policy.py" +ORACLE = TASK_DIR / "oracle" / "policy.py" +REFERENCE = TASK_DIR / "verifier" / "anchor_policies" / "reference_policy.py" +HIDDEN_EVALUATOR = TASK_DIR / "verifier" / "evaluate_hidden.py" +VERIFIER_SCRIPT = TASK_DIR / "verifier" / "test.sh" +SMOKE_CONFIG = TASK_DIR / "verifier" / "config_smoke.toml" +PROVENANCE = TASK_DIR / "evidence" / "source_provenance.json" def test_starter_policy_short_smoke() -> None: @@ -71,7 +69,47 @@ def test_json_uses_public_return_key() -> None: assert "return_" not in episode +def test_article_reference_can_plan_from_copied_model() -> None: + result = evaluate_ant_policy( + REFERENCE, + seeds=(0,), + max_steps=1, + ) + + assert result.episodes[0].length == 1 + assert result.invalid_episode_rate == 0.0 + assert result.mean_action_latency_ms > 0.0 + + def test_anchor_normalization_is_platform_local(tmp_path: Path) -> None: + config = tmp_path / "config.toml" + config.write_text( + """ +version = "1.0" + +[evaluation] +max_steps = 1 +failure_return = -1000.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.7 +seeds = [701] + +[[evaluation.suites]] +name = "hidden_robustness" +weight = 0.3 +seeds = [703] + +[[evaluation.variants]] +name = "test_variant" +density_scale = 1.02 +friction_scale = 0.98 +damping_scale = 1.01 +actuator_scale = 0.99 +""".lstrip() + ) + results = {} for name, policy in (("starter", STARTER), ("reference", REFERENCE)): output = tmp_path / f"{name}.json" @@ -80,6 +118,8 @@ def test_anchor_normalization_is_platform_local(tmp_path: Path) -> None: sys.executable, str(HIDDEN_EVALUATOR), str(policy), + "--config", + str(config), "--output", str(output), ], @@ -91,3 +131,66 @@ def test_anchor_normalization_is_platform_local(tmp_path: Path) -> None: assert results["starter"]["normalized_score"] == 0 assert results["reference"]["normalized_score"] == 100 + assert results["starter"]["unique_policy_evaluations"] == 2 + assert results["reference"]["unique_policy_evaluations"] == 2 + + +def test_task_package_validates() -> None: + assert ( + validate_task( + TASK_DIR, + runtime_source=REPO_ROOT / "src" / "genesisbench", + ) + == [] + ) + + +def test_article_provenance_is_machine_readable() -> None: + provenance = json.loads(PROVENANCE.read_text()) + + assert ( + provenance["source"]["revision"] == "3555c2956c257d49a5015b782cbe485b14fd659e" + ) + assert provenance["article_reproduction"]["seeds"] == [0, 1, 2, 3, 4] + assert provenance["article_reproduction"]["reported_mean_return"] == 6005.521 + assert provenance["genesisbench_adaptation"]["source_and_gymnasium_xml_parse_equal"] + assert provenance["local_genesisbench_oracle_on_gymnasium"][ + "matches_local_source_policy" + ] + + +def test_verifier_timeout_fails_closed_before_benchflow_deadline() -> None: + script = VERIFIER_SCRIPT.read_text() + + assert "timeout --signal=TERM --kill-after=30s 3900s" in script + assert '"verifier_timeout": True' in script + assert '"normalized_score": 0.0' in script + + +def test_ci_smoke_config_is_bounded_and_covers_dynamics() -> None: + config = tomllib.loads(SMOKE_CONFIG.read_text())["evaluation"] + suites = {suite["name"]: suite for suite in config["suites"]} + + assert config["max_steps"] == 1 + assert suites["hidden_nominal"]["seeds"] == [101] + assert suites["hidden_robustness"]["seeds"] == [503] + assert len(config["variants"]) == 1 + assert config["variants"][0]["name"] == "light_low_friction" + + +@pytest.mark.skipif( + os.environ.get("GENESISBENCH_RUN_SLOW_ANT_MPC") != "1", + reason="set GENESISBENCH_RUN_SLOW_ANT_MPC=1 for the full MPC reproduction", +) +def test_article_mpc_full_reproduction() -> None: + result = evaluate_ant_policy( + ORACLE, + seeds=range(5), + max_steps=1000, + ) + + assert [episode.length for episode in result.episodes] == [1000] * 5 + assert result.invalid_episode_rate == 0.0 + assert result.mean_return == pytest.approx(6005.521, abs=200.0) + assert result.min_return == pytest.approx(5776.805, abs=100.0) + assert result.max_return == pytest.approx(6146.208, abs=100.0) diff --git a/tests/test_simulation_heuristics_atari57_v1.py b/tests/test_simulation_heuristics_atari57_v1.py new file mode 100644 index 0000000..39dd0d1 --- /dev/null +++ b/tests/test_simulation_heuristics_atari57_v1.py @@ -0,0 +1,661 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +import tomllib + +import numpy as np +import pytest + +from genesisbench.atari57 import ( + ATARI57_GAMES, + ATARI_SETTINGS, + Atari57ArtifactError, + AtariEpisode, + FRAME_BUDGET_PER_SEARCH, + HNSReference, + OBSERVATION_MODES, + aggregate_atari57_episodes, + evaluate_atari57_artifact, + expected_search_trajectories, + load_hns_references, + load_atari57_artifact, +) +from scripts.validate_tasks import validate_task + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TASK_DIR = REPO_ROOT / "tasks" / "simulation_heuristics_atari57_v1" +STARTER_ARTIFACT = TASK_DIR / "starter_artifact" +REFERENCE_ARTIFACT = TASK_DIR / "oracle" / "reference_artifact" +HIDDEN_EVALUATOR = TASK_DIR / "verifier" / "evaluate_hidden.py" +PUBLIC_EVALUATOR = TASK_DIR / "evaluate.py" +RUNTIME_SOURCE = REPO_ROOT / "src" / "genesisbench" + + +def test_article_protocol_expands_to_342_search_trajectories() -> None: + trajectories = expected_search_trajectories() + + assert len(ATARI57_GAMES) == 57 + assert len(set(ATARI57_GAMES)) == 57 + assert OBSERVATION_MODES == ("ram", "native_obs") + assert len(trajectories) == 342 + assert ( + len({(item.env_id, item.obs_mode, item.repeat_index) for item in trajectories}) + == 342 + ) + assert {item.repeat_index for item in trajectories} == {0, 1, 2} + assert sum(item.frame_budget for item in trajectories) == ( + 342 * FRAME_BUDGET_PER_SEARCH + ) + + +def test_starter_artifact_is_complete_and_honest_about_unused_budget() -> None: + artifact = load_atari57_artifact(STARTER_ARTIFACT) + + assert len(artifact.policies) == 342 + assert { + (policy.env_id, policy.obs_mode, policy.repeat_index) + for policy in artifact.policies + } == { + (env_id, obs_mode, repeat_index) + for env_id in ATARI57_GAMES + for obs_mode in OBSERVATION_MODES + for repeat_index in range(3) + } + assert all(policy.module_path.is_file() for policy in artifact.policies) + + budget = artifact.interaction_budget + assert budget.planned_trajectories == 342 + assert budget.completed_trajectories == 0 + assert budget.counted_env_steps == 0 + assert budget.target_env_steps == 6_840_000_000 + + +def test_completed_search_record_requires_article_evidence( + tmp_path: Path, +) -> None: + artifact_root = tmp_path / "artifact" + artifact_root.mkdir() + (artifact_root / "manifest.json").write_text( + (STARTER_ARTIFACT / "manifest.json").read_text() + ) + (artifact_root / "policy.py").write_text( + (STARTER_ARTIFACT / "policy.py").read_text() + ) + record = { + "env_id": "Breakout-v5", + "obs_mode": "native_obs", + "repeat_index": 0, + "cumulative_env_steps": FRAME_BUDGET_PER_SEARCH, + "cumulative_episodes": 100, + "status": "complete", + "evidence_path": "searches/breakout/native_obs/repeat_0", + } + ledger_path = artifact_root / "interaction_ledger.json" + ledger_path.write_text(json.dumps({"schema_version": "1.0", "records": [record]})) + + with pytest.raises( + Atari57ArtifactError, + match="complete search evidence is missing", + ): + load_atari57_artifact(artifact_root) + + evidence = artifact_root / record["evidence_path"] + evidence.mkdir(parents=True) + for filename in ("policy.py", "trials.jsonl", "summary.csv", "README.md"): + (evidence / filename).write_text("fixture\n") + (evidence / "sample_efficiency.png").write_bytes(b"fixture") + + artifact = load_atari57_artifact(artifact_root) + assert artifact.interaction_budget.completed_trajectories == 1 + assert artifact.interaction_records[0].evidence_path == evidence + + +def test_repeat_specific_policy_override_is_evaluated_for_that_repeat( + tmp_path: Path, +) -> None: + artifact_root = tmp_path / "artifact" + artifact_root.mkdir() + manifest = json.loads((STARTER_ARTIFACT / "manifest.json").read_text()) + manifest["policies"] = [ + { + "env_id": "Breakout-v5", + "obs_mode": "native_obs", + "repeat_index": 1, + "module": "repeat_one.py", + } + ] + (artifact_root / "manifest.json").write_text(json.dumps(manifest)) + (artifact_root / "interaction_ledger.json").write_text( + '{"schema_version": "1.0", "records": []}\n' + ) + (artifact_root / "policy.py").write_text( + """ +class Policy: + def act(self, observation, info=None): + return 0 +""".strip() + + "\n" + ) + (artifact_root / "repeat_one.py").write_text( + """ +class Policy: + def __init__(self, *, repeat_index, **kwargs): + assert repeat_index == 1 + + def act(self, observation, info=None): + return 1 +""".strip() + + "\n" + ) + + class FakeActionSpace: + n = 2 + + class FakeEnv: + action_space = FakeActionSpace() + + def reset(self): + return np.asarray([[0.0]]), {} + + def step(self, action): + return ( + np.asarray([[0.0]]), + np.asarray([float(action[0])]), + np.asarray([True]), + {}, + ) + + def close(self) -> None: + return None + + artifact = load_atari57_artifact(artifact_root) + repeat_one = next( + policy + for policy in artifact.policies + if ( + policy.env_id, + policy.obs_mode, + policy.repeat_index, + ) + == ("Breakout-v5", "native_obs", 1) + ) + assert repeat_one.module_path.name == "repeat_one.py" + + result = evaluate_atari57_artifact( + artifact, + games=("Breakout-v5",), + obs_modes=("native_obs",), + seeds=(10, 11, 12), + max_steps=1, + hns_references={ + "Breakout-v5": HNSReference( + env_id="Breakout-v5", + known_best_score=864.0, + random_score=0.0, + human_score=1.0, + ) + }, + env_factory=lambda env_id, seed, settings: FakeEnv(), + ) + + assert [episode.return_ for episode in result.episodes] == [0.0, 1.0, 0.0] + assert result.score == pytest.approx(1.0 / 3.0) + + +def test_hns_aggregation_matches_article_best_input_semantics() -> None: + values = { + "Alien-v5": { + "ram": (0.1, 0.2, 0.3), + "native_obs": (0.4, 0.5, 0.6), + }, + "Breakout-v5": { + "ram": (1.0, 2.0, 3.0), + "native_obs": (4.0, 0.0, 0.0), + }, + "Pong-v5": { + "ram": (-0.2, 0.0, 0.2), + "native_obs": (0.1, 0.1, 0.1), + }, + } + episodes = tuple( + AtariEpisode( + env_id=env_id, + obs_mode=obs_mode, + repeat_index=repeat_index, + seed=repeat_index, + return_=hns, + hns=hns, + length=1, + terminated=True, + truncated=False, + invalid_action=False, + policy_error=None, + mean_action_latency_ms=0.0, + ) + for env_id, modes in values.items() + for obs_mode, repeats in modes.items() + for repeat_index, hns in enumerate(repeats) + ) + + result = aggregate_atari57_episodes(episodes) + + assert result.evaluation_trajectories == 18 + assert result.games_evaluated == 3 + assert result.per_game["Alien-v5"]["best_input_mean_hns"] == 0.5 + assert result.per_game["Alien-v5"]["modes"]["ram"]["repeat_count"] == 3 + assert result.per_game["Breakout-v5"]["best_input_mean_hns"] == 2.0 + assert result.per_game["Breakout-v5"]["best_single_run_hns"] == 4.0 + assert result.score == 0.5 + assert result.median_best_single_run_hns == 0.6 + + +def test_public_hns_table_covers_atari57_and_uses_random_human_anchors() -> None: + references = load_hns_references(TASK_DIR / "task_context" / "atari57_games.csv") + + assert len(references) == 57 + assert set(references) == set(ATARI57_GAMES) + + breakout = references["Breakout-v5"] + assert breakout.random_score == 1.7 + assert breakout.human_score == 30.5 + assert breakout.normalize(1.7) == 0.0 + assert breakout.normalize(30.5) == 1.0 + assert round(breakout.normalize(864.0), 12) == round( + 29.94097222222222, + 12, + ) + + +def test_evaluator_enforces_observation_mode_boundary_and_fixed_settings( + tmp_path: Path, +) -> None: + artifact_root = tmp_path / "artifact" + artifact_root.mkdir() + manifest = json.loads((STARTER_ARTIFACT / "manifest.json").read_text()) + (artifact_root / "manifest.json").write_text(json.dumps(manifest)) + (artifact_root / "interaction_ledger.json").write_text( + '{"schema_version": "1.0", "records": []}\n' + ) + (artifact_root / "policy.py").write_text( + """ +class Policy: + def __init__(self, *, obs_mode, action_count, **kwargs): + self.obs_mode = obs_mode + self.action_count = action_count + + def reset(self, seed=0): + self.seed = seed + + def act(self, observation, info=None): + if self.obs_mode == "native_obs" and info is not None: + raise RuntimeError("native_obs leaked info") + if self.obs_mode == "ram": + if set(info) != {"ram"}: + raise RuntimeError("RAM policy received hidden info") + if int(info["ram"][0]) != 9: + raise RuntimeError("RAM payload was not unbatched") + return 1 +""".strip() + + "\n" + ) + + factory_calls: list[tuple[str, int, dict[str, object]]] = [] + + class FakeActionSpace: + n = 3 + + class FakeEnv: + action_space = FakeActionSpace() + + def __init__(self) -> None: + self.steps = 0 + + def reset(self): + return ( + np.asarray([[1.0, 2.0]]), + { + "ram": np.asarray([[9, 8]], dtype=np.uint8), + "secret": np.asarray([123]), + }, + ) + + def step(self, action): + assert action.tolist() == [1] + self.steps += 1 + return ( + np.asarray([[3.0, 4.0]]), + np.asarray([1.0]), + np.asarray([self.steps == 2]), + { + "ram": np.asarray([[9, 7]], dtype=np.uint8), + "secret": np.asarray([456]), + }, + ) + + def close(self) -> None: + return None + + def fake_env_factory( + env_id: str, + seed: int, + settings: dict[str, object], + ) -> FakeEnv: + factory_calls.append((env_id, seed, settings)) + return FakeEnv() + + result = evaluate_atari57_artifact( + artifact_root, + games=("Breakout-v5",), + obs_modes=OBSERVATION_MODES, + seeds=(7,), + max_steps=5, + hns_references={ + "Breakout-v5": HNSReference( + env_id="Breakout-v5", + known_best_score=864.0, + random_score=0.0, + human_score=2.0, + ) + }, + env_factory=fake_env_factory, + ) + + assert result.score == 1.0 + assert result.evaluation_trajectories == 2 + assert result.counted_evaluation_steps == 4 + assert result.invalid_episode_rate == 0.0 + assert factory_calls == [ + ("Breakout-v5", 7, ATARI_SETTINGS), + ("Breakout-v5", 7, ATARI_SETTINGS), + ] + + +def test_invalid_policy_call_counts_zero_environment_steps( + tmp_path: Path, +) -> None: + artifact_root = tmp_path / "artifact" + artifact_root.mkdir() + (artifact_root / "manifest.json").write_text( + (STARTER_ARTIFACT / "manifest.json").read_text() + ) + (artifact_root / "interaction_ledger.json").write_text( + '{"schema_version": "1.0", "records": []}\n' + ) + (artifact_root / "policy.py").write_text( + """ +class Policy: + def act(self, observation, info=None): + return 99 +""".strip() + + "\n" + ) + + class FakeActionSpace: + n = 2 + + class FakeEnv: + action_space = FakeActionSpace() + + def reset(self): + return np.asarray([[0.0]]), {} + + def step(self, action): + raise AssertionError("invalid policy must fail before env.step") + + def close(self) -> None: + return None + + result = evaluate_atari57_artifact( + artifact_root, + games=("Breakout-v5",), + obs_modes=("native_obs",), + seeds=(1,), + max_steps=5, + hns_references={ + "Breakout-v5": HNSReference( + env_id="Breakout-v5", + known_best_score=864.0, + random_score=0.0, + human_score=1.0, + ) + }, + env_factory=lambda env_id, seed, settings: FakeEnv(), + ) + + assert result.counted_evaluation_steps == 0 + assert result.invalid_episode_rate == 1.0 + assert result.episodes[0].length == 0 + assert result.episodes[0].hns == 0.0 + + +def test_numeric_anchors_map_natural_origin_and_article_reference_to_0_100( + tmp_path: Path, +) -> None: + anchors_path = TASK_DIR / "verifier" / "anchors.json" + anchors = json.loads(anchors_path.read_text()) + assert anchors["starter_policy"]["score"] == 0.0 + assert anchors["reference_policy"]["score"] == 0.8283015254994576 + assert "path" not in anchors["starter_policy"] + assert "path" not in anchors["reference_policy"] + oracle = load_atari57_artifact(REFERENCE_ARTIFACT) + assert len(oracle.policies) == 342 + assert oracle.interaction_budget.completed_trajectories == 0 + + output = tmp_path / "numeric.json" + subprocess.run( + [ + sys.executable, + str(HIDDEN_EVALUATOR), + str(STARTER_ARTIFACT), + "--config", + str(TASK_DIR / "verifier" / "config_smoke.toml"), + "--anchors", + str(anchors_path), + "--output", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(output.read_text()) + assert payload["starter_score"] == 0.0 + assert payload["reference_score"] == 0.8283015254994576 + assert ( + 100.0 + * payload["reference_score"] + / (payload["reference_score"] - payload["starter_score"]) + == 100.0 + ) + assert payload["evaluation"]["evaluation_trajectories"] == 18 + + +def test_incomplete_artifact_is_disqualified_before_full_envpool_evaluation( + tmp_path: Path, +) -> None: + output = tmp_path / "incomplete.json" + completed = subprocess.run( + [ + sys.executable, + str(HIDDEN_EVALUATOR), + str(STARTER_ARTIFACT), + "--config", + str(TASK_DIR / "verifier" / "config.toml"), + "--anchors", + str(TASK_DIR / "verifier" / "anchors.json"), + "--output", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + + payload = json.loads(output.read_text()) + assert completed.returncode == 0 + assert payload["score"] == 0.0 + assert payload["normalized_score"] == 0.0 + assert payload["protocol_complete"] is False + assert payload["evaluation"] is None + assert "342 completed" in payload["disqualification_reason"] + + +def test_full_hidden_matrix_contract_has_342_evaluation_trajectories( + tmp_path: Path, +) -> None: + config = tmp_path / "full_deterministic.toml" + config.write_text( + f""" +version = "1.0" + +[evaluation] +backend = "deterministic" +games = "atari57" +obs_modes = "both" +seeds = [10001, 20001, 30001] +max_steps = 1 +hns_table = "{TASK_DIR / "verifier" / "hns_normalization.csv"}" +require_complete_search_ledger = false +""".strip() + + "\n" + ) + output = tmp_path / "full.json" + subprocess.run( + [ + sys.executable, + str(HIDDEN_EVALUATOR), + str(STARTER_ARTIFACT), + "--config", + str(config), + "--anchors", + str(TASK_DIR / "verifier" / "anchors.json"), + "--output", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + + payload = json.loads(output.read_text()) + assert payload["evaluation"]["games_evaluated"] == 57 + assert payload["evaluation"]["evaluation_trajectories"] == 342 + + +def test_private_complete_ledger_runs_only_after_protocol_preflight( + tmp_path: Path, +) -> None: + artifact_root = tmp_path / "artifact" + artifact_root.mkdir() + (artifact_root / "manifest.json").write_text( + (STARTER_ARTIFACT / "manifest.json").read_text() + ) + (artifact_root / "policy.py").write_text( + (STARTER_ARTIFACT / "policy.py").read_text() + ) + records = [] + for index, trajectory in enumerate(expected_search_trajectories()): + relative = Path("searches") / str(index) + evidence = artifact_root / relative + evidence.mkdir(parents=True) + for filename in ("policy.py", "trials.jsonl", "summary.csv", "README.md"): + (evidence / filename).write_text("fixture\n") + (evidence / "sample_efficiency.png").write_bytes(b"fixture") + records.append( + { + "env_id": trajectory.env_id, + "obs_mode": trajectory.obs_mode, + "repeat_index": trajectory.repeat_index, + "cumulative_env_steps": FRAME_BUDGET_PER_SEARCH, + "cumulative_episodes": 1, + "status": "complete", + "evidence_path": str(relative), + } + ) + (artifact_root / "interaction_ledger.json").write_text( + json.dumps({"schema_version": "1.0", "records": records}) + ) + + config = tmp_path / "private.toml" + config.write_text( + f""" +version = "1.0" + +[evaluation] +backend = "deterministic" +games = ["Breakout-v5"] +obs_modes = ["native_obs"] +seeds = [101, 202, 303] +max_steps = 4 +hns_table = "{TASK_DIR / "verifier" / "hns_normalization.csv"}" +require_complete_search_ledger = true +""".strip() + + "\n" + ) + output = tmp_path / "private.json" + subprocess.run( + [ + sys.executable, + str(HIDDEN_EVALUATOR), + str(artifact_root), + "--config", + str(config), + "--anchors", + str(TASK_DIR / "verifier" / "anchors.json"), + "--output", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + + payload = json.loads(output.read_text()) + assert payload["protocol_complete"] is True + assert payload["evaluation"] is not None + assert payload["interaction_budget"]["completed_trajectories"] == 342 + assert payload["interaction_budget"]["counted_env_steps"] == 6_840_000_000 + + +def test_native_task_package_and_public_validation_entrypoint( + tmp_path: Path, +) -> None: + assert ( + validate_task( + TASK_DIR, + runtime_source=RUNTIME_SOURCE, + ) + == [] + ) + + output = tmp_path / "validation.json" + subprocess.run( + [ + sys.executable, + str(PUBLIC_EVALUATOR), + "--artifact", + str(STARTER_ARTIFACT), + "--validate-only", + "--json-output-file", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(output.read_text()) + assert payload["valid"] is True + assert payload["policy_slots"] == 342 + assert payload["interaction_budget"]["planned_trajectories"] == 342 + + task_text = (TASK_DIR / "task.md").read_text() + full_config = tomllib.loads((TASK_DIR / "verifier" / "config.toml").read_text()) + assert full_config["evaluation"]["require_complete_search_ledger"] is True + assert "57 games x 2 observation modes x 3 repeats = 342" in task_text + assert "6,840,000,000" in task_text + assert "0.8283015254994576" in task_text + assert "1.1813031161473089" in task_text diff --git a/tests/test_simulation_heuristics_breakout_v1.py b/tests/test_simulation_heuristics_breakout_v1.py new file mode 100644 index 0000000..6f8886c --- /dev/null +++ b/tests/test_simulation_heuristics_breakout_v1.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import sys +import tomllib + +import pytest +import yaml + +from genesisbench.breakout import _validate_action, evaluate_breakout_policy + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TASK_ROOT = REPO_ROOT / "tasks" +TASKS = { + "ram": TASK_ROOT / "simulation_heuristics_breakout_ram_v1", + "rgb": TASK_ROOT / "simulation_heuristics_breakout_rgb_v1", +} + + +def _envpool_runtime_available() -> bool: + try: + import envpool + + env = envpool.make_gym( + "Breakout-v5", + num_envs=1, + batch_size=1, + seed=0, + max_episode_steps=2, + img_height=210, + img_width=160, + stack_num=1, + gray_scale=False, + frame_skip=1, + noop_max=1, + use_fire_reset=True, + episodic_life=False, + reward_clip=False, + repeat_action_probability=0.0, + full_action_space=False, + ) + close = getattr(env, "close", None) + if close is not None: + close() + except Exception: + return False + return True + + +ENVPOOL_RUNTIME_AVAILABLE = _envpool_runtime_available() +requires_envpool = pytest.mark.skipif( + not ENVPOOL_RUNTIME_AVAILABLE, + reason="envpool==1.1.1 is not installed on this platform", +) + + +def _task_config(task_dir: Path) -> dict: + document = task_dir.joinpath("task.md").read_text() + _, front_matter, _ = document.split("---", 2) + return yaml.safe_load(front_matter) + + +def _load_hidden_evaluator(task_dir: Path): + path = task_dir / "verifier" / "evaluate_hidden.py" + name = f"test_hidden_{task_dir.name}" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + ("mode", "task_name", "development_steps"), + ( + ("ram", "genesisbench/simulation_heuristics_breakout_ram_v1", 27000), + ("rgb", "genesisbench/simulation_heuristics_breakout_rgb_v1", 30000), + ), +) +def test_native_packages_declare_separate_observation_contracts( + mode: str, + task_name: str, + development_steps: int, +) -> None: + task_dir = TASKS[mode] + config = _task_config(task_dir) + + assert config["task"]["name"] == task_name + assert config["metadata"]["genesisbench"]["development"] == { + "episodes": 1, + "max_steps": development_steps, + "seeds": [0], + "observation_mode": mode, + } + assert config["metadata"]["genesisbench"]["submission"] == { + "directory": "final_policy", + "entrypoint": "policy.py", + } + + required = ( + "README.md", + "task.md", + "evaluate.py", + "environment/Dockerfile", + "starter_policy/policy.py", + "oracle/policy.py", + "oracle/solve.sh", + "task_context/evaluation.md", + "task_context/article_progression.md", + "task_context/policy_api.md", + "task_context/provenance.md", + "verifier/anchors.json", + "verifier/config.toml", + "verifier/evaluate_hidden.py", + "verifier/test.sh", + "verifier/verifier.md", + "verifier/rubrics/verifier.md", + "verifier/rubrics/verifier.toml", + ) + assert all(task_dir.joinpath(path).is_file() for path in required) + assert "cp -R /app/starter_policy /app/final_policy" in task_dir.joinpath( + "environment/Dockerfile" + ).read_text() + dockerfile = task_dir.joinpath("environment/Dockerfile").read_text() + assert '"envpool==1.1.1"' in dockerfile + assert "AutoROM" not in dockerfile + + +def test_runtime_uses_article_envpool_configuration() -> None: + source = REPO_ROOT.joinpath("src/genesisbench/breakout.py").read_text() + for snippet in ( + 'version("envpool")', + '"Breakout-v5"', + "frame_skip=1", + "noop_max=1", + "use_fire_reset=True", + "episodic_life=False", + "reward_clip=False", + "repeat_action_probability=variant.repeat_action_probability", + "full_action_space=False", + ): + assert snippet in source + + +def test_action_validation_accepts_only_reduced_breakout_actions() -> None: + assert [_validate_action(action) for action in (0, 1, 2, 3)] == [ + 0, + 1, + 2, + 3, + ] + with pytest.raises(ValueError): + _validate_action(4) + with pytest.raises(ValueError): + _validate_action(1.5) + with pytest.raises(ValueError): + _validate_action([1, 2]) + + +@requires_envpool +@pytest.mark.parametrize( + ("mode", "expected_shape"), + (("ram", (128,)), ("rgb", (3, 210, 160))), +) +def test_evaluator_passes_only_the_declared_observation( + tmp_path: Path, + mode: str, + expected_shape: tuple[int, ...], +) -> None: + policy = tmp_path / f"{mode}_policy.py" + policy.write_text( + "import numpy as np\n" + f"EXPECTED = {expected_shape!r}\n" + "class Policy:\n" + " def reset(self, seed=0):\n" + " self.calls = 0\n" + " def act(self, observation):\n" + " value = np.asarray(observation)\n" + " assert value.shape == EXPECTED\n" + " assert value.dtype == np.uint8\n" + " self.calls += 1\n" + " return 0\n" + ) + + result = evaluate_breakout_policy( + policy, + observation_mode=mode, + seeds=(7,), + max_steps=2, + ) + + assert result.episodes[0].length == 2 + assert result.invalid_episode_rate == 0.0 + + +@requires_envpool +@pytest.mark.parametrize("mode", ("ram", "rgb")) +def test_starter_policy_short_smoke(mode: str) -> None: + result = evaluate_breakout_policy( + TASKS[mode] / "starter_policy" / "policy.py", + observation_mode=mode, + seeds=(0,), + max_steps=10, + ) + + assert result.episodes[0].length == 10 + assert result.invalid_episode_rate == 0.0 + assert result.mean_return == 0.0 + + +@requires_envpool +def test_rgb_intermediate_reproduces_428(tmp_path: Path) -> None: + starter_source = TASKS["rgb"].joinpath("starter_policy/policy.py").read_text() + assert "CHASE_LEAD_STEPS = 6.0" in starter_source + intermediate = tmp_path / "rgb_428_policy.py" + intermediate.write_text( + starter_source.replace( + "CHASE_LEAD_STEPS = 6.0", + "CHASE_LEAD_STEPS = 8.0", + 1, + ) + ) + + result = evaluate_breakout_policy( + intermediate, + observation_mode="rgb", + seeds=(0,), + max_steps=30000, + ) + + assert result.mean_return == 428.0 + + +@requires_envpool +@pytest.mark.parametrize( + ("mode", "starter_nominal"), + (("ram", 387.0), ("rgb", 310.0)), +) +def test_anchor_normalization_and_article_target( + mode: str, + starter_nominal: float, +) -> None: + task_dir = TASKS[mode] + hidden = _load_hidden_evaluator(task_dir) + evaluation = tomllib.loads( + task_dir.joinpath("verifier/config.toml").read_text() + )["evaluation"] + anchors = json.loads(task_dir.joinpath("verifier/anchors.json").read_text()) + + starter_path = task_dir / "verifier" / anchors["starter_policy"]["path"] + reference_path = task_dir / "verifier" / anchors["reference_policy"]["path"] + assert starter_path.read_bytes() == task_dir.joinpath( + "starter_policy/policy.py" + ).read_bytes() + assert reference_path.read_bytes() == task_dir.joinpath( + "oracle/policy.py" + ).read_bytes() + + starter_suites, starter_score = hidden._evaluate_raw( + starter_path, + evaluation=evaluation, + ) + reference_suites, reference_score = hidden._evaluate_raw( + reference_path, + evaluation=evaluation, + ) + + assert starter_score < reference_score + assert reference_score == 864.0 + assert starter_suites["hidden_nominal"].mean_return == starter_nominal + assert all( + episode.return_ == 864.0 + for result in reference_suites.values() + for episode in result.episodes + ) + assert 100.0 * (starter_score - starter_score) / ( + reference_score - starter_score + ) == 0.0 + assert 100.0 * (reference_score - starter_score) / ( + reference_score - starter_score + ) == 100.0 + assert starter_suites diff --git a/tests/test_simulation_heuristics_halfcheetah_v1.py b/tests/test_simulation_heuristics_halfcheetah_v1.py new file mode 100644 index 0000000..abd268c --- /dev/null +++ b/tests/test_simulation_heuristics_halfcheetah_v1.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + +from genesisbench.halfcheetah import ( + DynamicsVariant, + evaluate_halfcheetah_policy, +) +from scripts.prepare_task import prepare_task +from scripts.validate_tasks import validate_task + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TASK_DIR = REPO_ROOT / "tasks" / "simulation_heuristics_halfcheetah_v1" +STARTER = TASK_DIR / "starter_policy" / "policy.py" +REFERENCE = TASK_DIR / "verifier" / "anchor_policies" / "reference_policy.py" +HIDDEN_EVALUATOR = TASK_DIR / "verifier" / "evaluate_hidden.py" +ARTICLE_EVIDENCE = TASK_DIR / "evidence" / "article_reproduction.json" + + +def test_starter_policy_short_smoke() -> None: + result = evaluate_halfcheetah_policy( + STARTER, + seeds=(100,), + max_steps=10, + ) + + assert len(result.episodes) == 1 + assert result.episodes[0].length == 10 + assert result.invalid_episode_rate == 0.0 + assert result.mean_return > 0.0 + + +def test_dynamics_variant_short_smoke() -> None: + result = evaluate_halfcheetah_policy( + STARTER, + seeds=(101,), + max_steps=5, + variants=( + DynamicsVariant( + name="test_variant", + mass_scale=1.05, + friction_scale=0.95, + damping_scale=1.05, + actuator_scale=0.95, + ), + ), + ) + + assert result.episodes[0].variant == "test_variant" + assert result.invalid_episode_rate == 0.0 + + +def test_json_uses_public_return_key() -> None: + result = evaluate_halfcheetah_policy( + STARTER, + seeds=(102,), + max_steps=2, + ) + + episode = result.to_dict()["episodes"][0] + assert "return" in episode + assert "return_" not in episode + + +def test_reference_policy_can_plan_from_observations() -> None: + result = evaluate_halfcheetah_policy( + REFERENCE, + seeds=(100,), + max_steps=1, + ) + + assert result.episodes[0].length == 1 + assert result.invalid_episode_rate == 0.0 + assert result.mean_action_latency_ms > 0.0 + + +def test_task_package_validates() -> None: + assert ( + validate_task( + TASK_DIR, + runtime_source=REPO_ROOT / "src" / "genesisbench", + ) + == [] + ) + + +def test_prepared_workspace_seeds_final_policy(tmp_path: Path) -> None: + prepared = prepare_task( + TASK_DIR.name, + tmp_path / TASK_DIR.name, + tasks_root=TASK_DIR.parent, + runtime_source=REPO_ROOT / "src" / "genesisbench", + ) + + assert not (prepared / "verifier").exists() + assert not (prepared / "oracle").exists() + assert not (prepared / "evidence").exists() + assert ( + prepared / "final_policy" / "policy.py" + ).read_bytes() == STARTER.read_bytes() + + +def test_anchor_normalization_is_platform_local(tmp_path: Path) -> None: + config = tmp_path / "config.toml" + config.write_text( + """ +version = "1.0" + +[evaluation] +max_steps = 2 +failure_return = -1000.0 + +[[evaluation.suites]] +name = "hidden_nominal" +weight = 0.7 +seeds = [701] + +[[evaluation.suites]] +name = "hidden_robustness" +weight = 0.3 +seeds = [703] + +[[evaluation.variants]] +name = "test_variant" +mass_scale = 1.02 +friction_scale = 0.98 +damping_scale = 1.01 +actuator_scale = 0.99 +""".lstrip() + ) + + results = {} + for name, policy in (("starter", STARTER), ("reference", REFERENCE)): + output = tmp_path / f"{name}.json" + subprocess.run( + [ + sys.executable, + str(HIDDEN_EVALUATOR), + str(policy), + "--config", + str(config), + "--output", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + results[name] = json.loads(output.read_text()) + + assert results["starter"]["normalized_score"] == 0 + assert results["reference"]["normalized_score"] == 100 + + +def test_article_five_seed_reproduction_is_recorded() -> None: + evidence = json.loads(ARTICLE_EVIDENCE.read_text()) + + assert evidence["article"]["seeds"] == [100, 101, 102, 103, 104] + assert evidence["observed"]["mean_return"] == 11836.693449819431 + assert evidence["observed"]["min_return"] == 11735.02927325886 + assert evidence["observed"]["max_return"] == 12041.189857475818 diff --git a/tests/test_simulation_heuristics_montezuma_v1.py b/tests/test_simulation_heuristics_montezuma_v1.py new file mode 100644 index 0000000..b0ef4e6 --- /dev/null +++ b/tests/test_simulation_heuristics_montezuma_v1.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +import tomllib + +import numpy as np +import pytest + +from genesisbench.montezuma import MontezumaVariant, evaluate_montezuma_policy +from scripts.validate_tasks import validate_task + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TASK_DIR = REPO_ROOT / "tasks" / "simulation_heuristics_montezuma_v1" +STARTER = TASK_DIR / "starter_policy" / "policy.py" +REFERENCE = TASK_DIR / "oracle" / "policy.py" +REFERENCE_DATA = TASK_DIR / "oracle" / "reference_trajectory.npz" +HIDDEN_EVALUATOR = TASK_DIR / "verifier" / "evaluate_hidden.py" + + +class _ImageActionEnv: + def __init__(self) -> None: + self.state = 0 + self.closed = False + + def _observation(self) -> np.ndarray: + return np.full((1, 3, 4, 5), self.state, dtype=np.uint8) + + def reset(self) -> tuple[np.ndarray, dict[str, object]]: + self.state = 0 + return self._observation(), {"private_state": "not for the policy"} + + def step( + self, + actions: np.ndarray, + ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, dict[str, object]]: + action = int(np.asarray(actions).reshape(-1)[0]) + if action == self.state: + self.state += 1 + reward = 400.0 if self.state == 3 else 0.0 + terminated = self.state == 3 + return ( + self._observation(), + np.asarray([reward], dtype=np.float32), + np.asarray([terminated]), + np.asarray([False]), + {"private_state": self.state}, + ) + + def close(self) -> None: + self.closed = True + + +class _RecoveryEnv: + def __init__(self) -> None: + self.state = 0 + + def _observation(self) -> np.ndarray: + return np.full((1, 3, 4, 5), self.state, dtype=np.uint8) + + def reset(self) -> tuple[np.ndarray, dict[str, object]]: + self.state = 0 + return self._observation(), {} + + def step( + self, + actions: np.ndarray, + ) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, dict[str, object]]: + action = int(np.asarray(actions).reshape(-1)[0]) + failed = action != self.state + if not failed: + self.state += 1 + reward = 400.0 if self.state == 6 else 0.0 + terminated = failed or self.state == 6 + return ( + self._observation(), + np.asarray([reward], dtype=np.float32), + np.asarray([terminated]), + np.asarray([False]), + {}, + ) + + def close(self) -> None: + return None + + +def test_policy_receives_native_rgb_only_and_reaches_target( + tmp_path: Path, + monkeypatch, +) -> None: + policy = tmp_path / "policy.py" + policy.write_text( + """ +import numpy as np + + +class Policy: + def reset(self, seed=0): + self.seed = seed + + def act(self, observation): + frame = np.asarray(observation) + assert frame.shape == (3, 4, 5) + assert frame.dtype == np.uint8 + return int(frame[0, 0, 0]) +""".lstrip() + ) + environments: list[_ImageActionEnv] = [] + + def make_env(*, seed: int, max_steps: int) -> _ImageActionEnv: + assert seed == 17 + assert max_steps == 8 + environment = _ImageActionEnv() + environments.append(environment) + return environment + + monkeypatch.setattr( + "genesisbench.montezuma._make_montezuma_env", + make_env, + ) + + result = evaluate_montezuma_policy( + policy, + seeds=(17,), + variants=(MontezumaVariant(),), + max_steps=8, + ) + + assert result.mean_return == 400.0 + assert result.target_success_rate == 1.0 + assert result.invalid_episode_rate == 0.0 + assert result.episodes[0].policy_steps == 3 + assert environments[0].closed + + +def test_recovery_variant_requires_reentry_from_the_observed_state( + tmp_path: Path, + monkeypatch, +) -> None: + image_policy = tmp_path / "image_policy.py" + image_policy.write_text( + """ +import numpy as np + + +class Policy: + def reset(self, seed=0): + pass + + def act(self, observation): + return int(np.asarray(observation)[0, 0, 0]) +""".lstrip() + ) + open_loop_policy = tmp_path / "open_loop_policy.py" + open_loop_policy.write_text( + """ +class Policy: + def reset(self, seed=0): + self.step = 0 + + def act(self, observation): + action = self.step + self.step += 1 + return action +""".lstrip() + ) + monkeypatch.setattr( + "genesisbench.montezuma._make_montezuma_env", + lambda **_: _RecoveryEnv(), + ) + variant = MontezumaVariant( + name="checkpoint_reentry", + bootstrap_steps=3, + ) + + recovered = evaluate_montezuma_policy( + image_policy, + seeds=(5,), + variants=(variant,), + bootstrap_policy_path=image_policy, + max_steps=12, + ) + copied_route = evaluate_montezuma_policy( + open_loop_policy, + seeds=(5,), + variants=(variant,), + bootstrap_policy_path=image_policy, + max_steps=12, + ) + + assert recovered.mean_return == 400.0 + assert recovered.recovery_success_rate == 1.0 + assert recovered.episodes[0].policy_steps == 3 + assert copied_route.mean_return == 0.0 + assert copied_route.recovery_success_rate == 0.0 + + +def test_invalid_action_receives_fixed_failure_score( + tmp_path: Path, + monkeypatch, +) -> None: + policy = tmp_path / "policy.py" + policy.write_text( + """ +class Policy: + def act(self, observation): + return 18 +""".lstrip() + ) + monkeypatch.setattr( + "genesisbench.montezuma._make_montezuma_env", + lambda **_: _ImageActionEnv(), + ) + + result = evaluate_montezuma_policy( + policy, + seeds=(0,), + max_steps=4, + failure_score=-25.0, + ) + + episode = result.episodes[0] + assert episode.return_ == -25.0 + assert episode.invalid_action + assert episode.policy_error is not None + assert result.invalid_episode_rate == 1.0 + assert "return" in result.to_dict()["episodes"][0] + assert "return_" not in result.to_dict()["episodes"][0] + + +def test_native_task_package_and_history_validate() -> None: + assert ( + validate_task( + TASK_DIR, + runtime_source=REPO_ROOT / "src" / "genesisbench", + ) + == [] + ) + + task_text = (TASK_DIR / "task.md").read_text() + provenance = (TASK_DIR / "task_context" / "provenance.md").read_text() + assert "final_policy/policy.py" in task_text + assert "native RGB" in task_text + assert "`72` to `28`" in provenance + assert "`86`" in provenance + assert "`1769`" in provenance + assert "mostly" in provenance and "open-loop" in provenance + + config = tomllib.loads((TASK_DIR / "verifier" / "config.toml").read_text()) + weights = { + suite["name"]: suite["weight"] for suite in config["evaluation"]["suites"] + } + assert weights["recovery"] > 0.5 + assert sum(weight for name, weight in weights.items() if name != "recovery") < 0.5 + + with np.load(REFERENCE_DATA, allow_pickle=False) as trajectory: + assert set(trajectory.files) == {"actions", "hashes", "features"} + assert trajectory["actions"].shape == (1769,) + + +def test_real_reference_reproduces_400_and_blocks_plain_replay( + tmp_path: Path, +) -> None: + pytest.importorskip("envpool") + canonical = evaluate_montezuma_policy( + REFERENCE, + seeds=(10001,), + max_steps=2000, + ) + assert canonical.mean_return == 400.0 + assert canonical.episodes[0].length == 1769 + + open_loop = tmp_path / "policy.py" + open_loop.write_text( + f""" +from pathlib import Path +import numpy as np + +with np.load(Path({str(REFERENCE_DATA)!r}), allow_pickle=False) as data: + ACTIONS = np.asarray(data["actions"], dtype=np.int64) + + +class Policy: + def reset(self, seed=0): + self.step = 0 + + def act(self, observation): + del observation + if self.step >= len(ACTIONS): + return 0 + action = int(ACTIONS[self.step]) + self.step += 1 + return action +""".lstrip() + ) + recovery_variants = ( + MontezumaVariant("recover_512_noop_4", 512, 4), + MontezumaVariant("recover_512_noop_8", 512, 8), + MontezumaVariant("recover_768_noop_2", 768, 2), + MontezumaVariant("recover_900", 900, 0), + ) + recovered = evaluate_montezuma_policy( + REFERENCE, + seeds=(10001,), + variants=recovery_variants, + bootstrap_policy_path=REFERENCE, + ) + copied_route = evaluate_montezuma_policy( + open_loop, + seeds=(10001,), + variants=recovery_variants, + bootstrap_policy_path=REFERENCE, + ) + + assert recovered.capped_mean_score == 400.0 + assert recovered.recovery_success_rate == 1.0 + assert copied_route.capped_mean_score == 0.0 + assert copied_route.recovery_success_rate == 0.0 + + +def test_anchor_normalization_is_platform_local(tmp_path: Path) -> None: + pytest.importorskip("envpool") + results = {} + for name, policy in (("starter", STARTER), ("reference", REFERENCE)): + output = tmp_path / f"{name}.json" + subprocess.run( + [ + sys.executable, + str(HIDDEN_EVALUATOR), + str(policy), + "--output", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + results[name] = json.loads(output.read_text()) + + assert results["starter"]["normalized_score"] == 0 + assert results["reference"]["normalized_score"] == 100 diff --git a/tests/test_simulation_heuristics_pong_ram_v1.py b/tests/test_simulation_heuristics_pong_ram_v1.py new file mode 100644 index 0000000..0f5a8b9 --- /dev/null +++ b/tests/test_simulation_heuristics_pong_ram_v1.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +from importlib.util import find_spec +import json +from pathlib import Path +import subprocess +import sys +import tomllib +from types import SimpleNamespace + +import numpy as np +import pytest + +from genesisbench.pong import PongVariant, evaluate_pong_policy + + +REPO_ROOT = Path(__file__).resolve().parents[1] +TASK_DIR = REPO_ROOT / "tasks" / "simulation_heuristics_pong_ram_v1" +STARTER = TASK_DIR / "starter_policy" / "policy.py" +REFERENCE = TASK_DIR / "oracle" / "policy.py" +HIDDEN_EVALUATOR = TASK_DIR / "verifier" / "evaluate_hidden.py" + + +class ScriptedPongEnv: + def __init__(self) -> None: + self.action_space = SimpleNamespace(n=6) + self.actions: list[int] = [] + self.step_index = 0 + + @staticmethod + def _info() -> dict[str, np.ndarray]: + ram = np.zeros((1, 128), dtype=np.uint8) + ram[0, 49] = 120 + ram[0, 50] = 100 + ram[0, 51] = 100 + ram[0, 54] = 100 + return {"ram": ram} + + def reset(self) -> tuple[np.ndarray, dict[str, np.ndarray]]: + self.step_index = 0 + return np.zeros((1, 1), dtype=np.uint8), self._info() + + def step( + self, + action: np.ndarray, + ) -> tuple[ + np.ndarray, + np.ndarray, + np.ndarray, + np.ndarray, + dict[str, np.ndarray], + ]: + self.actions.append(int(action[0])) + self.step_index += 1 + return ( + np.zeros((1, 1), dtype=np.uint8), + np.asarray([1.0]), + np.asarray([False]), + np.asarray([self.step_index == 2]), + self._info(), + ) + + def close(self) -> None: + pass + + +def test_policy_is_scored_from_ram_observations(tmp_path: Path) -> None: + policy = tmp_path / "policy.py" + policy.write_text( + "\n".join( + [ + "import numpy as np", + "", + "class Policy:", + " def reset(self, seed=0):", + " self.seed = seed", + "", + " def act(self, observation):", + " ram = np.asarray(observation)", + " assert ram.shape == (128,)", + " assert ram.dtype == np.uint8", + " return np.int64(0)", + "", + ] + ) + ) + environments: list[ScriptedPongEnv] = [] + + def make_env(seed: int, variant: PongVariant) -> ScriptedPongEnv: + assert seed == 7 + assert variant.name == "test" + environment = ScriptedPongEnv() + environments.append(environment) + return environment + + result = evaluate_pong_policy( + policy, + seeds=(7,), + max_steps=10, + variants=(PongVariant(name="test"),), + environment_factory=make_env, + ) + + assert result.mean_score == 2.0 + assert result.invalid_episode_rate == 0.0 + assert result.episodes[0].points_for == 2 + assert result.episodes[0].points_against == 0 + assert result.episodes[0].length == 2 + assert environments[0].actions == [0, 0] + + +def test_invalid_discrete_action_receives_failure_score( + tmp_path: Path, +) -> None: + policy = tmp_path / "policy.py" + policy.write_text( + "\n".join( + [ + "class Policy:", + " def act(self, observation):", + " return 99", + "", + ] + ) + ) + + result = evaluate_pong_policy( + policy, + seeds=(3,), + variants=(PongVariant(name="invalid"),), + environment_factory=lambda seed, variant: ScriptedPongEnv(), + ) + + assert result.mean_score == -21.0 + assert result.invalid_episode_rate == 1.0 + assert result.episodes[0].points_for == 0 + assert result.episodes[0].points_against == 21 + assert "must be in [0, 5]" in (result.episodes[0].policy_error or "") + + +@pytest.mark.skipif( + find_spec("envpool") is None, + reason="EnvPool is installed in the packaged Pong task environment.", +) +def test_reference_reproduces_article_score_21() -> None: + result = evaluate_pong_policy( + REFERENCE, + seeds=(0,), + max_steps=27_000, + variants=(PongVariant(name="article", noop_max=1),), + ) + + assert result.mean_score == 21.0 + assert result.target_score_rate == 1.0 + assert result.invalid_episode_rate == 0.0 + + +def test_native_task_package_validates_and_hides_trusted_files( + tmp_path: Path, +) -> None: + from scripts.prepare_task import prepare_task + from scripts.validate_tasks import validate_task + + runtime_source = REPO_ROOT / "src" / "genesisbench" + assert ( + validate_task( + TASK_DIR, + runtime_source=runtime_source, + ) + == [] + ) + + prepared = prepare_task( + TASK_DIR.name, + tmp_path / TASK_DIR.name, + tasks_root=TASK_DIR.parent, + runtime_source=runtime_source, + ) + + assert not (prepared / "verifier").exists() + assert not (prepared / "oracle").exists() + assert not (prepared / "evidence").exists() + assert (prepared / "final_policy" / "policy.py").read_bytes() == ( + STARTER.read_bytes() + ) + assert (prepared / "_runtime" / "genesisbench" / "pong.py").is_file() + + +@pytest.mark.skipif( + find_spec("envpool") is None, + reason="EnvPool is installed in the packaged Pong task environment.", +) +def test_anchor_normalization_is_platform_local(tmp_path: Path) -> None: + results = {} + for name, policy in (("starter", STARTER), ("reference", REFERENCE)): + output = tmp_path / f"{name}.json" + subprocess.run( + [ + sys.executable, + str(HIDDEN_EVALUATOR), + str(policy), + "--output", + str(output), + ], + check=True, + capture_output=True, + text=True, + ) + results[name] = json.loads(output.read_text()) + + assert results["starter"]["normalized_score"] == 0 + assert results["reference"]["normalized_score"] == 100 + assert results["reference"]["score"] == 21 + assert results["reference"]["reference_score"] == 21 + + +def test_hidden_suite_uses_multiple_seeds_and_reset_configs() -> None: + config = tomllib.loads((TASK_DIR / "verifier" / "config.toml").read_text()) + evaluation = config["evaluation"] + suites = evaluation["suites"] + variants = evaluation["variants"] + + seeds = [int(seed) for suite in suites for seed in suite["seeds"]] + assert len(seeds) >= 6 + assert len(set(seeds)) == len(seeds) + assert {suite["variant"] for suite in suites} == { + "article_nominal", + "randomized_reset", + } + assert {int(variant["noop_max"]) for variant in variants} == {1, 30} + assert all(int(variant["frame_skip"]) == 1 for variant in variants) + assert all( + float(variant["repeat_action_probability"]) == 0.0 for variant in variants + ) diff --git a/tests/test_simulation_heuristics_vizdoom_v1.py b/tests/test_simulation_heuristics_vizdoom_v1.py new file mode 100644 index 0000000..262b9b6 --- /dev/null +++ b/tests/test_simulation_heuristics_vizdoom_v1.py @@ -0,0 +1,714 @@ +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import tomllib + +import numpy as np +import pytest + +from genesisbench.vizdoom import ( + D1_ALLOWED_VARIABLES, + D3_ALLOWED_VARIABLES, + PolicySourceViolation, + VIZDOOM_ARTICLE_ENVPOOL_VERSION, + VizDoomEpisode, + VizDoomEvaluation, + _require_article_envpool_version, + _validate_d1_action, + _validate_d3_action, + audit_vizdoom_policy, + evaluate_vizdoom_policy, +) + + +REPO_ROOT = Path(__file__).resolve().parents[1] +D1_TASK = REPO_ROOT / "tasks" / "simulation_heuristics_vizdoom_d1_v1" +D3_TASK = REPO_ROOT / "tasks" / "simulation_heuristics_vizdoom_d3_v1" +D1_HIDDEN_EVALUATOR = D1_TASK / "verifier" / "evaluate_hidden.py" +D3_HIDDEN_EVALUATOR = D3_TASK / "verifier" / "evaluate_hidden.py" +RUN_SLOW = os.environ.get("GENESISBENCH_RUN_VIZDOOM_SLOW") == "1" + + +def _runtime_platform_key() -> str: + machine = platform.machine().lower() + machine = { + "amd64": "x86_64", + "aarch64": "arm64", + }.get(machine, machine) + return f"{sys.platform}-{machine}" + + +def _load_policy(path: Path): + spec = importlib.util.spec_from_file_location( + f"test_policy_{path.parent.parent.name}_{path.parent.name}", + path, + ) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.Policy() + + +@pytest.mark.parametrize("task", [D1_TASK, D3_TASK]) +def test_vizdoom_task_validates_and_prepares(task: Path, tmp_path: Path) -> None: + from scripts.prepare_task import prepare_task + from scripts.validate_tasks import validate_task + + runtime = REPO_ROOT / "src" / "genesisbench" + assert validate_task(task, runtime_source=runtime) == [] + + prepared = prepare_task( + task.name, + tmp_path / task.name, + tasks_root=task.parent, + runtime_source=runtime, + ) + assert not (prepared / "verifier").exists() + assert not (prepared / "oracle").exists() + assert (prepared / "final_policy" / "policy.py").is_file() + assert ( + prepared / "final_policy" / "policy.py" + ).read_bytes() == ( + task / "starter_policy" / "policy.py" + ).read_bytes() + + +def test_policy_input_allowlists_match_article_contracts() -> None: + assert D1_ALLOWED_VARIABLES == ("HEALTH",) + assert D3_ALLOWED_VARIABLES == ( + "HEALTH", + "AMMO2", + "HITCOUNT", + "DAMAGECOUNT", + "KILLCOUNT", + ) + + +def test_declared_runtime_matches_article_source() -> None: + assert VIZDOOM_ARTICLE_ENVPOOL_VERSION == "1.1.1" + for task in (D1_TASK, D3_TASK): + dockerfile = (task / "environment" / "Dockerfile").read_text() + anchors = json.loads((task / "verifier" / "anchors.json").read_text()) + assert '"envpool==1.1.1"' in dockerfile + assert "EnvPool 1.1.1" in anchors["environment"] + assert "1.2.5" not in dockerfile + assert "1.2.5" not in anchors["environment"] + + +def test_runtime_guard_rejects_silent_version_drift() -> None: + class WrongEnvPool: + __version__ = "1.2.5" + + with pytest.raises(RuntimeError, match="require EnvPool 1.1.1"): + _require_article_envpool_version(WrongEnvPool) + + +def test_action_validation_contracts() -> None: + assert _validate_d1_action(np.asarray(5)) == 5 + with pytest.raises(ValueError): + _validate_d1_action(np.asarray([1])) + with pytest.raises(ValueError): + _validate_d1_action(6) + + valid = _validate_d3_action(np.asarray([1, 1, 1, 0, 0, 1, 0, -8])) + assert valid.shape == (8,) + with pytest.raises(ValueError): + _validate_d3_action(np.zeros(7)) + with pytest.raises(ValueError): + _validate_d3_action(np.asarray([2, 0, 0, 0, 0, 0, 0, 0])) + + +def test_policy_source_audit_rejects_privileged_access(tmp_path: Path) -> None: + safe = tmp_path / "safe" / "policy.py" + safe.parent.mkdir() + safe.write_text( + "class Policy:\n" + " def act(self, frame, variables):\n" + " return 0\n" + ) + audit_vizdoom_policy(safe) + + privileged = tmp_path / "privileged" / "policy.py" + privileged.parent.mkdir() + privileged.write_text( + "import envpool\n" + "class Policy:\n" + " def act(self, frame, variables):\n" + " return 0\n" + ) + with pytest.raises(PolicySourceViolation): + audit_vizdoom_policy(privileged) + + +@pytest.mark.parametrize( + "path", + [ + D1_TASK / "oracle" / "policy.py", + D1_TASK / "verifier" / "anchor_policies" / "reference_policy.py", + D3_TASK / "oracle" / "policy.py", + D3_TASK / "verifier" / "anchor_policies" / "reference_policy.py", + ], +) +def test_reference_policy_sources_pass_privilege_audit(path: Path) -> None: + audit_vizdoom_policy(path) + + +def test_starter_policies_match_action_shapes() -> None: + d1 = _load_policy(D1_TASK / "starter_policy" / "policy.py") + d1_frame = np.zeros((180, 240, 3), dtype=np.uint8) + assert 0 <= d1.act(d1_frame, {"HEALTH": 100.0}) <= 5 + + d3 = _load_policy(D3_TASK / "starter_policy" / "policy.py") + d3_frame = np.zeros((480, 640, 3), dtype=np.uint8) + d3_action = d3.act( + d3_frame, + { + "HEALTH": 100.0, + "AMMO2": 20.0, + "HITCOUNT": 0.0, + "DAMAGECOUNT": 0.0, + "KILLCOUNT": 0.0, + }, + ) + assert d3_action.shape == (8,) + _validate_d3_action(d3_action) + + +def test_hidden_configs_use_unseen_seed_batches() -> None: + for task in (D1_TASK, D3_TASK): + config = tomllib.loads((task / "verifier" / "config.toml").read_text()) + seeds = [suite["seed"] for suite in config["evaluation"]["suites"]] + assert seeds + assert 0 not in seeds + assert len(seeds) == len(set(seeds)) + + +def test_anchor_metadata_records_article_targets() -> None: + d1 = json.loads((D1_TASK / "verifier" / "anchors.json").read_text()) + assert d1["public_article_target"]["envpool_version"] == "1.1.1" + assert d1["public_article_target"]["mean_return"] == pytest.approx( + 0.9440999741666019 + ) + assert d1["public_article_target"]["min_return"] == pytest.approx( + 0.28999998047947884 + ) + + d3 = json.loads((D3_TASK / "verifier" / "anchors.json").read_text()) + assert d3["public_article_target"]["envpool_version"] == "1.1.1" + assert d3["public_article_target"]["mean_return"] == 557.0 + assert d3["public_article_target"]["min_return"] == 440.0 + assert d3["public_article_target"]["returns"] == [ + 545, + 475, + 480, + 440, + 690, + 500, + 600, + 595, + 530, + 715, + ] + + +def test_d3_numeric_anchor_calibration_matches_hidden_config() -> None: + anchors = json.loads( + (D3_TASK / "verifier" / "anchors.json").read_text() + ) + config = tomllib.loads( + (D3_TASK / "verifier" / "config.toml").read_text() + ) + assert anchors["calibration"] == { + "envpool_version": "1.1.1", + "evaluation": config["evaluation"], + } + assert anchors["starter_policy"]["score"] == pytest.approx(31.9) + assert anchors["starter_policy"]["suite_mean_returns"] == { + "hidden_nominal": 28.0, + "hidden_secondary": 41.0, + } + assert anchors["reference_policy"]["score"] == pytest.approx(331.1) + assert anchors["reference_policy"]["suite_mean_returns"] == { + "hidden_nominal": 311.0, + "hidden_secondary": 378.0, + } + + +def test_d1_numeric_anchor_calibration_matches_hidden_config() -> None: + anchors = json.loads( + (D1_TASK / "verifier" / "anchors.json").read_text() + ) + config = tomllib.loads( + (D1_TASK / "verifier" / "config.toml").read_text() + ) + assert set(anchors["calibrations"]) == { + "darwin-arm64", + "linux-x86_64", + } + for calibration in anchors["calibrations"].values(): + assert calibration["envpool_version"] == "1.1.1" + assert calibration["evaluation"] == config["evaluation"] + assert calibration["starter_policy"]["score"] == pytest.approx( + 0.44161998964846133 + ) + assert calibration["starter_policy"]["suite_mean_returns"] == { + "hidden_nominal": pytest.approx(0.5395999886095524), + "hidden_secondary": pytest.approx(0.21299999207258224), + } + darwin = anchors["calibrations"]["darwin-arm64"] + assert darwin["reference_policy"]["score"] == pytest.approx( + 0.6982399759441613 + ) + assert darwin["reference_policy"]["suite_mean_returns"] == { + "hidden_nominal": pytest.approx(0.8401999741792678), + "hidden_secondary": pytest.approx(0.3669999800622463), + } + linux = anchors["calibrations"]["linux-x86_64"] + assert linux["reference_policy"]["score"] == pytest.approx( + 0.8178199748694897 + ) + assert linux["reference_policy"]["suite_mean_returns"] == { + "hidden_nominal": pytest.approx(0.9847999721765518), + "hidden_secondary": pytest.approx(0.42819998115301133), + } + assert linux["starter_policy"]["score"] == pytest.approx( + 0.44161998964846133 + ) + + +def test_evaluation_json_uses_public_return_key() -> None: + evaluation = VizDoomEvaluation( + scenario="d1", + policy_path="/tmp/policy.py", + envpool_version="1.1.1", + batch_seed=0, + max_steps=1, + frame_skip=1, + render_width=240, + render_height=180, + episodes=( + VizDoomEpisode( + seed=0, + lane=0, + return_=1.25, + length=1, + terminated=False, + truncated=True, + invalid_action=False, + policy_error=None, + mean_action_latency_ms=0.1, + final_variables={"HEALTH": 100.0}, + ), + ), + ) + episode = evaluation.to_dict()["episodes"][0] + assert episode["return"] == 1.25 + assert "return_" not in episode + + +@pytest.mark.skipif( + not RUN_SLOW, + reason="set GENESISBENCH_RUN_VIZDOOM_SLOW=1 for EnvPool regression", +) +def test_d1_article_reference_regression() -> None: + envpool = pytest.importorskip("envpool") + pytest.importorskip("cv2") + assert envpool.__version__ == VIZDOOM_ARTICLE_ENVPOOL_VERSION + result = evaluate_vizdoom_policy( + D1_TASK / "oracle" / "policy.py", + scenario="d1", + seed=0, + episodes=10, + max_steps=2100, + frame_skip=1, + render_width=240, + render_height=180, + failure_return=-1.0, + ) + assert result.envpool_version == "1.1.1" + assert result.mean_return == pytest.approx(0.9440999741666019) + assert result.min_return == pytest.approx(0.28999998047947884) + + +@pytest.mark.skipif( + not RUN_SLOW, + reason="set GENESISBENCH_RUN_VIZDOOM_SLOW=1 for EnvPool regression", +) +def test_d1_hidden_evaluator_isolates_each_suite_process( + tmp_path: Path, +) -> None: + envpool = pytest.importorskip("envpool") + assert envpool.__version__ == VIZDOOM_ARTICLE_ENVPOOL_VERSION + evaluation = { + "scenario": "d1", + "max_steps": 1, + "frame_skip": 1, + "render_width": 240, + "render_height": 180, + "failure_return": -1.0, + "suites": [ + { + "name": "smoke_a", + "weight": 0.5, + "seed": 911, + "episodes": 1, + }, + { + "name": "smoke_b", + "weight": 0.5, + "seed": 919, + "episodes": 1, + }, + ], + } + config = tmp_path / "config.toml" + config.write_text( + """version = "1.0" + +[evaluation] +scenario = "d1" +max_steps = 1 +frame_skip = 1 +render_width = 240 +render_height = 180 +failure_return = -1.0 + +[[evaluation.suites]] +name = "smoke_a" +weight = 0.5 +seed = 911 +episodes = 1 + +[[evaluation.suites]] +name = "smoke_b" +weight = 0.5 +seed = 919 +episodes = 1 +""" + ) + anchors = tmp_path / "anchors.json" + anchors.write_text( + json.dumps( + { + "calibrations": { + _runtime_platform_key(): { + "envpool_version": "1.1.1", + "evaluation": evaluation, + "starter_policy": { + "score": 0.0, + "suite_mean_returns": { + "smoke_a": 0.0, + "smoke_b": 0.0, + }, + }, + "reference_policy": { + "score": 1.0, + "suite_mean_returns": { + "smoke_a": 1.0, + "smoke_b": 1.0, + }, + }, + }, + }, + "starter_policy": { + "path": "unused-starter.py", + }, + "reference_policy": { + "path": "unused-reference.py", + }, + } + ) + ) + fake_shm = tmp_path / "dev-shm" + fake_boost = tmp_path / "boost-interprocess" + fake_shm.mkdir() + fake_boost.mkdir() + stale_size = 2_481_040 + stale_resources = [ + fake_shm / "ViZDoomSMstale-a", + fake_shm / "ViZDoomSMstale-b", + fake_shm / "sem.ViZDoomSMstale-c", + fake_boost / "ViZDoomSMstale-d", + fake_boost / "ViZDoomSMstale-e", + ] + for resource in stale_resources: + with resource.open("wb") as stream: + stream.truncate(stale_size) + unrelated = fake_shm / "unrelated-shared-memory" + unrelated.write_text("keep") + (tmp_path / "_vizdoom").mkdir() + output = tmp_path / "hidden.json" + worker_environment = os.environ.copy() + worker_environment["GENESISBENCH_VIZDOOM_RESOURCE_ROOTS"] = ( + f"{fake_shm}{os.pathsep}{fake_boost}" + ) + completed = subprocess.run( + [ + sys.executable, + str(D1_HIDDEN_EVALUATOR), + str(D1_TASK / "oracle" / "policy.py"), + "--config", + str(config), + "--anchors", + str(anchors), + "--output", + str(output), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=120, + env=worker_environment, + ) + assert completed.returncode == 0, ( + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + assert "Failed to create ./_vizdoom/" not in completed.stdout + payload = json.loads(output.read_text()) + assert payload["isolation"] == { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": True, + } + assert payload["anchor_calibration"]["platform"] == _runtime_platform_key() + assert payload["resource_cleanup"]["removed_resources"] == len( + stale_resources + ) + assert payload["resource_cleanup"]["removed_bytes"] == ( + len(stale_resources) * stale_size + ) + assert all(not resource.exists() for resource in stale_resources) + assert unrelated.read_text() == "keep" + + +@pytest.mark.skipif( + not RUN_SLOW or sys.platform != "linux", + reason="full hidden-oracle regression runs on the Linux verifier runtime", +) +def test_d1_hidden_evaluator_oracle_regression(tmp_path: Path) -> None: + envpool = pytest.importorskip("envpool") + assert envpool.__version__ == VIZDOOM_ARTICLE_ENVPOOL_VERSION + (tmp_path / "_vizdoom").mkdir() + output = tmp_path / "hidden.json" + completed = subprocess.run( + [ + sys.executable, + str(D1_HIDDEN_EVALUATOR), + str(D1_TASK / "oracle" / "policy.py"), + "--output", + str(output), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=300, + ) + assert completed.returncode == 0, ( + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + payload = json.loads(output.read_text()) + anchors = json.loads( + (D1_TASK / "verifier" / "anchors.json").read_text() + ) + calibration = anchors["calibrations"][_runtime_platform_key()] + assert payload["normalized_score"] == 100.0 + assert payload["score"] == pytest.approx( + calibration["reference_policy"]["score"] + ) + assert payload["starter_score"] == pytest.approx( + calibration["starter_policy"]["score"] + ) + assert payload["reference_score"] == pytest.approx( + calibration["reference_policy"]["score"] + ) + assert payload["isolation"]["mode"] == "subprocess_per_suite" + assert payload["anchor_calibration"]["platform"] == _runtime_platform_key() + + +@pytest.mark.skipif( + not RUN_SLOW, + reason="set GENESISBENCH_RUN_VIZDOOM_SLOW=1 for EnvPool regression", +) +def test_d3_article_reference_regression() -> None: + envpool = pytest.importorskip("envpool") + pytest.importorskip("cv2") + assert envpool.__version__ == VIZDOOM_ARTICLE_ENVPOOL_VERSION + result = evaluate_vizdoom_policy( + D3_TASK / "oracle" / "policy.py", + scenario="d3", + seed=0, + episodes=10, + max_steps=1050, + frame_skip=2, + render_width=640, + render_height=480, + failure_return=0.0, + ) + assert result.envpool_version == "1.1.1" + returns = [episode.return_ for episode in result.episodes] + assert returns == [ + 545.0, + 475.0, + 480.0, + 440.0, + 690.0, + 500.0, + 600.0, + 595.0, + 530.0, + 715.0, + ] + assert result.mean_return == 557.0 + assert result.min_return == 440.0 + + +@pytest.mark.skipif( + not RUN_SLOW, + reason="set GENESISBENCH_RUN_VIZDOOM_SLOW=1 for EnvPool regression", +) +def test_d3_hidden_evaluator_isolates_each_suite_process( + tmp_path: Path, +) -> None: + envpool = pytest.importorskip("envpool") + assert envpool.__version__ == VIZDOOM_ARTICLE_ENVPOOL_VERSION + evaluation = { + "scenario": "d3", + "max_steps": 1, + "frame_skip": 2, + "render_width": 640, + "render_height": 480, + "failure_return": 0.0, + "suites": [ + { + "name": "smoke_a", + "weight": 0.5, + "seed": 901, + "episodes": 1, + }, + { + "name": "smoke_b", + "weight": 0.5, + "seed": 907, + "episodes": 1, + }, + ], + } + config = tmp_path / "config.toml" + config.write_text( + """version = "1.0" + +[evaluation] +scenario = "d3" +max_steps = 1 +frame_skip = 2 +render_width = 640 +render_height = 480 +failure_return = 0.0 + +[[evaluation.suites]] +name = "smoke_a" +weight = 0.5 +seed = 901 +episodes = 1 + +[[evaluation.suites]] +name = "smoke_b" +weight = 0.5 +seed = 907 +episodes = 1 +""" + ) + anchors = tmp_path / "anchors.json" + anchors.write_text( + json.dumps( + { + "calibration": { + "envpool_version": "1.1.1", + "evaluation": evaluation, + }, + "starter_policy": { + "score": 0.0, + "suite_mean_returns": { + "smoke_a": 0.0, + "smoke_b": 0.0, + }, + }, + "reference_policy": { + "score": 1.0, + "suite_mean_returns": { + "smoke_a": 1.0, + "smoke_b": 1.0, + }, + }, + } + ) + ) + (tmp_path / "_vizdoom").mkdir() + output = tmp_path / "hidden.json" + completed = subprocess.run( + [ + sys.executable, + str(D3_HIDDEN_EVALUATOR), + str(D3_TASK / "oracle" / "policy.py"), + "--config", + str(config), + "--anchors", + str(anchors), + "--output", + str(output), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=120, + ) + assert completed.returncode == 0, ( + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + assert "Failed to create ./_vizdoom/" not in completed.stdout + payload = json.loads(output.read_text()) + assert payload["isolation"] == { + "mode": "subprocess_per_suite", + "suite_process_count": 2, + "unique_working_directories": True, + } + + +@pytest.mark.skipif( + not RUN_SLOW, + reason="set GENESISBENCH_RUN_VIZDOOM_SLOW=1 for EnvPool regression", +) +def test_d3_hidden_evaluator_oracle_regression(tmp_path: Path) -> None: + envpool = pytest.importorskip("envpool") + assert envpool.__version__ == VIZDOOM_ARTICLE_ENVPOOL_VERSION + (tmp_path / "_vizdoom").mkdir() + output = tmp_path / "hidden.json" + completed = subprocess.run( + [ + sys.executable, + str(D3_HIDDEN_EVALUATOR), + str(D3_TASK / "oracle" / "policy.py"), + "--output", + str(output), + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=300, + ) + assert completed.returncode == 0, ( + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + payload = json.loads(output.read_text()) + assert payload["normalized_score"] == 100.0 + assert payload["score"] == pytest.approx(331.1) + assert payload["starter_score"] == pytest.approx(31.9) + assert payload["reference_score"] == pytest.approx(331.1) + assert payload["isolation"]["mode"] == "subprocess_per_suite" diff --git a/tests/test_task_tooling.py b/tests/test_task_tooling.py index 18172d4..eb3a24b 100644 --- a/tests/test_task_tooling.py +++ b/tests/test_task_tooling.py @@ -5,7 +5,8 @@ from scripts.create_task import create_task from scripts.prepare_task import prepare_task -from scripts.validate_tasks import validate_task +from scripts.run_article_suite import TASKS as ARTICLE_TASKS +from scripts.validate_tasks import discover_tasks, validate_task REPO_ROOT = Path(__file__).resolve().parents[1] @@ -13,11 +14,16 @@ RUNTIME_SOURCE = REPO_ROOT / "src" / "genesisbench" -def test_reference_task_validates() -> None: - assert validate_task( - TASKS_ROOT / "simulation_heuristics_ant_v1", - runtime_source=RUNTIME_SOURCE, - ) == [] +def test_article_suite_contains_exactly_nine_valid_tasks() -> None: + discovered = discover_tasks(TASKS_ROOT) + + assert {task.name for task in discovered} == set(ARTICLE_TASKS) + assert len(discovered) == 9 + for task in discovered: + assert validate_task( + task, + runtime_source=RUNTIME_SOURCE, + ) == [] def test_create_and_prepare_task_scaffold(tmp_path: Path) -> None: diff --git a/uv.lock b/uv.lock index acd5121..f269520 100644 --- a/uv.lock +++ b/uv.lock @@ -29,6 +29,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/48/a1367dded7765f5489f9840389949d5aeac53a73aeb1b4e6c0ab61e1617a/agent_client_protocol-0.11.0-py3-none-any.whl", hash = "sha256:2d8570cd4911f8af9fbab4808f7b05f09204a756f346c7409ecdcae3f37cdb2f", size = 68089, upload-time = "2026-07-05T17:07:55.518Z" }, ] +[[package]] +name = "aiofiles" +version = "24.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/03/a88171e277e8caa88a4c77808c20ebb04ba74cc4681bf1e9416c862de237/aiofiles-24.1.0.tar.gz", hash = "sha256:22a075c9e5a3810f0c2e48f3008c94d68c65d763b9b03857924c99e57355166c", size = 30247, upload-time = "2024-06-24T11:02:03.584Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/45/30bb92d442636f570cb5651bc661f52b610e2eec3f891a5dc3a4c3667db0/aiofiles-24.1.0-py3-none-any.whl", hash = "sha256:b4ec55f4195e3eb5d7abd1bf7e061763e864dd4954231fb8539a0ef8bb8260e5", size = 15896, upload-time = "2024-06-24T11:02:01.529Z" }, +] + [[package]] name = "aiohappyeyeballs" version = "2.7.1" @@ -138,6 +147,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, ] +[[package]] +name = "aiohttp-retry" +version = "2.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/61/ebda4d8e3d8cfa1fd3db0fb428db2dd7461d5742cea35178277ad180b033/aiohttp_retry-2.9.1.tar.gz", hash = "sha256:8eb75e904ed4ee5c2ec242fefe85bf04240f685391c4879d8f541d6028ff01f1", size = 13608, upload-time = "2024-11-06T10:44:54.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/99/84ba7273339d0f3dfa57901b846489d2e5c2cd731470167757f1935fffbd/aiohttp_retry-2.9.1-py3-none-any.whl", hash = "sha256:66d2759d1921838256a05a3f80ad7e724936f083e35be5abb5e16eed6be6dc54", size = 9981, upload-time = "2024-11-06T10:44:52.917Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -277,6 +298,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c5/30/0caddec2ef504dc6a6cff10bae98a3936f6d9bfcac76bf4414f629b16482/benchflow-0.6.5-py3-none-any.whl", hash = "sha256:6ca34010ab3c47627214280c26b2bf487113534454b1c5a374bb6c3c34f7a87a", size = 913265, upload-time = "2026-07-11T05:34:50.515Z" }, ] +[package.optional-dependencies] +sandbox-daytona = [ + { name = "daytona" }, + { name = "tenacity" }, +] + [[package]] name = "boto3" version = "1.43.46" @@ -630,6 +657,111 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, ] +[[package]] +name = "daytona" +version = "0.196.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiofiles" }, + { name = "aiohttp" }, + { name = "daytona-api-client" }, + { name = "daytona-api-client-async" }, + { name = "daytona-toolbox-api-client" }, + { name = "daytona-toolbox-api-client-async" }, + { name = "deprecated" }, + { name = "httpx" }, + { name = "httpx-ws" }, + { name = "obstore" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-aiohttp-client" }, + { name = "opentelemetry-sdk" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "python-multipart" }, + { name = "toml" }, + { name = "urllib3" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/d5/c6fdcbeb0afc217b9cb88b0069fed0a144e57ff79382b27d479defdc9b6f/daytona-0.196.0.tar.gz", hash = "sha256:694cae4e1cf984f517a5938bfc24df61516fb99f552fe3c8287f485b5665bcda", size = 155416, upload-time = "2026-07-10T12:15:35.697Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/b6/e2fb2d79ac4a4f320cfb1b28871c5ba90a0951a268b96e510d16f4f2708c/daytona-0.196.0-py3-none-any.whl", hash = "sha256:0065f5eb3aa002a9d7fb69041153b0e65b9287be57311364a187fc59fbb5adc2", size = 188102, upload-time = "2026-07-10T12:15:37.236Z" }, +] + +[[package]] +name = "daytona-api-client" +version = "0.196.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/0a/6622fa4dbf0bac6d022c70e9d2829ea2700bddc425d053bcbbad8dedb9f6/daytona_api_client-0.196.0.tar.gz", hash = "sha256:d0ef69751b212abcf8d2ed560feda0edabe4523edc6c09c5a394343e1335f381", size = 156169, upload-time = "2026-07-10T12:08:05.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/7c/7a2de9eff668b49b5e85ab929a9a7a605da763b426d4749a7ebff9c0fd35/daytona_api_client-0.196.0-py3-none-any.whl", hash = "sha256:6bb567944a75cd03939a607dadc9599cfaf1477d072e0df9dcf7af0aa3b3d847", size = 431042, upload-time = "2026-07-10T12:08:07.218Z" }, +] + +[[package]] +name = "daytona-api-client-async" +version = "0.196.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiohttp-retry" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/cc/e42266475f7679b0c522d0622ad07f889f33450778a2023766f513813c39/daytona_api_client_async-0.196.0.tar.gz", hash = "sha256:f2e8170ebb822f70db691984b5d7e32b16d088e1fd5baf59a904bb3bb291cd6b", size = 156926, upload-time = "2026-07-10T12:08:05.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/13/d9d8d4241276a1d1117c302958fd4611a60185ef054e316ee4f137bd1639/daytona_api_client_async-0.196.0-py3-none-any.whl", hash = "sha256:ebd77e068d29aa81d0a2e7f95117bbd6cee878b6731334821b59e765798e301e", size = 434456, upload-time = "2026-07-10T12:08:08.114Z" }, +] + +[[package]] +name = "daytona-toolbox-api-client" +version = "0.196.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/2d/fe6ed38aec9b42eef1b11cc2eab846ec23e628dc18beaa2f3ae5037ed7ff/daytona_toolbox_api_client-0.196.0.tar.gz", hash = "sha256:4e72dfd89fabe905d96233b75e68a18c25c3bf25a99d4ad2a18bdfd1ce25cfe1", size = 86276, upload-time = "2026-07-10T12:09:49.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/99/5f2951e644d3c68d8aac7bba065a318cac557904ebcfe33aaeed7f5ed03b/daytona_toolbox_api_client-0.196.0-py3-none-any.whl", hash = "sha256:89548fc68fed8324b24eeb59137b4fba9c63eac315c3f17f08c0660fba1adec3", size = 247057, upload-time = "2026-07-10T12:09:50.463Z" }, +] + +[[package]] +name = "daytona-toolbox-api-client-async" +version = "0.196.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aiohttp-retry" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a0/d1/59c48db9571b0163e2c26d0b9a6204ac3e956fe4747a5dc5297126491c63/daytona_toolbox_api_client_async-0.196.0.tar.gz", hash = "sha256:fef1db3a3400e75158d54c3fe9d9916e2f74ebafdc2c5b6da49e712af16b40f4", size = 80154, upload-time = "2026-07-10T12:08:05.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/69/9ad8e2201007e4854978df275f9a265bbb484210b202b880166955385fc3/daytona_toolbox_api_client_async-0.196.0-py3-none-any.whl", hash = "sha256:11bb95461c68ce86d03a9289deb51378d2e41572872effd9059f8d7db7ae9583", size = 245526, upload-time = "2026-07-10T12:08:06.286Z" }, +] + +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -924,10 +1056,14 @@ dev = [ { name = "pytest" }, { name = "ruff" }, ] +sandbox-daytona = [ + { name = "benchflow", extra = ["sandbox-daytona"] }, +] [package.metadata] requires-dist = [ { name = "benchflow", marker = "extra == 'dev'", specifier = ">=0.6.5,<0.7" }, + { name = "benchflow", extras = ["sandbox-daytona"], marker = "extra == 'sandbox-daytona'", specifier = ">=0.6.5,<0.7" }, { name = "gymnasium", extras = ["mujoco"], specifier = ">=1.2.2,<2" }, { name = "matplotlib", marker = "extra == 'dev'", specifier = ">=3.10,<4" }, { name = "numpy", specifier = ">=1.26,<3" }, @@ -935,7 +1071,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6,<7" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.12,<1" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "sandbox-daytona"] [[package]] name = "glfw" @@ -969,6 +1105,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/9b/4366ad3e1c0688146c70aa6143584d6a8d88583b9390f106250e25a3d5cd/glfw-2.10.0-py2.py3-none-win_amd64.whl", hash = "sha256:7f787ee8645781f10e8800438ce4357ab38c573ffb191aba380c1e72eba6311c", size = 559423, upload-time = "2026-03-10T17:21:34.766Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, +] + [[package]] name = "granian" version = "2.7.9" @@ -1142,6 +1290,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, ] +[[package]] +name = "httpx-ws" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore" }, + { name = "httpx" }, + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/cd/ca91a07ae446451f7476bf3fcc909e98cb942ff032ebfda0e3fe449aca7b/httpx_ws-0.9.0.tar.gz", hash = "sha256:797373326f70eec1ae96f6e43ae9f12002fd7d73aee139a4985eaab964338a08", size = 107105, upload-time = "2026-03-28T14:11:10.781Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/f8/a6bc80313a9e93c888fa10534dfce2ad76ff86911b6f485777ce6de6a073/httpx_ws-0.9.0-py3-none-any.whl", hash = "sha256:71640d2fb1bf9a225775015b33cd755cfd4c5f7e21c885192fe3adc4c387b248", size = 15759, upload-time = "2026-03-28T14:11:11.887Z" }, +] + [[package]] name = "huggingface-hub" version = "1.23.0" @@ -1868,6 +2031,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, ] +[[package]] +name = "obstore" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/8c/9ec984edd0f3b72226adfaa19b1c61b15823b35b52f311ca4af36d009d15/obstore-0.8.2.tar.gz", hash = "sha256:a467bc4e97169e2ba749981b4fd0936015428d9b8f3fb83a5528536b1b6f377f", size = 168852, upload-time = "2025-09-16T15:34:55.786Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/dc/60fefbb5736e69eab56657bca04ca64dc07fdeccb3814164a31b62ad066b/obstore-0.8.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bb70ce297a47392b1d9a3e310f18d59cd5ebbb9453428210fef02ed60e4d75d1", size = 3612955, upload-time = "2025-09-16T15:33:29.527Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8b/844e8f382e5a12b8a3796a05d76a03e12c7aedc13d6900419e39207d7868/obstore-0.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1619bf618428abf1f607e0b219b2e230a966dcf697b717deccfa0983dd91f646", size = 3346564, upload-time = "2025-09-16T15:33:30.698Z" }, + { url = "https://files.pythonhosted.org/packages/89/73/8537f99e09a38a54a6a15ede907aa25d4da089f767a808f0b2edd9c03cec/obstore-0.8.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a4605c3ed7c9515aeb4c619b5f7f2c9986ed4a79fe6045e536b5e59b804b1476", size = 3460809, upload-time = "2025-09-16T15:33:31.837Z" }, + { url = "https://files.pythonhosted.org/packages/b4/99/7714dec721e43f521d6325a82303a002cddad089437640f92542b84e9cc8/obstore-0.8.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce42670417876dd8668cbb8659e860e9725e5f26bbc86449fd259970e2dd9d18", size = 3692081, upload-time = "2025-09-16T15:33:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bd/4ac4175fe95a24c220a96021c25c432bcc0c0212f618be0737184eebbaad/obstore-0.8.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4a3e893b2a06585f651c541c1972fe1e3bf999ae2a5fda052ee55eb7e6516f5", size = 3957466, upload-time = "2025-09-16T15:33:34.528Z" }, + { url = "https://files.pythonhosted.org/packages/4e/04/caa288fb735484fc5cb019bdf3d896eaccfae0ac4622e520d05692c46790/obstore-0.8.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08462b32f95a9948ed56ed63e88406e2e5a4cae1fde198f9682e0fb8487100ed", size = 3951293, upload-time = "2025-09-16T15:33:35.733Z" }, + { url = "https://files.pythonhosted.org/packages/44/2f/d380239da2d6a1fda82e17df5dae600a404e8a93a065784518ff8325d5f6/obstore-0.8.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a0bf7763292a8fc47d01cd66e6f19002c5c6ad4b3ed4e6b2729f5e190fa8a0d", size = 3766199, upload-time = "2025-09-16T15:33:36.904Z" }, + { url = "https://files.pythonhosted.org/packages/28/41/d391be069d3da82969b54266948b2582aeca5dd735abeda4d63dba36e07b/obstore-0.8.2-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:bcd47f8126cb192cbe86942b8f73b1c45a651ce7e14c9a82c5641dfbf8be7603", size = 3529678, upload-time = "2025-09-16T15:33:38.221Z" }, + { url = "https://files.pythonhosted.org/packages/b9/4c/4862fdd1a3abde459ee8eea699b1797df638a460af235b18ca82c8fffb72/obstore-0.8.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:57eda9fd8c757c3b4fe36cf3918d7e589cc1286591295cc10b34122fa36dd3fd", size = 3698079, upload-time = "2025-09-16T15:33:39.696Z" }, + { url = "https://files.pythonhosted.org/packages/68/ca/014e747bc53b570059c27e3565b2316fbe5c107d4134551f4cd3e24aa667/obstore-0.8.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ea44442aad8992166baa69f5069750979e4c5d9ffce772e61565945eea5774b9", size = 3687154, upload-time = "2025-09-16T15:33:40.92Z" }, + { url = "https://files.pythonhosted.org/packages/6f/89/6db5f8edd93028e5b8bfbeee15e6bd3e56f72106107d31cb208b57659de4/obstore-0.8.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:41496a3ab8527402db4142aaaf0d42df9d7d354b13ba10d9c33e0e48dd49dd96", size = 3773444, upload-time = "2025-09-16T15:33:42.123Z" }, + { url = "https://files.pythonhosted.org/packages/26/e5/c9e2cc540689c873beb61246e1615d6e38301e6a34dec424f5a5c63c1afd/obstore-0.8.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:43da209803f052df96c7c3cbec512d310982efd2407e4a435632841a51143170", size = 3939315, upload-time = "2025-09-16T15:33:43.252Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c9/bb53280ca50103c1ffda373cdc9b0f835431060039c2897cbc87ddd92e42/obstore-0.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:1836f5dcd49f9f2950c75889ab5c51fb290d3ea93cdc39a514541e0be3af016e", size = 3978234, upload-time = "2025-09-16T15:33:44.393Z" }, + { url = "https://files.pythonhosted.org/packages/f0/5d/8c3316cc958d386d5e6ab03e9db9ddc27f8e2141cee4a6777ae5b92f3aac/obstore-0.8.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:212f033e53fe6e53d64957923c5c88949a400e9027f7038c705ec2e9038be563", size = 3612027, upload-time = "2025-09-16T15:33:45.6Z" }, + { url = "https://files.pythonhosted.org/packages/ea/4d/699359774ce6330130536d008bfc32827fab0c25a00238d015a5974a3d1d/obstore-0.8.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bee21fa4ba148d08fa90e47a96df11161661ed31e09c056a373cb2154b0f2852", size = 3344686, upload-time = "2025-09-16T15:33:47.185Z" }, + { url = "https://files.pythonhosted.org/packages/82/37/55437341f10512906e02fd9fa69a8a95ad3f2f6a916d3233fda01763d110/obstore-0.8.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c66594b59832ff1ced4c72575d9beb8b5f9b4e404ac1150a42bfb226617fd50", size = 3459860, upload-time = "2025-09-16T15:33:48.382Z" }, + { url = "https://files.pythonhosted.org/packages/7a/51/4245a616c94ee4851965e33f7a563ab4090cc81f52cc73227ff9ceca2e46/obstore-0.8.2-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:089f33af5c2fe132d00214a0c1f40601b28f23a38e24ef9f79fb0576f2730b74", size = 3691648, upload-time = "2025-09-16T15:33:49.524Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f1/4e2fb24171e3ca3641a4653f006be826e7e17634b11688a5190553b00b83/obstore-0.8.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d87f658dfd340d5d9ea2d86a7c90d44da77a0db9e00c034367dca335735110cf", size = 3956867, upload-time = "2025-09-16T15:33:51.082Z" }, + { url = "https://files.pythonhosted.org/packages/42/f5/b703115361c798c9c1744e1e700d5908d904a8c2e2bd38bec759c9ffb469/obstore-0.8.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6e2e4fa92828c4fbc2d487f3da2d3588701a1b67d9f6ca3c97cc2afc912e9c63", size = 3950599, upload-time = "2025-09-16T15:33:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/53/20/08c6dc0f20c1394e2324b9344838e4e7af770cdcb52c30757a475f50daeb/obstore-0.8.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab440e89c5c37a8ec230857dd65147d4b923e0cada33297135d05e0f937d696a", size = 3765865, upload-time = "2025-09-16T15:33:53.291Z" }, + { url = "https://files.pythonhosted.org/packages/77/20/77907765e29b2eba6bd8821872284d91170d7084f670855b2dfcb249ea14/obstore-0.8.2-cp313-cp313-manylinux_2_24_aarch64.whl", hash = "sha256:b9beed107c5c9cd995d4a73263861fcfbc414d58773ed65c14f80eb18258a932", size = 3529807, upload-time = "2025-09-16T15:33:54.535Z" }, + { url = "https://files.pythonhosted.org/packages/a5/f5/f629d39cc30d050f52b1bf927e4d65c1cc7d7ffbb8a635cd546b5c5219a0/obstore-0.8.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b75b4e7746292c785e31edcd5aadc8b758238372a19d4c5e394db5c305d7d175", size = 3693629, upload-time = "2025-09-16T15:33:56.016Z" }, + { url = "https://files.pythonhosted.org/packages/30/ff/106763fd10f2a1cb47f2ef1162293c78ad52f4e73223d8d43fc6b755445d/obstore-0.8.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:f33e6c366869d05ab0b7f12efe63269e631c5450d95d6b4ba4c5faf63f69de70", size = 3686176, upload-time = "2025-09-16T15:33:57.247Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0c/d2ccb6f32feeca906d5a7c4255340df5262af8838441ca06c9e4e37b67d5/obstore-0.8.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:12c885a9ce5ceb09d13cc186586c0c10b62597eff21b985f6ce8ff9dab963ad3", size = 3773081, upload-time = "2025-09-16T15:33:58.475Z" }, + { url = "https://files.pythonhosted.org/packages/fa/79/40d1cc504cefc89c9b3dd8874287f3fddc7d963a8748d6dffc5880222013/obstore-0.8.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4accc883b93349a81c9931e15dd318cc703b02bbef2805d964724c73d006d00e", size = 3938589, upload-time = "2025-09-16T15:33:59.734Z" }, + { url = "https://files.pythonhosted.org/packages/14/dd/916c6777222db3271e9fb3cf9a97ed92b3a9b3e465bdeec96de9ab809d53/obstore-0.8.2-cp313-cp313-win_amd64.whl", hash = "sha256:ec850adf9980e5788a826ccfd5819989724e2a2f712bfa3258e85966c8d9981e", size = 3977768, upload-time = "2025-09-16T15:34:01.25Z" }, + { url = "https://files.pythonhosted.org/packages/f1/61/66f8dc98bbf5613bbfe5bf21747b4c8091442977f4bd897945895ab7325c/obstore-0.8.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1431e40e9bb4773a261e51b192ea6489d0799b9d4d7dbdf175cdf813eb8c0503", size = 3623364, upload-time = "2025-09-16T15:34:02.957Z" }, + { url = "https://files.pythonhosted.org/packages/1a/66/6d527b3027e42f625c8fc816ac7d19b0d6228f95bfe7666e4d6b081d2348/obstore-0.8.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ddb39d4da303f50b959da000aa42734f6da7ac0cc0be2d5a7838b62c97055bb9", size = 3347764, upload-time = "2025-09-16T15:34:04.236Z" }, + { url = "https://files.pythonhosted.org/packages/0d/79/c00103302b620192ea447a948921ad3fed031ce3d19e989f038e1183f607/obstore-0.8.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e01f4e13783db453e17e005a4a3ceff09c41c262e44649ba169d253098c775e8", size = 3460981, upload-time = "2025-09-16T15:34:05.595Z" }, + { url = "https://files.pythonhosted.org/packages/3d/d9/bfe4ed4b1aebc45b56644dd5b943cf8e1673505cccb352e66878a457e807/obstore-0.8.2-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df0fc2d0bc17caff9b538564ddc26d7616f7e8b7c65b1a3c90b5048a8ad2e797", size = 3692711, upload-time = "2025-09-16T15:34:06.796Z" }, + { url = "https://files.pythonhosted.org/packages/13/47/cd6c2cbb18e1f40c77e7957a4a03d2d83f1859a2e876a408f1ece81cad4c/obstore-0.8.2-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e439d06c99a140348f046c9f598ee349cc2dcd9105c15540a4b231f9cc48bbae", size = 3958362, upload-time = "2025-09-16T15:34:08.277Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ea/5ee82bf23abd71c7d6a3f2d008197ae8f8f569d41314c26a8f75318245be/obstore-0.8.2-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e37d9046669fcc59522d0faf1d105fcbfd09c84cccaaa1e809227d8e030f32c", size = 3957082, upload-time = "2025-09-16T15:34:09.477Z" }, + { url = "https://files.pythonhosted.org/packages/cb/ee/46650405e50fdaa8d95f30375491f9c91fac9517980e8a28a4a6af66927f/obstore-0.8.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2646fdcc4bbe92dc2bb5bcdff15574da1211f5806c002b66d514cee2a23c7cb8", size = 3775539, upload-time = "2025-09-16T15:34:10.726Z" }, + { url = "https://files.pythonhosted.org/packages/35/d6/348a7ebebe2ca3d94dfc75344ea19675ae45472823e372c1852844078307/obstore-0.8.2-cp314-cp314-manylinux_2_24_aarch64.whl", hash = "sha256:e31a7d37675056d93dfc244605089dee67f5bba30f37c88436623c8c5ad9ba9d", size = 3535048, upload-time = "2025-09-16T15:34:12.076Z" }, + { url = "https://files.pythonhosted.org/packages/41/07/b7a16cc0da91a4b902d47880ad24016abfe7880c63f7cdafda45d89a2f91/obstore-0.8.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:656313dd8170dde0f0cd471433283337a63912e8e790a121f7cc7639c83e3816", size = 3699035, upload-time = "2025-09-16T15:34:13.331Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/3269a3a58347e0b019742d888612c4b765293c9c75efa44e144b1e884c0d/obstore-0.8.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329038c9645d6d1741e77fe1a53e28a14b1a5c1461cfe4086082ad39ebabf981", size = 3687307, upload-time = "2025-09-16T15:34:14.501Z" }, + { url = "https://files.pythonhosted.org/packages/01/f9/4fd4819ad6a49d2f462a45be453561f4caebded0dc40112deeffc34b89b1/obstore-0.8.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1e4df99b369790c97c752d126b286dc86484ea49bff5782843a265221406566f", size = 3776076, upload-time = "2025-09-16T15:34:16.207Z" }, + { url = "https://files.pythonhosted.org/packages/14/dd/7c4f958fa0b9fc4778fb3d232e38b37db8c6b260f641022fbba48b049d7e/obstore-0.8.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9e1c65c65e20cc990414a8a9af88209b1bbc0dd9521b5f6b0293c60e19439bb7", size = 3947445, upload-time = "2025-09-16T15:34:17.423Z" }, +] + [[package]] name = "openai" version = "2.45.0" @@ -1887,6 +2099,127 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/b0/2291689e3ec4723fbf5bbf3b54afcd7b160f9ddc98ca7aedfd0132af5677/openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777", size = 1629470, upload-time = "2026-07-09T18:02:42.21Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/cc/e4c9584181f86494df0f6bdec1a4f3280c50db44704dc2a407e994fc87bb/opentelemetry_api-1.43.0.tar.gz", hash = "sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1", size = 73476, upload-time = "2026-06-24T15:19:55.323Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/83/6dba32b85f31868400440dc7ad2ca1eab94cbbf3a7b0459ed39f8311a9e2/opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd", size = 61912, upload-time = "2026-06-24T15:19:35.434Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/c1/e8098490ab15abf116dcaf9fa89ededcb35547c7d08d4b5a62f573dc1e63/opentelemetry_exporter_otlp_proto_common-1.43.0.tar.gz", hash = "sha256:c4e32ba6d6b13bdb2b8f6764c4fd28d00192826561aa04f6d14eedfce7ac076f", size = 20197, upload-time = "2026-06-24T15:20:00.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/b2/41ebc74ae1d5859901f1b69305de58724bf043381103d6ef413521cbc35a/opentelemetry_exporter_otlp_proto_common-1.43.0-py3-none-any.whl", hash = "sha256:123c3f9cc87218562490c63b36f497bf3a722faf174a515d1443f31ababa6264", size = 17048, upload-time = "2026-06-24T15:19:41.264Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/92/0b9f56412483a8891d4843890294796c9df8ab42417bd9bad8035d840cb3/opentelemetry_exporter_otlp_proto_http-1.43.0.tar.gz", hash = "sha256:fa8a42bb7d00ee5391f4c0b04d8e6a46c03caa437903296ab73a81dc11ba118f", size = 25406, upload-time = "2026-06-24T15:20:01.515Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/20/b685ed7af2e17c29ffc8af56f1fa8bc2033258fc30fb0d2b722f49d13ba0/opentelemetry_exporter_otlp_proto_http-1.43.0-py3-none-any.whl", hash = "sha256:647f603aa8efdbdb4dbff842e0729d0406a6fff26b295a72d3d60e7d963b2610", size = 21795, upload-time = "2026-06-24T15:19:43.164Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.64b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/97/02fe6e1c8b1ffac42d0b429c18080edb24e0e0d18c86612edf72b5752382/opentelemetry_instrumentation-0.64b0.tar.gz", hash = "sha256:b47d528dead6271d7743114417eb67fc915bd9258111c48dbf9a4951d2efa88d", size = 41935, upload-time = "2026-06-24T15:19:12.951Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/0c/cb9fe342de5299c7af24582eb7d788661cc53a1c4b904da92309caaa9417/opentelemetry_instrumentation-0.64b0-py3-none-any.whl", hash = "sha256:133ab7ffca796557aec059bf6be3190a34b6dea987f25be3d9409e230cbdad8b", size = 35880, upload-time = "2026-06-24T15:18:17.277Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-aiohttp-client" +version = "0.64b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/7c/1124a723ea77d866fae5cb1df78bea6133a06d93c132a8bd6d6bf08424cc/opentelemetry_instrumentation_aiohttp_client-0.64b0.tar.gz", hash = "sha256:ff03428052766c604e59ab802279d265841dd0d59eb78662da0a2878c48ac840", size = 19041, upload-time = "2026-06-24T15:19:14.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/55/87e9d0a3b9fc9fbecf044a67547b42d5423a7a3bb502113495b9a95a5e71/opentelemetry_instrumentation_aiohttp_client-0.64b0-py3-none-any.whl", hash = "sha256:661b1420b0012e43a92dca042f43bb8abafb80a952184ef256128fd9c12bb7ee", size = 13675, upload-time = "2026-06-24T15:18:19.296Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/b9/d357faefb40bda1d4799913e6af611171ff22a2dedcb93576bc92242d056/opentelemetry_proto-1.43.0.tar.gz", hash = "sha256:224778df17e1f3fafeaaa21d874236ca5f6ffc2f86e0899298ec7351aac27924", size = 46481, upload-time = "2026-06-24T15:20:07.625Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/a7/3e5308cf548b8f72529c7db1afdb3a404211982376a12927fd7759f77bf3/opentelemetry_proto-1.43.0-py3-none-any.whl", hash = "sha256:c58f1f7ef84bc7dc2834016c0c37fe0081dde7ca9f6339be1970fbf9cdaaa90d", size = 72489, upload-time = "2026-06-24T15:19:51.164Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/eb/5041074274ac0956b03637cc039d434569112468e875eddfcc9a0674ce06/opentelemetry_sdk-1.43.0.tar.gz", hash = "sha256:d8187c81c162df9913e4003dd6485f7390d9a24fc17026ec7387b8b8218b08e9", size = 254744, upload-time = "2026-06-24T15:20:08.467Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/e3/b17be23af124201c9f52eececd4cc8ddfed1597d37b4ee771895d325805c/opentelemetry_sdk-1.43.0-py3-none-any.whl", hash = "sha256:d1323a547c1ce69d6a069a17a44b7da82bb8b332051ecb074041f87642c86823", size = 178852, upload-time = "2026-06-24T15:19:52.169Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.64b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/30/5f26df29509eccd86b99b481ac9ffa39da49ba9577cc69071c552ae30447/opentelemetry_semantic_conventions-0.64b0.tar.gz", hash = "sha256:72f76fb2d1582d9d033dd1fcd84532e961e6ff3d90d24ba6fabc72975a83864c", size = 148340, upload-time = "2026-06-24T15:20:09.267Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/ca/23ba87a221b574a7c5a99d48849d80bfe8b047624681357e2b002e566187/opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6", size = 203713, upload-time = "2026-06-24T15:19:53.339Z" }, +] + +[[package]] +name = "opentelemetry-util-http" +version = "0.64b0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/1b/1029a805fd7242f7dfce91633b244c3b14a94d703232878f71e01ce862b1/opentelemetry_util_http-0.64b0.tar.gz", hash = "sha256:8a86a220dbfc56d736f47f1e5c4e7932a21fcf69052312e1bcf166444dc79322", size = 11102, upload-time = "2026-06-24T15:19:48.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/c7/5f8ec5b30546f2dc22cd5fc5759bce2ab5be6e89a2e710a405ac9ef64ed3/opentelemetry_util_http-0.64b0-py3-none-any.whl", hash = "sha256:c1e5350d25507c1afcd6076cf9ac062485a0a4f79cd9971366996fd3056bacdb", size = 8204, upload-time = "2026-06-24T15:19:09.02Z" }, +] + [[package]] name = "orjson" version = "3.11.9" @@ -2151,6 +2484,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "protobuf" +version = "7.35.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, + { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, + { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -2839,6 +3187,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "tiktoken" version = "0.13.0" @@ -2913,6 +3270,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomli-w" version = "1.2.0" @@ -3076,6 +3442,82 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] +[[package]] +name = "wrapt" +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" }, + { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, + { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, + { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, + { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" }, + { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" }, + { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" }, + { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" }, + { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, + { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, + { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" }, + { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" }, + { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" }, + { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, + { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, + { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, + { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" }, + { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" }, + { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" }, + { url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341, upload-time = "2026-06-20T23:48:49.39Z" }, + { url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921, upload-time = "2026-06-20T23:48:50.986Z" }, + { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" }, + { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" }, + { url = "https://files.pythonhosted.org/packages/99/eb/974e471a6a978b8180186b8a9dc5ae3361ce269a967190b709b8ce17abfb/wrapt-2.2.2-cp314-cp314-win32.whl", hash = "sha256:58f9f8d637c9a6e245c6ef5b109b67ec187d2faed23d1405656b51d96e0a5b56", size = 78062, upload-time = "2026-06-20T23:49:02.327Z" }, + { url = "https://files.pythonhosted.org/packages/49/ec/e1281156cdc7a66693838ad7a0865ad641c74abd337a957d668b575aaffb/wrapt-2.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:385cb1866f20479e83299af585375bfa0a4b0c6c9907a981483ea782ea8ae406", size = 80832, upload-time = "2026-06-20T23:49:03.837Z" }, + { url = "https://files.pythonhosted.org/packages/45/7d/1b6b5ddd94005a2dac97a4490c9838f3154977850d633abcb65b30089437/wrapt-2.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:8ffbeaea6771a6eba6e6eeb09767864995726bc8240bb54baf88a9bb1db34d5c", size = 80029, upload-time = "2026-06-20T23:49:05.237Z" }, + { url = "https://files.pythonhosted.org/packages/b0/33/9ebcf8aafe91c601127cbd93708c16aa8f688f34a10bf004046803ecdc4f/wrapt-2.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09f811d43f6f33ec7515f0be76b159569f4057ab54d3e079c3204dddb90afa2a", size = 83357, upload-time = "2026-06-20T23:49:06.632Z" }, + { url = "https://files.pythonhosted.org/packages/39/38/ec45b635153327b52e52732a0ea980e5f00b7efba65f9e018828f1e69daa/wrapt-2.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a795d3c06e5fbf9ea2f13196180b77aeab1b4685917256ee0d014cc163d90063", size = 83794, upload-time = "2026-06-20T23:49:08.098Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" }, + { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" }, + { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" }, + { url = "https://files.pythonhosted.org/packages/67/7c/e88313f16a99930b899ef970d91c281544a470749a359decad994483bbda/wrapt-2.2.2-cp314-cp314t-win32.whl", hash = "sha256:d8a15813215f33fa83667bfc978b300e35669ea8bb424e970a1426bcb7bc6cca", size = 79059, upload-time = "2026-06-20T23:49:20.107Z" }, + { url = "https://files.pythonhosted.org/packages/a0/4f/ac12fda57a55068a094ec42851fb0a40e8489d8941863d517452de62e507/wrapt-2.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d09db0f7e8357060d3c38fc22a018aba683a796bf184360fd1a58f6fc180dc77", size = 82462, upload-time = "2026-06-20T23:49:21.631Z" }, + { url = "https://files.pythonhosted.org/packages/48/a7/df732dac86d9b2027c56bd163dbc883e037b16c3469614752e148d219c61/wrapt-2.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f32fe639c39561ccc187bcae17e9271be0eb45f1c2952510d2f29b33ab577347", size = 81182, upload-time = "2026-06-20T23:49:23.199Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, +] + +[[package]] +name = "wsproto" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, +] + [[package]] name = "yarl" version = "1.24.2"