diff --git a/README.md b/README.md index c91c22e..af407ab 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,8 @@ All published article-suite rows run in Daytona sandboxes. See [`docs/article-suite-scoring.md`](docs/article-suite-scoring.md) for the formula, research precedent, and statistical limitations. +The selected-trajectory timeout audit and repaired cells are documented in +[`docs/article-suite-timeout-fairness.md`](docs/article-suite-timeout-fairness.md). ## Contribute a Task diff --git a/docs/article-suite-scoring.md b/docs/article-suite-scoring.md index a5054fb..d6b985a 100644 --- a/docs/article-suite-scoring.md +++ b/docs/article-suite-scoring.md @@ -2,6 +2,11 @@ GenesisBench separates per-task normalization from cross-task aggregation. +All compared models use identical task wall-clock, verifier, build, +60-minute agent-idle, and 60-minute Daytona PTY-read limits. Infrastructure +timeouts are repaired at the affected model-task-trial cell rather than scored +as model performance. + ## Per-task normalized score Every task evaluates a candidate on its hidden suite and compares the resulting diff --git a/docs/article-suite-timeout-fairness.md b/docs/article-suite-timeout-fairness.md new file mode 100644 index 0000000..ab2db67 --- /dev/null +++ b/docs/article-suite-timeout-fairness.md @@ -0,0 +1,52 @@ +# Article-suite timeout fairness audit + +The published five-trial leaderboard was audited cell by cell against the +selected source trajectories for all `4 × 9 × 5 = 180` model-task-trial cells. + +## Fairness policy + +- Task wall-clock, verifier, build, hidden-suite, and task-digest settings are + identical between models within each task. +- Protocol v2.2 sets both the agent-idle and Daytona PTY-read safeguards to + `3,600` seconds for every model. +- Infrastructure timeout attempts are not scored as model performance. Only + affected model-task-trial cells are rerun. + +## Audit and repairs + +The historical audit found 10 cells whose result or retry path was affected by +the earlier model-specific safeguards: + +- 8 cells had a `900`-second GPT Daytona PTY timeout in their retry history. +- 2 selected GPT cells ended at the earlier `600`-second idle watchdog. + +All 10 cells were rerun with protocol v2.2. The final audit reports: + +```text +selected cells: 180 +selected idle/PTY-influenced cells: 0 +historically affected cells rerun: 10 +``` + +Two Claude Ant cells reached the shared task wall-clock limit of `5,400` +seconds. They remain valid because the same primary limit applied to every +model; they were not caused by the earlier idle/PTY difference. + +## Leaderboard impact + +| Model | Previous IQM | Fair IQM | Change | +| --- | ---: | ---: | ---: | +| GPT-5.5 | `10.66` | `13.87` | `+3.21` | +| GPT-5.6 Sol | `36.65` | `40.68` | `+4.03` | +| Claude Opus 4.8 | `62.03` | `62.03` | `0.00` | +| GPT-5.4 Mini | `-17.41` | `-17.41` | `0.00` | + +The ranking order did not change. + +The machine-readable audit is +[`leaderboard/article_suite_timeout_fairness_audit.json`](../leaderboard/article_suite_timeout_fairness_audit.json). +Reproduce it with: + +```bash +uv run python scripts/audit_article_suite_timeout_fairness.py +``` diff --git a/experiments/article_suite/README.md b/experiments/article_suite/README.md index b1d6ee5..a164444 100644 --- a/experiments/article_suite/README.md +++ b/experiments/article_suite/README.md @@ -27,6 +27,8 @@ The current leaderboard protocol is defined in `protocol.toml`: normalized trial-task scores; - variability shown as the sample standard deviation of the five per-trial nine-task IQMs. +- identical 60-minute agent-idle and Daytona PTY-read safeguards for every + model, in addition to the task-specific wall-clock limits. The resulting matrix is: @@ -126,6 +128,11 @@ The leaderboard builder selects the latest complete per-model batch matching `protocol.toml`; it never mixes older single-run results or partial batches into the five-trial leaderboard. +The selected 180 cells and timeout safeguards are audited by +`scripts/audit_article_suite_timeout_fairness.py`. See +`docs/article-suite-timeout-fairness.md` for the 10 repaired cells and the +leaderboard impact. + GPT-5.6 Sol is routed directly through OpenCode's Azure Responses-API provider with reasoning effort `max`. BenchFlow still owns sandboxing, task staging, ACP trajectory capture, and verifier execution. Trusted LiteLLM usage tracking diff --git a/experiments/article_suite/models.toml b/experiments/article_suite/models.toml index 6b3a7c5..44b7b43 100644 --- a/experiments/article_suite/models.toml +++ b/experiments/article_suite/models.toml @@ -6,6 +6,8 @@ display_name = "GPT-5.6 Sol" model = "azure/gpt-5.6-sol" provider = "azure" provider_reasoning_effort = "max" +agent_idle_timeout_sec = 3600 +daytona_pty_readline_timeout_sec = 3600 [[models]] id = "gpt-5.5" @@ -13,6 +15,8 @@ display_name = "GPT-5.5" model = "azure/gpt-5.5" provider = "azure" provider_reasoning_effort = "xhigh" +agent_idle_timeout_sec = 3600 +daytona_pty_readline_timeout_sec = 3600 [[models]] id = "claude-opus-4.8" @@ -29,3 +33,5 @@ display_name = "GPT-5.4 Mini" model = "azure/gpt-5.4-mini" provider = "azure" provider_reasoning_effort = "xhigh" +agent_idle_timeout_sec = 3600 +daytona_pty_readline_timeout_sec = 3600 diff --git a/experiments/article_suite/protocol.toml b/experiments/article_suite/protocol.toml index cbb5a6f..5c066ac 100644 --- a/experiments/article_suite/protocol.toml +++ b/experiments/article_suite/protocol.toml @@ -1,7 +1,11 @@ -version = "2.1" +version = "2.2" trials = 5 agent_timeout_multiplier = 3 +[fairness] +agent_idle_timeout_sec = 3600 +daytona_pty_readline_timeout_sec = 3600 + [baseline_agent_timeout_sec] simulation_heuristics_ant_v1 = 1800 simulation_heuristics_pong_ram_v1 = 1800 diff --git a/leaderboard/ARTICLE_SUITE.md b/leaderboard/ARTICLE_SUITE.md index ac334bb..28c402f 100644 --- a/leaderboard/ARTICLE_SUITE.md +++ b/leaderboard/ARTICLE_SUITE.md @@ -15,3 +15,5 @@ The final score is RLiable-style IQM over the complete 5 × 9 trial-task score m ![Final GenesisBench article-suite leaderboard](article_suite_final_leaderboard.png) Machine-readable rankings and score-detail paths are available in [`article_suite.json`](article_suite.json). The scoring rationale is documented in [`docs/article-suite-scoring.md`](../docs/article-suite-scoring.md). + +The selected trajectory timeout audit is available in [`article_suite_timeout_fairness_audit.json`](article_suite_timeout_fairness_audit.json) and [`docs/article-suite-timeout-fairness.md`](../docs/article-suite-timeout-fairness.md). diff --git a/leaderboard/article_suite.json b/leaderboard/article_suite.json index 37b8aa1..a358ed9 100644 --- a/leaderboard/article_suite.json +++ b/leaderboard/article_suite.json @@ -39,7 +39,7 @@ "daytona" ] }, - "generated_at": "2026-07-15T12:56:18.690710+00:00", + "generated_at": "2026-07-15T19:02:46.670420+00:00", "inference_settings": { "cross_provider_comparability": "labels_are_not_a_shared_numeric_compute_scale", "field": "provider_reasoning_effort", @@ -130,20 +130,20 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": -4.9009087386, - "normalized_score_stddev": 11.991465810903719, + "normalized_score": -6.081625694, + "normalized_score_stddev": 12.597925216329989, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 2, - "raw_score": 2272.356693724495, + "raw_score": 2231.0491055302227, "raw_score_imputation_count": 0, - "raw_score_stddev": 419.52351859557746, + "raw_score_stddev": 440.7406064520731, "reference_score": 5942.333244755838, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" @@ -157,22 +157,22 @@ "model": "GPT-5.5", "model_id": "gpt-5.5", "model_route": "azure/gpt-5.5", - "normalized_score": -12.5116242462, - "normalized_score_stddev": 8.756962566015595, + "normalized_score": -12.2320084794, + "normalized_score_stddev": 9.211087046248206, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "xhigh", "rank": 3, - "raw_score": 2006.0944869926293, + "raw_score": 2015.8768932664839, "raw_score_imputation_count": 0, - "raw_score_stddev": 306.36385957656836, + "raw_score_stddev": 322.25148356656894, "reference_score": 5942.333244755838, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4" ], "starter_score": 2443.815838992282, @@ -276,22 +276,22 @@ "model": "GPT-5.5", "model_id": "gpt-5.5", "model_route": "azure/gpt-5.5", - "normalized_score": 49.1666666668, - "normalized_score_stddev": 40.44028519521613, + "normalized_score": 54.166666666800005, + "normalized_score_stddev": 38.188130791325946, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "xhigh", "rank": 3, - "raw_score": 8.8, + "raw_score": 10.0, "raw_score_imputation_count": 0, - "raw_score_stddev": 9.705668446840743, + "raw_score_stddev": 9.16515138991168, "reference_score": 21.0, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "starter_score": -2.9999999999999996, @@ -368,22 +368,22 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": 43.899371069400004, - "normalized_score_stddev": 47.34804435501101, + "normalized_score": 60.209643606, + "normalized_score_stddev": 50.34075134460858, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 2, - "raw_score": 596.4, + "raw_score": 674.2, "raw_score_imputation_count": 0, - "raw_score_stddev": 225.8501715739884, + "raw_score_stddev": 240.1253839143209, "reference_score": 864.0, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "starter_score": 387.0, @@ -487,19 +487,19 @@ "model": "GPT-5.5", "model_id": "gpt-5.5", "model_route": "azure/gpt-5.5", - "normalized_score": 60.86642599300001, - "normalized_score_stddev": 22.118506348049813, + "normalized_score": 76.462093863, + "normalized_score_stddev": 12.754467163779541, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "xhigh", "rank": 2, - "raw_score": 647.2, + "raw_score": 733.6, "raw_score_imputation_count": 0, - "raw_score_stddev": 122.53652516698847, + "raw_score_stddev": 70.65974808899335, "reference_score": 864.0, "sandbox": "daytona", "source_run": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", @@ -519,7 +519,7 @@ "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", - "rank": 2, + "rank": 3, "raw_score": 647.2, "raw_score_imputation_count": 0, "raw_score_stddev": 122.53652516698847, @@ -606,19 +606,19 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": 31.8442100886, - "normalized_score_stddev": 3.8873875677992533, + "normalized_score": 28.1034211084, + "normalized_score_stddev": 11.772801175021295, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 2, - "raw_score": 6385.135940620764, + "raw_score": 6105.10168077257, "raw_score_imputation_count": 0, - "raw_score_stddev": 291.0085829715705, + "raw_score_stddev": 881.3081092982512, "reference_score": 11487.256451471967, "sandbox": "daytona", "source_run": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", @@ -725,21 +725,21 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": 56.834663982799995, - "normalized_score_stddev": 33.111283305681766, + "normalized_score": 73.23338822320001, + "normalized_score_stddev": 37.18119323399795, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 2, - "raw_score": 0.6554319871515035, + "raw_score": 0.7171239853203296, "raw_score_imputation_count": 0, - "raw_score_stddev": 0.12456464290218099, + "raw_score_stddev": 0.13987564345039064, "reference_score": 0.8178199748694898, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], @@ -898,21 +898,21 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": 16.3034759358, - "normalized_score_stddev": 23.995771403261536, + "normalized_score": 13.850267379600002, + "normalized_score_stddev": 20.096747543003065, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 4, - "raw_score": 80.67999999999999, + "raw_score": 73.34, "raw_score_imputation_count": 0, - "raw_score_stddev": 71.79534803871347, + "raw_score_stddev": 60.12946864890792, "reference_score": 331.1, "sandbox": "daytona", "source_run": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], @@ -1151,7 +1151,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "starter_score": 0.0, @@ -1192,17 +1192,17 @@ "trial_count": 5 }, { - "arithmetic_mean_normalized_score": 41.223767221955555, - "average_normalized_score": 41.223767221955555, - "final_normalized_score": 36.651764607913044, - "final_normalized_score_stddev": 5.524217450335815, + "arithmetic_mean_normalized_score": 44.038687475866666, + "average_normalized_score": 44.038687475866666, + "final_normalized_score": 40.67930709160869, + "final_normalized_score_stddev": 11.031024092730283, "harness": "opencode", - "mean_trial_iqm_normalized_score": 37.73164397808, - "median_normalized_score": 32.792734613, + "mean_trial_iqm_normalized_score": 41.6809489228, + "median_normalized_score": 33.28729024, "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "positive_display_score": 136.65176460791304, + "positive_display_score": 140.67930709160868, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", @@ -1211,17 +1211,17 @@ "trial_count": 5 }, { - "arithmetic_mean_normalized_score": 22.06816884377778, - "average_normalized_score": 22.06816884377778, - "final_normalized_score": 10.662970787652174, - "final_normalized_score_stddev": 18.67984640949238, + "arithmetic_mean_normalized_score": 24.387644803422226, + "average_normalized_score": 24.387644803422226, + "final_normalized_score": 13.868245803826087, + "final_normalized_score_stddev": 18.296053164135497, "harness": "opencode", - "mean_trial_iqm_normalized_score": 16.7775760408, + "mean_trial_iqm_normalized_score": 17.8186386096, "median_normalized_score": 5.870020964, "model": "GPT-5.5", "model_id": "gpt-5.5", "model_route": "azure/gpt-5.5", - "positive_display_score": 110.66297078765217, + "positive_display_score": 113.86824580382608, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "xhigh", @@ -1271,8 +1271,12 @@ "simulation_heuristics_vizdoom_d1_v1": 1800, "simulation_heuristics_vizdoom_d3_v1": 1800 }, + "fairness": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600 + }, "trials": 5, - "version": "2.1" + "version": "2.2" }, "rows": [ { @@ -1527,13 +1531,13 @@ } }, { - "arithmetic_mean_normalized_score": 41.223767221955555, - "average_normalized_score": 41.223767221955555, - "final_normalized_score": 36.651764607913044, - "final_normalized_score_stddev": 5.524217450335815, + "arithmetic_mean_normalized_score": 44.038687475866666, + "average_normalized_score": 44.038687475866666, + "final_normalized_score": 40.67930709160869, + "final_normalized_score_stddev": 11.031024092730283, "harness": "opencode", - "mean_trial_iqm_normalized_score": 37.73164397808, - "median_normalized_score": 32.792734613, + "mean_trial_iqm_normalized_score": 41.6809489228, + "median_normalized_score": 33.28729024, "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", @@ -1553,32 +1557,32 @@ "simulation_heuristics_vizdoom_d3_v1": 0 }, "raw_task_score_stddevs": { - "simulation_heuristics_ant_v1": 419.52351859557746, + "simulation_heuristics_ant_v1": 440.7406064520731, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 225.8501715739884, + "simulation_heuristics_breakout_ram_v1": 240.1253839143209, "simulation_heuristics_breakout_rgb_v1": 122.53652516698847, - "simulation_heuristics_halfcheetah_v1": 291.0085829715705, + "simulation_heuristics_halfcheetah_v1": 881.3081092982512, "simulation_heuristics_montezuma_v1": 173.26280616450836, "simulation_heuristics_pong_ram_v1": 2.345207879911715, - "simulation_heuristics_vizdoom_d1_v1": 0.12456464290218099, - "simulation_heuristics_vizdoom_d3_v1": 71.79534803871347 + "simulation_heuristics_vizdoom_d1_v1": 0.13987564345039064, + "simulation_heuristics_vizdoom_d3_v1": 60.12946864890792 }, "raw_task_scores": { - "simulation_heuristics_ant_v1": 2272.356693724495, + "simulation_heuristics_ant_v1": 2231.0491055302227, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 596.4, + "simulation_heuristics_breakout_ram_v1": 674.2, "simulation_heuristics_breakout_rgb_v1": 647.2, - "simulation_heuristics_halfcheetah_v1": 6385.135940620764, + "simulation_heuristics_halfcheetah_v1": 6105.10168077257, "simulation_heuristics_montezuma_v1": 298.0, "simulation_heuristics_pong_ram_v1": 19.0, - "simulation_heuristics_vizdoom_d1_v1": 0.6554319871515035, - "simulation_heuristics_vizdoom_d3_v1": 80.67999999999999 + "simulation_heuristics_vizdoom_d1_v1": 0.7171239853203296, + "simulation_heuristics_vizdoom_d3_v1": 73.34 }, "sandbox": "daytona", "source_runs": { "simulation_heuristics_ant_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" @@ -1594,7 +1598,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_breakout_rgb_v1": [ @@ -1605,7 +1609,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_halfcheetah_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", @@ -1628,14 +1632,14 @@ "simulation_heuristics_vizdoom_d1_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_vizdoom_d3_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ] @@ -1690,33 +1694,33 @@ } }, "task_score_stddevs": { - "simulation_heuristics_ant_v1": 11.991465810903719, + "simulation_heuristics_ant_v1": 12.597925216329989, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 47.34804435501101, + "simulation_heuristics_breakout_ram_v1": 50.34075134460858, "simulation_heuristics_breakout_rgb_v1": 22.118506348049813, - "simulation_heuristics_halfcheetah_v1": 3.8873875677992533, + "simulation_heuristics_halfcheetah_v1": 11.772801175021295, "simulation_heuristics_montezuma_v1": 43.31570154112709, "simulation_heuristics_pong_ram_v1": 9.771699499561077, - "simulation_heuristics_vizdoom_d1_v1": 33.111283305681766, - "simulation_heuristics_vizdoom_d3_v1": 23.995771403261536 + "simulation_heuristics_vizdoom_d1_v1": 37.18119323399795, + "simulation_heuristics_vizdoom_d3_v1": 20.096747543003065 }, "task_scores": { - "simulation_heuristics_ant_v1": -4.9009087386, + "simulation_heuristics_ant_v1": -6.081625694, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 43.899371069400004, + "simulation_heuristics_breakout_ram_v1": 60.209643606, "simulation_heuristics_breakout_rgb_v1": 60.86642599300001, - "simulation_heuristics_halfcheetah_v1": 31.8442100886, + "simulation_heuristics_halfcheetah_v1": 28.1034211084, "simulation_heuristics_montezuma_v1": 74.5, "simulation_heuristics_pong_ram_v1": 91.6666666666, - "simulation_heuristics_vizdoom_d1_v1": 56.834663982799995, - "simulation_heuristics_vizdoom_d3_v1": 16.3034759358 + "simulation_heuristics_vizdoom_d1_v1": 73.23338822320001, + "simulation_heuristics_vizdoom_d3_v1": 13.850267379600002 }, "trial_count": 5, "trial_final_normalized_scores": { - "1": 37.797729594399996, + "1": 27.0114860686, "2": 41.203895079, - "3": 28.388586371000002, - "4": 42.346736302800004, + "3": 43.444415417200005, + "4": 57.82367550600001, "5": 38.921272543200004 }, "trial_task_normalized_scores": { @@ -1725,14 +1729,14 @@ "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 100.0, "simulation_heuristics_breakout_rgb_v1": 70.758122744, - "simulation_heuristics_halfcheetah_v1": 26.12140084, + "simulation_heuristics_halfcheetah_v1": 7.417455939, "simulation_heuristics_montezuma_v1": 0.0, "simulation_heuristics_pong_ram_v1": 79.166666667, "simulation_heuristics_vizdoom_d1_v1": 42.828284045, - "simulation_heuristics_vizdoom_d3_v1": 40.106951872 + "simulation_heuristics_vizdoom_d3_v1": 4.879679144 }, "2": { - "simulation_heuristics_ant_v1": -7.591479, + "simulation_heuristics_ant_v1": -13.495063777, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 6.918238994, "simulation_heuristics_breakout_rgb_v1": 70.758122744, @@ -1750,13 +1754,13 @@ "simulation_heuristics_halfcheetah_v1": 36.605449762, "simulation_heuristics_montezuma_v1": 100.0, "simulation_heuristics_pong_ram_v1": 83.333333333, - "simulation_heuristics_vizdoom_d1_v1": 31.015418049, - "simulation_heuristics_vizdoom_d3_v1": 0.0 + "simulation_heuristics_vizdoom_d1_v1": 113.009039251, + "simulation_heuristics_vizdoom_d3_v1": 22.961229947 }, "4": { "simulation_heuristics_ant_v1": -19.696444908, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 18.448637317, + "simulation_heuristics_breakout_ram_v1": 100.0, "simulation_heuristics_breakout_rgb_v1": 70.758122744, "simulation_heuristics_halfcheetah_v1": 32.792734613, "simulation_heuristics_montezuma_v1": 100.0, @@ -1778,12 +1782,12 @@ } }, { - "arithmetic_mean_normalized_score": 22.06816884377778, - "average_normalized_score": 22.06816884377778, - "final_normalized_score": 10.662970787652174, - "final_normalized_score_stddev": 18.67984640949238, + "arithmetic_mean_normalized_score": 24.387644803422226, + "average_normalized_score": 24.387644803422226, + "final_normalized_score": 13.868245803826087, + "final_normalized_score_stddev": 18.296053164135497, "harness": "opencode", - "mean_trial_iqm_normalized_score": 16.7775760408, + "mean_trial_iqm_normalized_score": 17.8186386096, "median_normalized_score": 5.870020964, "model": "GPT-5.5", "model_id": "gpt-5.5", @@ -1804,24 +1808,24 @@ "simulation_heuristics_vizdoom_d3_v1": 0 }, "raw_task_score_stddevs": { - "simulation_heuristics_ant_v1": 306.36385957656836, + "simulation_heuristics_ant_v1": 322.25148356656894, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 131.153345363357, - "simulation_heuristics_breakout_rgb_v1": 122.53652516698847, + "simulation_heuristics_breakout_rgb_v1": 70.65974808899335, "simulation_heuristics_halfcheetah_v1": 698.5070322948952, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 9.705668446840743, + "simulation_heuristics_pong_ram_v1": 9.16515138991168, "simulation_heuristics_vizdoom_d1_v1": 0.3579053781713541, "simulation_heuristics_vizdoom_d3_v1": 122.81457568220475 }, "raw_task_scores": { - "simulation_heuristics_ant_v1": 2006.0944869926293, + "simulation_heuristics_ant_v1": 2015.8768932664839, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 486.8, - "simulation_heuristics_breakout_rgb_v1": 647.2, + "simulation_heuristics_breakout_rgb_v1": 733.6, "simulation_heuristics_halfcheetah_v1": 4717.6162449605235, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 8.8, + "simulation_heuristics_pong_ram_v1": 10.0, "simulation_heuristics_vizdoom_d1_v1": 0.6197079806402326, "simulation_heuristics_vizdoom_d3_v1": 101.49999999999999 }, @@ -1831,7 +1835,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4" ], "simulation_heuristics_atari57_v1": [ @@ -1849,7 +1853,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_breakout_rgb_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", @@ -1866,14 +1870,14 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_pong_ram_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_vizdoom_d1_v1": [ @@ -1941,33 +1945,33 @@ } }, "task_score_stddevs": { - "simulation_heuristics_ant_v1": 8.756962566015595, + "simulation_heuristics_ant_v1": 9.211087046248206, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 27.49546024414613, - "simulation_heuristics_breakout_rgb_v1": 22.118506348049813, + "simulation_heuristics_breakout_rgb_v1": 12.754467163779541, "simulation_heuristics_halfcheetah_v1": 9.330884765209642, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 40.44028519521613, + "simulation_heuristics_pong_ram_v1": 38.188130791325946, "simulation_heuristics_vizdoom_d1_v1": 95.13699952993868, "simulation_heuristics_vizdoom_d3_v1": 41.04765230004701 }, "task_scores": { - "simulation_heuristics_ant_v1": -12.5116242462, + "simulation_heuristics_ant_v1": -12.2320084794, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 20.9224318658, - "simulation_heuristics_breakout_rgb_v1": 60.86642599300001, + "simulation_heuristics_breakout_rgb_v1": 76.462093863, "simulation_heuristics_halfcheetah_v1": 9.5689381094, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 49.1666666668, + "simulation_heuristics_pong_ram_v1": 54.166666666800005, "simulation_heuristics_vizdoom_d1_v1": 47.33864911959999, "simulation_heuristics_vizdoom_d3_v1": 23.2620320856 }, "trial_count": 5, "trial_final_normalized_scores": { - "1": 5.4339319906, + "1": 5.6392448345999995, "2": 48.2436373978, "3": 18.718893057000002, - "4": 9.604625305799999, + "4": 14.604625305799999, "5": 1.8867924528 }, "trial_task_normalized_scores": { @@ -1975,7 +1979,7 @@ "simulation_heuristics_ant_v1": -6.720960755, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 5.870020964, - "simulation_heuristics_breakout_rgb_v1": 21.299638989, + "simulation_heuristics_breakout_rgb_v1": 99.277978339, "simulation_heuristics_halfcheetah_v1": 0.0, "simulation_heuristics_montezuma_v1": 0.0, "simulation_heuristics_pong_ram_v1": 41.666666667, @@ -2005,13 +2009,13 @@ "simulation_heuristics_vizdoom_d3_v1": -0.601604278 }, "4": { - "simulation_heuristics_ant_v1": -1.398078834, + "simulation_heuristics_ant_v1": 0.0, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 9.224318658, "simulation_heuristics_breakout_rgb_v1": 70.758122744, "simulation_heuristics_halfcheetah_v1": 13.798807871, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 25.0, + "simulation_heuristics_pong_ram_v1": 50.0, "simulation_heuristics_vizdoom_d1_v1": 111.839447567, "simulation_heuristics_vizdoom_d3_v1": 0.0 }, @@ -2416,7 +2420,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4" ], "simulation_heuristics_atari57_v1": [ @@ -2434,7 +2438,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_breakout_rgb_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", @@ -2451,14 +2455,14 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_pong_ram_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_vizdoom_d1_v1": [ @@ -2479,7 +2483,7 @@ "gpt-5.6-sol": { "simulation_heuristics_ant_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" @@ -2495,7 +2499,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_breakout_rgb_v1": [ @@ -2506,7 +2510,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_halfcheetah_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", @@ -2529,14 +2533,14 @@ "simulation_heuristics_vizdoom_d1_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_vizdoom_d3_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ] diff --git a/leaderboard/article_suite_final_leaderboard.png b/leaderboard/article_suite_final_leaderboard.png index cb471e3..f122139 100644 Binary files a/leaderboard/article_suite_final_leaderboard.png and b/leaderboard/article_suite_final_leaderboard.png differ diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-01/metadata.json index f442b81..45b5cbf 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-01/metadata.json @@ -1,9 +1,21 @@ { + "agent_timeout_info": { + "n_tool_calls": 52, + "pending_tool_call_ids": [ + "toolu_012bacyDsfbB9vYNNfLEwViD" + ], + "reason": "wall_clock_timeout", + "terminal_event_recorded": true, + "terminal_trajectory_complete": false, + "timeout_sec": 5400.0 + }, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784105005.326783, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +27,27 @@ }, "normalized_score": 0.0, "observed_raw_score": null, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 2443.815838992282, "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01/tasks-81ec8d98a4", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": "Agent prompt exceeded wall-clock budget 5400s", + "terminal_error_category": "timeout", + "timing": { + "agent_execution": 5400.5, + "agent_setup": 8.3, + "environment_setup": 4.4, + "total": 9351.8, + "verifier": 3913.4 + }, "token_usage": { "final_input_tokens": 0.0, "final_output_tokens": 0.0, @@ -29,5 +56,7 @@ "total_tokens": 0.0 }, "tool_calls": 52, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-02/metadata.json index 18a11ef..bbc1c82 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-02/metadata.json @@ -1,9 +1,21 @@ { + "agent_timeout_info": { + "n_tool_calls": 75, + "pending_tool_call_ids": [ + "toolu_01QYC55dM1tSkifzHEshxQKU" + ], + "reason": "wall_clock_timeout", + "terminal_event_recorded": true, + "terminal_trajectory_complete": false, + "timeout_sec": 5400.0 + }, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784103481.173677, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +27,27 @@ }, "normalized_score": 9.101740845, "observed_raw_score": 2762.241826695058, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 2762.241826695058, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": "Agent prompt exceeded wall-clock budget 5400s", + "terminal_error_category": "timeout", + "timing": { + "agent_execution": 5401.6, + "agent_setup": 9.7, + "environment_setup": 13.6, + "total": 7826.9, + "verifier": 2360.7 + }, "token_usage": { "final_input_tokens": 0.0, "final_output_tokens": 0.0, @@ -29,5 +56,7 @@ "total_tokens": 0.0 }, "tool_calls": 75, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-03/metadata.json index 411ac21..b2bf236 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": null, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 2443.815838992282, "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 2700.5, + "agent_setup": 10.1, + "environment_setup": 3.4, + "total": 6652.4, + "verifier": 3912.8 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 639.0, @@ -29,5 +47,7 @@ "total_tokens": 47729.0 }, "tool_calls": 39, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-04/metadata.json index 92f84b1..9a3f66e 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": null, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 2443.815838992282, "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 2970.9, + "agent_setup": 10.8, + "environment_setup": 3.5, + "total": 6923.0, + "verifier": 3912.9 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 742.0, @@ -29,5 +47,7 @@ "total_tokens": 40033.0 }, "tool_calls": 26, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-05/metadata.json index 7d597aa..246777b 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_ant_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": null, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 2443.815838992282, "raw_score_source": "starter_anchor_equivalent_for_fail_closed_timeout", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 3510.4, + "agent_setup": 8.4, + "environment_setup": 4.5, + "total": 7459.1, + "verifier": 3911.6 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 565.0, @@ -29,5 +47,7 @@ "total_tokens": 45246.0 }, "tool_calls": 33, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-01/metadata.json index ea092b4..f3f5a84 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784091892.627182, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 0, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": null, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 384.0, + "agent_setup": 8.4, + "environment_setup": 3.2, + "total": 430.5, + "verifier": 11.0 + }, "token_usage": null, "tool_calls": 33, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-02/metadata.json index 3868622..3d9dbee 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1170.3, + "agent_setup": 7.8, + "environment_setup": 3.0, + "total": 1213.5, + "verifier": 10.4 + }, "token_usage": null, "tool_calls": 39, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-03/metadata.json index 4e03187..ad1db99 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 480.1, + "agent_setup": 8.3, + "environment_setup": 3.6, + "total": 528.6, + "verifier": 11.4 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 664.0, @@ -29,5 +47,7 @@ "total_tokens": 50513.0 }, "tool_calls": 29, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-04/metadata.json index c2a9574..6283d01 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 450.1, + "agent_setup": 8.5, + "environment_setup": 3.6, + "total": 497.9, + "verifier": 11.7 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 879.0, @@ -29,5 +47,7 @@ "total_tokens": 51441.0 }, "tool_calls": 34, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-05/metadata.json index 623771e..4941005 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_atari57_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 510.2, + "agent_setup": 8.5, + "environment_setup": 3.6, + "total": 556.7, + "verifier": 10.9 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 678.0, @@ -29,5 +47,7 @@ "total_tokens": 59275.0 }, "tool_calls": 33, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json index 750732f..6bd4ce2 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784091892.627182, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 0, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 98.742138365, "observed_raw_score": 858.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 858.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": null, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1921.6, + "agent_setup": 10.9, + "environment_setup": 2.3, + "total": 2089.4, + "verifier": 126.6 + }, "token_usage": null, "tool_calls": 77, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json index bf63b6a..8c219e6 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 100.0, "observed_raw_score": 864.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 864.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1170.3, + "agent_setup": 6.0, + "environment_setup": 2.7, + "total": 1319.4, + "verifier": 118.4 + }, "token_usage": null, "tool_calls": 64, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json index 95899f5..6742d7c 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 98.322851153, "observed_raw_score": 856.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 856.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1440.1, + "agent_setup": 7.6, + "environment_setup": 3.4, + "total": 1634.6, + "verifier": 159.9 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 874.0, @@ -29,5 +47,7 @@ "total_tokens": 70599.0 }, "tool_calls": 52, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json index a9af6dc..d400f84 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 84.696016771, "observed_raw_score": 791.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 791.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1260.4, + "agent_setup": 7.6, + "environment_setup": 3.3, + "total": 1408.9, + "verifier": 113.9 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 805.0, @@ -29,5 +47,7 @@ "total_tokens": 91245.0 }, "tool_calls": 77, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json index b4164e6..2b54401 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 88.259958071, "observed_raw_score": 808.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 808.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 930.1, + "agent_setup": 6.2, + "environment_setup": 2.9, + "total": 1053.1, + "verifier": 93.0 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 1032.0, @@ -29,5 +47,7 @@ "total_tokens": 76194.0 }, "tool_calls": 64, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json index 55861a2..27e6fd9 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784091892.627182, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 0, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": null, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 509.5, + "agent_setup": 9.9, + "environment_setup": 35.6, + "total": 726.0, + "verifier": 127.9 + }, "token_usage": null, "tool_calls": 27, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json index 02c4fe0..1358f83 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 79.783393502, "observed_raw_score": 752.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 752.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 2520.5, + "agent_setup": 7.6, + "environment_setup": 4.1, + "total": 2645.7, + "verifier": 91.5 + }, "token_usage": null, "tool_calls": 29, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json index 73c7249..a45e2a8 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 864.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 864.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1740.2, + "agent_setup": 8.3, + "environment_setup": 3.0, + "total": 1872.8, + "verifier": 98.4 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 722.0, @@ -29,5 +47,7 @@ "total_tokens": 66867.0 }, "tool_calls": 51, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json index 40d16ef..091af31 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 95.848375451, "observed_raw_score": 841.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 841.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1830.2, + "agent_setup": 10.0, + "environment_setup": 5.0, + "total": 2005.8, + "verifier": 121.0 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 653.0, @@ -29,5 +47,7 @@ "total_tokens": 47410.0 }, "tool_calls": 35, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json index 7d688f4..df1b8e5 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 864.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 864.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1020.3, + "agent_setup": 10.0, + "environment_setup": 4.9, + "total": 1194.3, + "verifier": 119.9 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 541.0, @@ -29,5 +47,7 @@ "total_tokens": 42018.0 }, "tool_calls": 26, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json index bdff75f..1ad1133 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784105005.326783, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 41.669121704, "observed_raw_score": 7120.625663985211, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 7120.625663985211, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01/tasks-81ec8d98a4", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 2190.2, + "agent_setup": 9.3, + "environment_setup": 48.1, + "total": 4729.7, + "verifier": 2444.7 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 879.0, @@ -29,5 +47,7 @@ "total_tokens": 66620.0 }, "tool_calls": 42, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json index 8cbd51e..da40475 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784103481.173677, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 42.425113199, "observed_raw_score": 7177.218944151435, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 7177.218944151435, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1620.2, + "agent_setup": 7.5, + "environment_setup": 37.4, + "total": 4151.8, + "verifier": 2464.5 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 713.0, @@ -29,5 +47,7 @@ "total_tokens": 63371.0 }, "tool_calls": 54, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json index 680e091..5912ba5 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 9.827504681, "observed_raw_score": 4736.972455374246, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 4736.972455374246, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1920.2, + "agent_setup": 8.2, + "environment_setup": 2.4, + "total": 4225.0, + "verifier": 2270.1 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 715.0, @@ -29,5 +47,7 @@ "total_tokens": 77110.0 }, "tool_calls": 43, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json index d9f008e..8981fab 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 77.681592164, "observed_raw_score": 9816.507619476308, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 9816.507619476308, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 4260.3, + "agent_setup": 7.7, + "environment_setup": 1.9, + "total": 7570.4, + "verifier": 3275.0 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 685.0, @@ -29,5 +47,7 @@ "total_tokens": 60238.0 }, "tool_calls": 43, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json index 843e40b..ffed7d2 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 33.070043627, "observed_raw_score": 6476.901445020279, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 6476.901445020279, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 6420.7, + "agent_setup": 7.6, + "environment_setup": 5.4, + "total": 8881.7, + "verifier": 2425.9 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 755.0, @@ -29,5 +47,7 @@ "total_tokens": 81343.0 }, "tool_calls": 78, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-01/metadata.json index fe3443f..f082e58 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784091892.627182, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 0, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": null, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1543.2, + "agent_setup": 8.8, + "environment_setup": 2.6, + "total": 1638.4, + "verifier": 50.3 + }, "token_usage": null, "tool_calls": 83, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-02/metadata.json index cadf07a..5d124cc 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784103481.173677, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 3600.8, + "agent_setup": 9.5, + "environment_setup": 4.3, + "total": 3689.5, + "verifier": 45.2 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 1154.0, @@ -29,5 +47,7 @@ "total_tokens": 203477.0 }, "tool_calls": 176, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-03/metadata.json index 012080e..5c0fe38 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 3264.4, + "agent_setup": 8.6, + "environment_setup": 3.3, + "total": 3686.4, + "verifier": 49.9 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 1116.0, @@ -29,5 +47,7 @@ "total_tokens": 179280.0 }, "tool_calls": 157, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-04/metadata.json index e9cf50e..6adc1cb 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1860.8, + "agent_setup": 8.7, + "environment_setup": 2.2, + "total": 1947.1, + "verifier": 46.3 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 957.0, @@ -29,5 +47,7 @@ "total_tokens": 138378.0 }, "tool_calls": 108, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-05/metadata.json index 02b1eab..f6f100c 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_montezuma_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1410.2, + "agent_setup": 9.5, + "environment_setup": 2.8, + "total": 1505.1, + "verifier": 51.3 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 829.0, @@ -29,5 +47,7 @@ "total_tokens": 109401.0 }, "tool_calls": 82, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-01/metadata.json index d6e56bc..410364b 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784091892.627182, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 0, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 100.0, "observed_raw_score": 21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": null, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 587.8, + "agent_setup": 9.8, + "environment_setup": 2.5, + "total": 679.5, + "verifier": 54.3 + }, "token_usage": null, "tool_calls": 33, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-02/metadata.json index 3419217..392bc58 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784103481.173677, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1680.3, + "agent_setup": 7.7, + "environment_setup": 2.6, + "total": 1772.1, + "verifier": 57.8 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 718.0, @@ -29,5 +47,7 @@ "total_tokens": 89133.0 }, "tool_calls": 91, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-03/metadata.json index eaf7605..df1aa6e 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 98.333333333, "observed_raw_score": 20.6, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 20.6, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 960.2, + "agent_setup": 7.4, + "environment_setup": 1.8, + "total": 1052.6, + "verifier": 60.0 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 636.0, @@ -29,5 +47,7 @@ "total_tokens": 53285.0 }, "tool_calls": 39, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-04/metadata.json index 29a4f07..d48eddd 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 450.1, + "agent_setup": 7.8, + "environment_setup": 1.9, + "total": 539.5, + "verifier": 56.6 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 655.0, @@ -29,5 +47,7 @@ "total_tokens": 44227.0 }, "tool_calls": 36, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-05/metadata.json index a291875..448c310 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_pong_ram_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 570.1, + "agent_setup": 7.5, + "environment_setup": 2.2, + "total": 653.5, + "verifier": 51.8 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 526.0, @@ -29,5 +47,7 @@ "total_tokens": 55373.0 }, "tool_calls": 46, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json index 45a1e7c..1973343 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784091892.627182, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 0, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 124.779364311, "observed_raw_score": 0.9110399397462606, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.9110399397462606, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": null, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 820.0, + "agent_setup": 11.0, + "environment_setup": 10.8, + "total": 933.8, + "verifier": 45.1 + }, "token_usage": null, "tool_calls": 56, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json index bf9b224..b0360a2 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784103481.173677, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 100.2179605, "observed_raw_score": 0.8186399422399699, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.8186399422399699, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1200.7, + "agent_setup": 11.3, + "environment_setup": 2.7, + "total": 1298.4, + "verifier": 42.8 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 497.0, @@ -29,5 +47,7 @@ "total_tokens": 44166.0 }, "tool_calls": 45, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json index f2ef1d3..b033076 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 85.890478581, "observed_raw_score": 0.7647399573773146, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.7647399573773146, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1522.8, + "agent_setup": 10.5, + "environment_setup": 3.1, + "total": 2385.6, + "verifier": 38.5 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 643.0, @@ -29,5 +47,7 @@ "total_tokens": 80001.0 }, "tool_calls": 82, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json index 1d499e4..d8177b6 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 113.721417769, "observed_raw_score": 0.869439946487546, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.869439946487546, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1410.6, + "agent_setup": 11.5, + "environment_setup": 3.3, + "total": 1525.7, + "verifier": 57.6 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 690.0, @@ -29,5 +47,7 @@ "total_tokens": 75108.0 }, "tool_calls": 77, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json index 2ff2704..f50aad7 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 111.116423225, "observed_raw_score": 0.8596399573981761, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.8596399573981761, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1020.4, + "agent_setup": 10.9, + "environment_setup": 3.6, + "total": 1121.8, + "verifier": 39.2 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 666.0, @@ -29,5 +47,7 @@ "total_tokens": 73959.0 }, "tool_calls": 58, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json index 2f90e0c..5813b55 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784091892.627182, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 0, "daytona_pty_readline_timeout_sec": 3600, @@ -15,13 +18,30 @@ }, "normalized_score": 84.45855615, "observed_raw_score": 284.6, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 284.6, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": null, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-01", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1928.3, + "agent_setup": 10.5, + "environment_setup": 2.6, + "total": 2632.7, + "verifier": 658.9 + }, "token_usage": null, "tool_calls": 46, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json index 33af5e0..2302b4c 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784103481.173677, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 114.104278075, "observed_raw_score": 373.3, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 373.3, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-02/tasks-e7ee9d1b4f", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1741.0, + "agent_setup": 8.3, + "environment_setup": 2.1, + "total": 1832.4, + "verifier": 52.1 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 781.0, @@ -29,5 +47,7 @@ "total_tokens": 83729.0 }, "tool_calls": 81, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json index 3394278..a8e98f1 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784113784.201615, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 27.907754011, "observed_raw_score": 115.4, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 115.4, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-03", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 3094.1, + "agent_setup": 9.3, + "environment_setup": 2.3, + "total": 3200.4, + "verifier": 64.5 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 766.0, @@ -29,5 +47,7 @@ "total_tokens": 105494.0 }, "tool_calls": 111, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json index 92e162e..2bcd9e2 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784120178.69071, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 134.35828877, "observed_raw_score": 433.9, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 433.9, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-04", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1021.9, + "agent_setup": 9.1, + "environment_setup": 2.2, + "total": 1119.8, + "verifier": 58.4 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 693.0, @@ -29,5 +47,7 @@ "total_tokens": 55411.0 }, "tool_calls": 53, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json index af7fab6..44658d0 100644 --- a/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/claude-opus-4.8/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784110074.558851, "harness": "opencode", + "idle_timeout_info": null, "model": { "agent_idle_timeout_sec": 3600, "daytona_pty_readline_timeout_sec": 3600, @@ -15,12 +18,27 @@ }, "normalized_score": 121.356951872, "observed_raw_score": 394.99999999999994, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 394.99999999999994, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-claude/claude-opus-4.8/trial-05", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1590.2, + "agent_setup": 9.8, + "environment_setup": 2.3, + "total": 1682.4, + "verifier": 50.9 + }, "token_usage": { "final_input_tokens": 2.0, "final_output_tokens": 944.0, @@ -29,5 +47,7 @@ "total_tokens": 63917.0 }, "tool_calls": 76, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-01/metadata.json index 9195a3f..824c638 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784074742.653706, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -36.632193477, "observed_raw_score": 1162.2321740972948, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 1162.2321740972948, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.8, + "environment_setup": 3.6, + "total": 1983.0, + "verifier": 1886.3 + }, "token_usage": { "final_input_tokens": 1068.0, "final_output_tokens": 214.0, @@ -27,5 +45,7 @@ "total_tokens": 13570.0 }, "tool_calls": 13, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-02/metadata.json index cbaf302..5b9841f 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784074623.04269, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -50.421772091, "observed_raw_score": 679.8013660996173, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 679.8013660996173, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 63.8, + "agent_setup": 3.2, + "environment_setup": 6.7, + "total": 2320.6, + "verifier": 2146.7 + }, "token_usage": { "final_input_tokens": 1804.0, "final_output_tokens": 227.0, @@ -27,5 +45,7 @@ "total_tokens": 14831.0 }, "tool_calls": 15, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-03/metadata.json index b925bb9..88c3214 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -40.65567106, "observed_raw_score": 1021.4701105134167, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 1021.4701105134167, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.6, + "environment_setup": 3.4, + "total": 2009.0, + "verifier": 1914.9 + }, "token_usage": { "final_input_tokens": 955.0, "final_output_tokens": 186.0, @@ -27,5 +45,7 @@ "total_tokens": 12917.0 }, "tool_calls": 13, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-04/metadata.json index 65c1129..41f1a06 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -63.436898931, "observed_raw_score": 224.46488820508256, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 224.46488820508256, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 3.2, + "environment_setup": 3.3, + "total": 2311.9, + "verifier": 2189.3 + }, "token_usage": { "final_input_tokens": 417.0, "final_output_tokens": 195.0, @@ -27,5 +45,7 @@ "total_tokens": 20580.0 }, "tool_calls": 14, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-05/metadata.json index 8e9bb83..86ee809 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_ant_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -45.015963942, "observed_raw_score": 868.924505125946, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 868.924505125946, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.9, + "environment_setup": 3.0, + "total": 2193.8, + "verifier": 2092.8 + }, "token_usage": { "final_input_tokens": 563.0, "final_output_tokens": 155.0, @@ -27,5 +45,7 @@ "total_tokens": 14030.0 }, "tool_calls": 17, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-01/metadata.json index 70a7e86..171d4be 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784074742.653706, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.2, + "environment_setup": 3.3, + "total": 105.9, + "verifier": 15.0 + }, "token_usage": { "final_input_tokens": 5807.0, "final_output_tokens": 235.0, @@ -27,5 +45,7 @@ "total_tokens": 16282.0 }, "tool_calls": 17, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-02/metadata.json index 16bad36..11080a6 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784074623.04269, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 929.2, + "agent_setup": 4.2, + "environment_setup": 4.1, + "total": 1102.2, + "verifier": 12.9 + }, "token_usage": { "final_input_tokens": 651.0, "final_output_tokens": 207.0, @@ -27,5 +45,7 @@ "total_tokens": 15706.0 }, "tool_calls": 13, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-03/metadata.json index 401447c..6f1fe23 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.1, + "agent_setup": 3.4, + "environment_setup": 3.2, + "total": 103.6, + "verifier": 12.2 + }, "token_usage": { "final_input_tokens": 5482.0, "final_output_tokens": 145.0, @@ -27,5 +45,7 @@ "total_tokens": 17915.0 }, "tool_calls": 16, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-04/metadata.json index 727efa5..21bcc5d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.6, + "environment_setup": 3.1, + "total": 105.9, + "verifier": 15.5 + }, "token_usage": { "final_input_tokens": 573.0, "final_output_tokens": 197.0, @@ -27,5 +45,7 @@ "total_tokens": 15618.0 }, "tool_calls": 11, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-05/metadata.json index 895da78..499f7da 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_atari57_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 121.7, + "agent_setup": 3.4, + "environment_setup": 3.0, + "total": 164.3, + "verifier": 12.7 + }, "token_usage": { "final_input_tokens": 5648.0, "final_output_tokens": 237.0, @@ -27,5 +45,7 @@ "total_tokens": 23293.0 }, "tool_calls": 22, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json index cb77ad5..6eec74b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784074742.653706, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.209643606, "observed_raw_score": 388.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 388.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 3.4, + "environment_setup": 3.2, + "total": 290.8, + "verifier": 167.7 + }, "token_usage": { "final_input_tokens": 542.0, "final_output_tokens": 214.0, @@ -27,5 +45,7 @@ "total_tokens": 16628.0 }, "tool_calls": 18, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json index 2dd3d22..64076c0 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784075014.388146, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -0.628930818, "observed_raw_score": 384.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 384.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02/simulation_heuristics_breakout_ram_v1", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 2.8, + "environment_setup": 3.0, + "total": 222.7, + "verifier": 133.2 + }, "token_usage": { "final_input_tokens": 782.0, "final_output_tokens": 192.0, @@ -27,5 +45,7 @@ "total_tokens": 17358.0 }, "tool_calls": 20, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json index 72d5dbb..786227a 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 3.983228512, "observed_raw_score": 406.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 406.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.1, + "agent_setup": 3.1, + "environment_setup": 8.0, + "total": 271.1, + "verifier": 176.4 + }, "token_usage": { "final_input_tokens": 2251.0, "final_output_tokens": 194.0, @@ -27,5 +45,7 @@ "total_tokens": 16781.0 }, "tool_calls": 17, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json index 43b215a..1eb291a 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 6.708595388, "observed_raw_score": 419.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 419.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.3, + "agent_setup": 3.1, + "environment_setup": 3.4, + "total": 227.2, + "verifier": 106.6 + }, "token_usage": { "final_input_tokens": 661.0, "final_output_tokens": 166.0, @@ -27,5 +45,7 @@ "total_tokens": 16187.0 }, "tool_calls": 22, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json index 06ebc44..2446b7e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -16.981132075, "observed_raw_score": 306.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 306.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.8, + "environment_setup": 6.8, + "total": 206.0, + "verifier": 109.5 + }, "token_usage": { "final_input_tokens": 999.0, "final_output_tokens": 197.0, @@ -27,5 +45,7 @@ "total_tokens": 15020.0 }, "tool_calls": 20, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json index 510347e..5c7609b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784074742.653706, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 8.664259928, "observed_raw_score": 358.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 358.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 30.9, + "agent_setup": 5.0, + "environment_setup": 4.6, + "total": 198.6, + "verifier": 108.7 + }, "token_usage": { "final_input_tokens": 505.0, "final_output_tokens": 161.0, @@ -27,5 +45,7 @@ "total_tokens": 15002.0 }, "tool_calls": 8, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json index b62066c..2b62e41 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784074623.04269, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 95.848375451, "observed_raw_score": 841.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 841.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.2, + "agent_setup": 7.2, + "environment_setup": 4.6, + "total": 1171.8, + "verifier": 1013.9 + }, "token_usage": { "final_input_tokens": 3353.0, "final_output_tokens": 179.0, @@ -27,5 +45,7 @@ "total_tokens": 17868.0 }, "tool_calls": 9, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json index dd72a79..2902eb0 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 310.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 310.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 3.4, + "environment_setup": 2.8, + "total": 201.4, + "verifier": 80.1 + }, "token_usage": { "final_input_tokens": 356.0, "final_output_tokens": 123.0, @@ -27,5 +45,7 @@ "total_tokens": 19423.0 }, "tool_calls": 13, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json index e2d0019..a286291 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -48.194945848, "observed_raw_score": 43.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 43.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.4, + "environment_setup": 3.0, + "total": 169.6, + "verifier": 72.5 + }, "token_usage": { "final_input_tokens": 416.0, "final_output_tokens": 150.0, @@ -27,5 +45,7 @@ "total_tokens": 18998.0 }, "tool_calls": 11, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json index 136892c..ff4970f 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 15.342960289, "observed_raw_score": 395.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 395.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.2, + "agent_setup": 4.8, + "environment_setup": 3.6, + "total": 236.4, + "verifier": 126.2 + }, "token_usage": { "final_input_tokens": 3633.0, "final_output_tokens": 160.0, @@ -27,5 +45,7 @@ "total_tokens": 18641.0 }, "tool_calls": 11, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json index 37fe11b..0bf045d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784074742.653706, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -53.572769951, "observed_raw_score": -9.15171128521311, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -9.15171128521311, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.8, + "environment_setup": 2.4, + "total": 2090.9, + "verifier": 1997.1 + }, "token_usage": { "final_input_tokens": 1058.0, "final_output_tokens": 280.0, @@ -27,5 +45,7 @@ "total_tokens": 19770.0 }, "tool_calls": 18, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json index 7aeb83b..d3bf528 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784074623.04269, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -12.137156309, "observed_raw_score": 3092.704998440495, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 3092.704998440495, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.4, + "environment_setup": 3.6, + "total": 2107.2, + "verifier": 2009.7 + }, "token_usage": { "final_input_tokens": 987.0, "final_output_tokens": 225.0, @@ -27,5 +45,7 @@ "total_tokens": 17084.0 }, "tool_calls": 16, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json index 7d8ef70..7b90bc7 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -53.770506118, "observed_raw_score": -23.954177164674096, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -23.954177164674096, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.4, + "environment_setup": 2.2, + "total": 2062.6, + "verifier": 1972.1 + }, "token_usage": { "final_input_tokens": 1211.0, "final_output_tokens": 235.0, @@ -27,5 +45,7 @@ "total_tokens": 19366.0 }, "tool_calls": 17, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json index 4af4919..a287967 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -27.203049655, "observed_raw_score": 1964.877068442624, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 1964.877068442624, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 5.2, + "environment_setup": 2.6, + "total": 2118.1, + "verifier": 2023.5 + }, "token_usage": { "final_input_tokens": 1077.0, "final_output_tokens": 226.0, @@ -27,5 +45,7 @@ "total_tokens": 18199.0 }, "tool_calls": 17, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json index 2609b97..21305d9 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -52.735512453, "observed_raw_score": 53.52511570897907, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 53.52511570897907, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.1, + "environment_setup": 4.2, + "total": 2627.3, + "verifier": 2529.3 + }, "token_usage": { "final_input_tokens": 1245.0, "final_output_tokens": 213.0, @@ -27,5 +45,7 @@ "total_tokens": 21938.0 }, "tool_calls": 16, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-01/metadata.json index 38320ff..0e95153 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784074742.653706, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 4.3, + "environment_setup": 2.6, + "total": 149.2, + "verifier": 52.9 + }, "token_usage": { "final_input_tokens": 674.0, "final_output_tokens": 192.0, @@ -27,5 +45,7 @@ "total_tokens": 14690.0 }, "tool_calls": 24, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-02/metadata.json index 867df8a..c5bd87c 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784074623.04269, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 4.5, + "environment_setup": 2.6, + "total": 198.1, + "verifier": 98.7 + }, "token_usage": { "final_input_tokens": 694.0, "final_output_tokens": 320.0, @@ -27,5 +45,7 @@ "total_tokens": 16886.0 }, "tool_calls": 18, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-03/metadata.json index f0873f9..724b36d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.8, + "environment_setup": 2.2, + "total": 142.5, + "verifier": 45.5 + }, "token_usage": { "final_input_tokens": 951.0, "final_output_tokens": 235.0, @@ -27,5 +45,7 @@ "total_tokens": 17570.0 }, "tool_calls": 14, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-04/metadata.json index 7c0ffd4..c19c6e5 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 4.6, + "environment_setup": 2.9, + "total": 140.6, + "verifier": 42.1 + }, "token_usage": { "final_input_tokens": 317.0, "final_output_tokens": 173.0, @@ -27,5 +45,7 @@ "total_tokens": 14826.0 }, "tool_calls": 18, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-05/metadata.json index 4252f9d..6ed276c 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_montezuma_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 4.0, + "environment_setup": 3.1, + "total": 148.7, + "verifier": 51.5 + }, "token_usage": { "final_input_tokens": 514.0, "final_output_tokens": 258.0, @@ -27,5 +45,7 @@ "total_tokens": 12548.0 }, "tool_calls": 16, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-01/metadata.json index 0117bd0..25c9eb2 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784074742.653706, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -75.0, "observed_raw_score": -21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 4.2, + "environment_setup": 2.3, + "total": 145.9, + "verifier": 51.6 + }, "token_usage": { "final_input_tokens": 497.0, "final_output_tokens": 207.0, @@ -27,5 +45,7 @@ "total_tokens": 17600.0 }, "tool_calls": 20, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-02/metadata.json index a351970..f346f97 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784074623.04269, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -75.0, "observed_raw_score": -21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 120.0, + "agent_setup": 3.8, + "environment_setup": 3.1, + "total": 207.4, + "verifier": 55.5 + }, "token_usage": { "final_input_tokens": 720.0, "final_output_tokens": 190.0, @@ -27,5 +45,7 @@ "total_tokens": 17806.0 }, "tool_calls": 21, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-03/metadata.json index 77fa72c..9bac49b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -58.333333333, "observed_raw_score": -17.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -17.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.3, + "environment_setup": 2.5, + "total": 153.3, + "verifier": 61.3 + }, "token_usage": { "final_input_tokens": 845.0, "final_output_tokens": 262.0, @@ -27,5 +45,7 @@ "total_tokens": 13907.0 }, "tool_calls": 15, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-04/metadata.json index 5bd783f..47e4e4b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -75.0, "observed_raw_score": -21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.7, + "environment_setup": 2.6, + "total": 138.5, + "verifier": 47.1 + }, "token_usage": { "final_input_tokens": 544.0, "final_output_tokens": 229.0, @@ -27,5 +45,7 @@ "total_tokens": 14597.0 }, "tool_calls": 13, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-05/metadata.json index ed0b350..b5d4398 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_pong_ram_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 79.166666667, "observed_raw_score": 16.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 16.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 3.7, + "environment_setup": 4.3, + "total": 208.2, + "verifier": 64.9 + }, "token_usage": { "final_input_tokens": 885.0, "final_output_tokens": 198.0, @@ -27,5 +45,7 @@ "total_tokens": 13371.0 }, "tool_calls": 14, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json index d906837..7df2309 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784074742.653706, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -180.053168387, "observed_raw_score": -0.2357400032132864, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -0.2357400032132864, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 5.4, + "environment_setup": 3.4, + "total": 157.7, + "verifier": 42.5 + }, "token_usage": { "final_input_tokens": 921.0, "final_output_tokens": 262.0, @@ -27,5 +45,7 @@ "total_tokens": 11423.0 }, "tool_calls": 11, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json index 8cf2768..bfde530 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784074623.04269, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -187.56513016, "observed_raw_score": -0.2640000022947788, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -0.2640000022947788, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 4.8, + "environment_setup": 3.6, + "total": 148.2, + "verifier": 35.0 + }, "token_usage": { "final_input_tokens": 1505.0, "final_output_tokens": 220.0, @@ -27,5 +45,7 @@ "total_tokens": 14013.0 }, "tool_calls": 12, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json index f806f9b..7005032 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -187.56513016, "observed_raw_score": -0.2640000022947788, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -0.2640000022947788, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.2, + "agent_setup": 4.9, + "environment_setup": 4.1, + "total": 148.3, + "verifier": 33.8 + }, "token_usage": { "final_input_tokens": 1478.0, "final_output_tokens": 227.0, @@ -27,5 +45,7 @@ "total_tokens": 11945.0 }, "tool_calls": 10, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json index 581a69f..8fd7e7e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -187.56513016, "observed_raw_score": -0.2640000022947788, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -0.2640000022947788, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 5.0, + "environment_setup": 10.5, + "total": 150.0, + "verifier": 32.6 + }, "token_usage": { "final_input_tokens": 765.0, "final_output_tokens": 195.0, @@ -27,5 +45,7 @@ "total_tokens": 12736.0 }, "tool_calls": 12, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json index f697e5a..c06307c 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -187.56513016, "observed_raw_score": -0.2640000022947788, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -0.2640000022947788, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 30.0, + "agent_setup": 5.0, + "environment_setup": 6.8, + "total": 222.0, + "verifier": 134.3 + }, "token_usage": { "final_input_tokens": 1720.0, "final_output_tokens": 217.0, @@ -27,5 +45,7 @@ "total_tokens": 12177.0 }, "tool_calls": 12, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json index d117398..d77aa96 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784075921.181849, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 97.159090909, "observed_raw_score": 322.6, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 322.6, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-01/simulation_heuristics_vizdoom_d3_v1", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.8, + "environment_setup": 2.4, + "total": 193.4, + "verifier": 97.8 + }, "token_usage": { "final_input_tokens": 634.0, "final_output_tokens": 199.0, @@ -27,5 +45,7 @@ "total_tokens": 19777.0 }, "tool_calls": 18, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json index 06b01f4..01c8db2 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784074623.04269, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -10.160427807, "observed_raw_score": 1.5, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 1.5, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-02", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.8, + "environment_setup": 2.7, + "total": 122.9, + "verifier": 26.4 + }, "token_usage": { "final_input_tokens": 632.0, "final_output_tokens": 194.0, @@ -27,5 +45,7 @@ "total_tokens": 14138.0 }, "tool_calls": 15, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json index d727fd6..4328d61 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784076792.366179, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": 1.771390374, "observed_raw_score": 37.2, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 37.2, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-03", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 4.2, + "environment_setup": 2.9, + "total": 147.1, + "verifier": 49.9 + }, "token_usage": { "final_input_tokens": 796.0, "final_output_tokens": 213.0, @@ -27,5 +45,7 @@ "total_tokens": 12785.0 }, "tool_calls": 12, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json index 38609fe..2aea3c0 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784079105.716872, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -2.506684492, "observed_raw_score": 24.4, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 24.4, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-04", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 4.3, + "environment_setup": 2.7, + "total": 172.0, + "verifier": 44.5 + }, "token_usage": { "final_input_tokens": 1817.0, "final_output_tokens": 217.0, @@ -27,5 +45,7 @@ "total_tokens": 20466.0 }, "tool_calls": 19, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json index ce7e2fe..151e36c 100644 --- a/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.4-mini/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784081904.218542, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.4 Mini", "id": "gpt-5.4-mini", @@ -13,12 +16,27 @@ }, "normalized_score": -3.609625668, "observed_raw_score": 21.1, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 21.1, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt54mini/gpt-5.4-mini/trial-05", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 802.7, + "agent_setup": 5.6, + "environment_setup": 2.7, + "total": 1041.3, + "verifier": 73.9 + }, "token_usage": { "final_input_tokens": 558.0, "final_output_tokens": 251.0, @@ -27,5 +45,7 @@ "total_tokens": 14633.0 }, "tool_calls": 9, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/metadata.json index b376b48..7e7bfe2 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/metadata.json @@ -9,17 +9,17 @@ "provider": "azure", "provider_reasoning_effort": "xhigh" }, - "normalized_score": -12.5116242462, - "normalized_score_stddev": 8.756962566015595, + "normalized_score": -12.2320084794, + "normalized_score_stddev": 9.211087046248206, "provider_reasoning_effort": "xhigh", - "raw_score": 2006.0944869926293, + "raw_score": 2015.8768932664839, "raw_score_imputation_count": 0, - "raw_score_stddev": 306.36385957656836, + "raw_score_stddev": 322.25148356656894, "source_run_ids": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4" ], "task": "simulation_heuristics_ant_v1", diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/score.json index 845c316..27cb58d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/score.json @@ -1,11 +1,11 @@ { "benchmark": "learning_beyond_gradients_article_suite", - "normalized_score": -12.5116242462, - "normalized_score_stddev": 8.756962566015595, + "normalized_score": -12.2320084794, + "normalized_score_stddev": 9.211087046248206, "raw_score_imputation_count": 0, "reference_score": 5942.333244755838, - "score": 2006.0944869926293, - "score_stddev": 306.36385957656836, + "score": 2015.8768932664839, + "score_stddev": 322.25148356656894, "starter_score": 2443.815838992282, "task": "simulation_heuristics_ant_v1", "trial_count": 5, @@ -38,12 +38,12 @@ "trial": 3 }, { - "normalized_score": -1.398078834, - "observed_raw_score": 2394.903807623008, - "raw_score": 2394.903807623008, + "normalized_score": 0.0, + "observed_raw_score": 2443.815838992282, + "raw_score": 2443.815838992282, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "trial": 4 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-01/metadata.json index 5d37c9c..5b6dcb0 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784074045.3527198, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": -6.720960755, "observed_raw_score": 2208.681857129609, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 2208.681857129609, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 240.7, + "agent_setup": 3.1, + "environment_setup": 35.1, + "total": 4668.7, + "verifier": 4365.0 + }, "token_usage": { "final_input_tokens": 1609.0, "final_output_tokens": 282.0, @@ -27,5 +45,7 @@ "total_tokens": 17763.0 }, "tool_calls": 18, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-02/metadata.json index 3254321..4aac9f0 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": -24.004147668, "observed_raw_score": 1604.0265547373397, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 1604.0265547373397, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 243.4, + "agent_setup": 3.5, + "environment_setup": 3.9, + "total": 2176.6, + "verifier": 1898.2 + }, "token_usage": { "final_input_tokens": 981.0, "final_output_tokens": 215.0, @@ -27,5 +45,7 @@ "total_tokens": 22700.0 }, "tool_calls": 28, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-03/metadata.json index d89edf6..3bca6e5 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": -13.99811598, "observed_raw_score": 1954.0893149644958, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 1954.0893149644958, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 315.9, + "agent_setup": 3.3, + "environment_setup": 3.6, + "total": 2273.2, + "verifier": 1915.2 + }, "token_usage": { "final_input_tokens": 2383.0, "final_output_tokens": 202.0, @@ -27,5 +45,7 @@ "total_tokens": 36889.0 }, "tool_calls": 17, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/metadata.json index 93a1317..59cad0b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/metadata.json @@ -1,31 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, - "finished_at": 1784093477.189311, + "finished_at": 1784140206.649261, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.5", "id": "gpt-5.5", "model": "azure/gpt-5.5", "provider": "azure", "provider_reasoning_effort": "xhigh" }, - "normalized_score": -1.398078834, - "observed_raw_score": 2394.903807623008, + "normalized_score": 0.0, + "observed_raw_score": 2443.815838992282, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", - "raw_score": 2394.903807623008, + "published_protocol_version": "2.2", + "raw_score": 2443.815838992282, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 3.8, + "environment_setup": 3.1, + "total": 2294.3, + "verifier": 2168.3 + }, "token_usage": { - "final_input_tokens": 4200.0, - "final_output_tokens": 244.0, - "input_tokens": 4200.0, - "output_tokens": 244.0, - "total_tokens": 20316.0 + "final_input_tokens": 855.0, + "final_output_tokens": 176.0, + "input_tokens": 855.0, + "output_tokens": 176.0, + "total_tokens": 18439.0 }, - "tool_calls": 23, - "trial": 4 + "tool_calls": 19, + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/score.json index 908c7ae..8e334e6 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-04/score.json @@ -4,66 +4,66 @@ { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.015127851627767086, + "mean_action_latency_ms": 0.019004330039024353, "policy_error": null, - "return": 2632.738449893457, + "return": 2289.7232686812636, "seed": 101, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 86.59133294253101, - "x_velocity": 2.1494054053925993 + "x_position": 69.0973157207015, + "x_velocity": -0.49558561649178046 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.014626999851316214, + "mean_action_latency_ms": 0.01928306045010686, "policy_error": null, - "return": 2759.2392216673875, + "return": 2797.4710760738367, "seed": 211, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 92.80698215403275, - "x_velocity": 1.9592731817380127 + "x_position": 94.40942573977297, + "x_velocity": 1.9289379496191827 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.014990258030593395, + "mean_action_latency_ms": 0.018419045489281416, "policy_error": null, - "return": 2468.526557576306, + "return": 2522.2229765019474, "seed": 307, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 78.33519158866122, - "x_velocity": 1.6909310063846306 + "x_position": 80.69326425148162, + "x_velocity": 1.114561459622223 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.01611954253166914, + "mean_action_latency_ms": 0.018489803187549114, "policy_error": null, - "return": 1606.5678228059596, + "return": 2539.470332732216, "seed": 401, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 35.227475531807826, - "x_velocity": -0.4673749187759313 + "x_position": 81.55640383531237, + "x_velocity": -0.5628020664562428 } ], "fall_rate": 0.0, "invalid_episode_rate": 0.0, - "max_return": 2759.2392216673875, + "max_return": 2797.4710760738367, "max_steps": 1000, - "mean_action_latency_ms": 0.015216163010336459, - "mean_return": 2366.768012985777, - "mean_x_position": 73.2402455542582, - "min_return": 1606.5678228059596, + "mean_action_latency_ms": 0.018799059791490436, + "mean_return": 2537.2219134973157, + "mean_x_position": 81.43910238681711, + "min_return": 2289.7232686812636, "policy_path": "submitted_artifact", - "score": 2366.768012985777 + "score": 2537.2219134973157 }, "hidden_nominal_weight": 0.7, "hidden_robustness": { @@ -71,71 +71,71 @@ { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.014720248524099588, + "mean_action_latency_ms": 0.01842273212969303, "policy_error": null, - "return": 2560.1209086767167, + "return": 2703.792461897773, "seed": 503, "terminated": false, "truncated": true, "variant": "light_low_friction", - "x_position": 82.81068500616887, - "x_velocity": 0.48071101945112105 + "x_position": 89.65583612193788, + "x_velocity": 1.9894651276723607 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.015089788008481264, + "mean_action_latency_ms": 0.02069002715870738, "policy_error": null, - "return": 2779.6893993691415, + "return": 2358.4274147080555, "seed": 601, "terminated": false, "truncated": true, "variant": "light_low_friction", - "x_position": 93.91444720784214, - "x_velocity": 0.6902947820412919 + "x_position": 72.46751601168395, + "x_velocity": 2.2021194196955207 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.015854522120207548, + "mean_action_latency_ms": 0.01968169491738081, "policy_error": null, - "return": 2273.636836518902, + "return": 1491.6697450838283, "seed": 503, "terminated": false, "truncated": true, "variant": "heavy_weak_actuators", - "x_position": 68.94608289857779, - "x_velocity": 1.8331548234507977 + "x_position": 29.491943987328916, + "x_velocity": -0.36521004981096894 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.015502279158681631, + "mean_action_latency_ms": 0.019597063772380352, "policy_error": null, - "return": 2228.768835874762, + "return": 2349.583705565823, "seed": 601, "terminated": false, "truncated": true, "variant": "heavy_weak_actuators", - "x_position": 66.80544008518254, - "x_velocity": 0.6067153490798205 + "x_position": 72.46803385324752, + "x_velocity": 1.9787478655692325 } ], "fall_rate": 0.0, "invalid_episode_rate": 0.0, - "max_return": 2779.6893993691415, + "max_return": 2703.792461897773, "max_steps": 1000, - "mean_action_latency_ms": 0.015291709452867508, - "mean_return": 2460.553995109881, - "mean_x_position": 78.11916379944284, - "min_return": 2228.768835874762, + "mean_action_latency_ms": 0.019597879494540393, + "mean_return": 2225.86833181387, + "mean_x_position": 66.02083249354956, + "min_return": 1491.6697450838283, "policy_path": "submitted_artifact", - "score": 2460.553995109881 + "score": 2225.86833181387 }, "hidden_robustness_weight": 0.3, - "normalized_score": -1.398078834, + "normalized_score": 0.0, "reference_score": 5942.333244755838, - "score": 2394.903807623008, + "score": 2443.815838992282, "starter_score": 2443.815838992282, "unique_policy_evaluations": 3 } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-05/metadata.json index 8e5241a..d09e570 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_ant_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784097871.7138891, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": -16.436817994, "observed_raw_score": 1868.7709005086936, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 1868.7709005086936, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 248.6, + "agent_setup": 2.8, + "environment_setup": 4.3, + "total": 2217.6, + "verifier": 1937.6 + }, "token_usage": { "final_input_tokens": 610.0, "final_output_tokens": 234.0, @@ -27,5 +45,7 @@ "total_tokens": 26444.0 }, "tool_calls": 24, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-01/metadata.json index 03fcbbb..dd91222 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784074045.3527198, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 92.9, + "agent_setup": 3.0, + "environment_setup": 3.4, + "total": 137.0, + "verifier": 12.6 + }, "token_usage": { "final_input_tokens": 5475.0, "final_output_tokens": 286.0, @@ -27,5 +45,7 @@ "total_tokens": 20097.0 }, "tool_calls": 17, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-02/metadata.json index 4fadd6e..aa69864 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 91.2, + "agent_setup": 2.9, + "environment_setup": 3.8, + "total": 135.6, + "verifier": 14.8 + }, "token_usage": { "final_input_tokens": 5698.0, "final_output_tokens": 300.0, @@ -27,5 +45,7 @@ "total_tokens": 19310.0 }, "tool_calls": 14, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-03/metadata.json index dec7816..f09748a 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.6, + "environment_setup": 2.9, + "total": 352.1, + "verifier": 16.6 + }, "token_usage": { "final_input_tokens": 5847.0, "final_output_tokens": 283.0, @@ -27,5 +45,7 @@ "total_tokens": 19442.0 }, "tool_calls": 14, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-04/metadata.json index b7c2149..a4c1595 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784093477.189311, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.1, + "agent_setup": 3.2, + "environment_setup": 11.4, + "total": 767.9, + "verifier": 640.1 + }, "token_usage": { "final_input_tokens": 5923.0, "final_output_tokens": 258.0, @@ -27,5 +45,7 @@ "total_tokens": 20005.0 }, "tool_calls": 14, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-05/metadata.json index 11dfb29..6ac3385 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_atari57_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,13 +16,30 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.1, + "agent_setup": 3.1, + "environment_setup": 2.5, + "total": 97.8, + "verifier": 10.7 + }, "token_usage": null, "tool_calls": 14, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json index 61fcf5d..3d5a67a 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784074045.3527198, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 5.870020964, "observed_raw_score": 415.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 415.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 180.2, + "agent_setup": 3.0, + "environment_setup": 34.1, + "total": 352.0, + "verifier": 115.4 + }, "token_usage": { "final_input_tokens": 636.0, "final_output_tokens": 164.0, @@ -27,5 +45,7 @@ "total_tokens": 32544.0 }, "tool_calls": 30, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json index 80e1c0b..66d6997 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 70.020964361, "observed_raw_score": 721.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 721.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 242.3, + "agent_setup": 3.1, + "environment_setup": 3.5, + "total": 389.0, + "verifier": 117.1 + }, "token_usage": { "final_input_tokens": 1147.0, "final_output_tokens": 202.0, @@ -27,5 +45,7 @@ "total_tokens": 18245.0 }, "tool_calls": 23, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json index 1ccdf1c..4faaaa3 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 10.062893082, "observed_raw_score": 435.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 435.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1847.3, + "agent_setup": 3.2, + "environment_setup": 3.1, + "total": 1991.7, + "verifier": 113.1 + }, "token_usage": { "final_input_tokens": 655.0, "final_output_tokens": 187.0, @@ -27,5 +45,7 @@ "total_tokens": 18250.0 }, "tool_calls": 21, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json index 7ecbf1d..bccfc92 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784098096.514206, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 9.224318658, "observed_raw_score": 431.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 431.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e6fa3546aa", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1800.3, + "agent_setup": 3.0, + "environment_setup": 3.3, + "total": 1942.0, + "verifier": 110.8 + }, "token_usage": { "final_input_tokens": 396.0, "final_output_tokens": 185.0, @@ -27,5 +45,7 @@ "total_tokens": 31813.0 }, "tool_calls": 29, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json index 5daa07c..a78a1f5 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,13 +16,30 @@ }, "normalized_score": 9.433962264, "observed_raw_score": 432.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 432.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 240.2, + "agent_setup": 3.0, + "environment_setup": 2.7, + "total": 405.6, + "verifier": 136.7 + }, "token_usage": null, "tool_calls": 36, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/metadata.json index 8ad11aa..d85f80e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/metadata.json @@ -9,14 +9,14 @@ "provider": "azure", "provider_reasoning_effort": "xhigh" }, - "normalized_score": 60.86642599300001, - "normalized_score_stddev": 22.118506348049813, + "normalized_score": 76.462093863, + "normalized_score_stddev": 12.754467163779541, "provider_reasoning_effort": "xhigh", - "raw_score": 647.2, + "raw_score": 733.6, "raw_score_imputation_count": 0, - "raw_score_stddev": 122.53652516698847, + "raw_score_stddev": 70.65974808899335, "source_run_ids": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/score.json index 28986ca..e24f0c9 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/score.json @@ -1,22 +1,22 @@ { "benchmark": "learning_beyond_gradients_article_suite", - "normalized_score": 60.86642599300001, - "normalized_score_stddev": 22.118506348049813, + "normalized_score": 76.462093863, + "normalized_score_stddev": 12.754467163779541, "raw_score_imputation_count": 0, "reference_score": 864.0, - "score": 647.2, - "score_stddev": 122.53652516698847, + "score": 733.6, + "score_stddev": 70.65974808899335, "starter_score": 310.0, "task": "simulation_heuristics_breakout_rgb_v1", "trial_count": 5, "trials": [ { - "normalized_score": 21.299638989, - "observed_raw_score": 428.0, - "raw_score": 428.0, + "normalized_score": 99.277978339, + "observed_raw_score": 860.0, + "raw_score": 860.0, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "trial": 1 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json index a29fd10..97300e5 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json @@ -1,31 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, - "finished_at": 1784074045.3527198, + "finished_at": 1784138408.201535, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.5", "id": "gpt-5.5", "model": "azure/gpt-5.5", "provider": "azure", "provider_reasoning_effort": "xhigh" }, - "normalized_score": 21.299638989, - "observed_raw_score": 428.0, + "normalized_score": 99.277978339, + "observed_raw_score": 860.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", - "raw_score": 428.0, + "published_protocol_version": "2.2", + "raw_score": 860.0, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 360.1, + "agent_setup": 3.1, + "environment_setup": 2.3, + "total": 496.1, + "verifier": 105.4 + }, "token_usage": { - "final_input_tokens": 1288.0, - "final_output_tokens": 168.0, - "input_tokens": 1288.0, - "output_tokens": 168.0, - "total_tokens": 17840.0 + "final_input_tokens": 944.0, + "final_output_tokens": 224.0, + "input_tokens": 944.0, + "output_tokens": 224.0, + "total_tokens": 17552.0 }, - "tool_calls": 21, - "trial": 1 + "tool_calls": 20, + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/score.json index 2259765..e5f6c0e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-01/score.json @@ -1,7 +1,7 @@ { - "normalized_score": 21.299638989, + "normalized_score": 99.277978339, "reference_score": 864.0, - "score": 428.0, + "score": 860.0, "starter_score": 310.0, "suites": { "hidden_nominal": { @@ -10,11 +10,11 @@ { "final_lives": 0, "invalid_action": false, - "length": 14382, - "mean_action_latency_ms": 0.4408362879364866, + "length": 21521, + "mean_action_latency_ms": 0.4799090789567914, "observation_mode": "rgb", "policy_error": null, - "return": 428.0, + "return": 860.0, "seed": 101, "terminated": true, "truncated": false, @@ -23,14 +23,14 @@ } ], "invalid_episode_rate": 0.0, - "max_return": 428.0, + "max_return": 860.0, "max_steps": 30000, - "mean_action_latency_ms": 0.4408362879364866, - "mean_return": 428.0, - "min_return": 428.0, + "mean_action_latency_ms": 0.4799090789567914, + "mean_return": 860.0, + "min_return": 860.0, "observation_mode": "rgb", "policy_path": "submitted_artifact", - "score": 428.0 + "score": 860.0 }, "hidden_shifted_start": { "completion_rate": 0.0, @@ -38,11 +38,11 @@ { "final_lives": 0, "invalid_action": false, - "length": 14376, - "mean_action_latency_ms": 0.4348243627143722, + "length": 21515, + "mean_action_latency_ms": 0.49106069311197453, "observation_mode": "rgb", "policy_error": null, - "return": 428.0, + "return": 860.0, "seed": 211, "terminated": true, "truncated": false, @@ -51,16 +51,16 @@ } ], "invalid_episode_rate": 0.0, - "max_return": 428.0, + "max_return": 860.0, "max_steps": 30000, - "mean_action_latency_ms": 0.4348243627143722, - "mean_return": 428.0, - "min_return": 428.0, + "mean_action_latency_ms": 0.49106069311197453, + "mean_return": 860.0, + "min_return": 860.0, "observation_mode": "rgb", "policy_path": "submitted_artifact", - "score": 428.0 + "score": 860.0 } }, - "target_gap": -436.0, + "target_gap": -4.0, "target_score": 864.0 } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json index 4305d0f..fca6929 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 182.3, + "agent_setup": 3.2, + "environment_setup": 3.4, + "total": 312.5, + "verifier": 96.8 + }, "token_usage": { "final_input_tokens": 1446.0, "final_output_tokens": 195.0, @@ -27,5 +45,7 @@ "total_tokens": 19049.0 }, "tool_calls": 24, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json index 50da221..559c338 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 328.4, + "agent_setup": 5.2, + "environment_setup": 3.5, + "total": 509.6, + "verifier": 129.7 + }, "token_usage": { "final_input_tokens": 871.0, "final_output_tokens": 214.0, @@ -27,5 +45,7 @@ "total_tokens": 17981.0 }, "tool_calls": 22, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json index 3ff7d6c..2c1f026 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,13 +16,30 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 210.1, + "agent_setup": 4.2, + "environment_setup": 4.2, + "total": 387.7, + "verifier": 129.0 + }, "token_usage": null, "tool_calls": 27, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json index 670c72d..aaece3f 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,13 +16,30 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 330.1, + "agent_setup": 4.6, + "environment_setup": 5.5, + "total": 508.5, + "verifier": 127.4 + }, "token_usage": null, "tool_calls": 17, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json index a09a064..a497fb0 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784076184.464024, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 4001.2886158917113, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 4001.2886158917113, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_halfcheetah_v1", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.5, + "agent_setup": 3.3, + "environment_setup": 2.3, + "total": 2065.5, + "verifier": 1974.3 + }, "token_usage": { "final_input_tokens": 1006.0, "final_output_tokens": 269.0, @@ -27,5 +45,7 @@ "total_tokens": 17659.0 }, "tool_calls": 16, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json index 343d4a5..bc15c79 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 21.272433217, "observed_raw_score": 5593.736124362126, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 5593.736124362126, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 240.0, + "agent_setup": 3.4, + "environment_setup": 12.1, + "total": 2358.5, + "verifier": 2076.3 + }, "token_usage": { "final_input_tokens": 1301.0, "final_output_tokens": 237.0, @@ -27,5 +45,7 @@ "total_tokens": 18946.0 }, "tool_calls": 19, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json index acad8e3..c7a52e0 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 12.773449459, "observed_raw_score": 4957.504933873245, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 4957.504933873245, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 780.1, + "agent_setup": 5.8, + "environment_setup": 2.9, + "total": 3567.1, + "verifier": 2751.4 + }, "token_usage": { "final_input_tokens": 1235.0, "final_output_tokens": 302.0, @@ -27,5 +45,7 @@ "total_tokens": 23553.0 }, "tool_calls": 27, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json index 3d078f1..dc13467 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784098096.514206, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 13.798807871, "observed_raw_score": 5034.262934783822, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 5034.262934783822, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e6fa3546aa", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 180.0, + "agent_setup": 3.0, + "environment_setup": 38.2, + "total": 2441.4, + "verifier": 2198.1 + }, "token_usage": { "final_input_tokens": 574.0, "final_output_tokens": 244.0, @@ -27,5 +45,7 @@ "total_tokens": 16690.0 }, "tool_calls": 15, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json index 95e8c94..8755ac2 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784097871.7138891, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 4001.2886158917113, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 4001.2886158917113, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.1, + "agent_setup": 3.2, + "environment_setup": 2.7, + "total": 2070.0, + "verifier": 1952.0 + }, "token_usage": { "final_input_tokens": 1336.0, "final_output_tokens": 257.0, @@ -27,5 +45,7 @@ "total_tokens": 17977.0 }, "tool_calls": 17, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/metadata.json index 12dcf3c..875dc7b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/metadata.json @@ -19,7 +19,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "task": "simulation_heuristics_montezuma_v1", diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/score.json index 7057b63..01f551a 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/score.json @@ -43,7 +43,7 @@ "raw_score": 0.0, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "trial": 4 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-01/metadata.json index f711341..b9a3582 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784074045.3527198, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 3.5, + "environment_setup": 2.3, + "total": 170.8, + "verifier": 47.6 + }, "token_usage": { "final_input_tokens": 923.0, "final_output_tokens": 177.0, @@ -27,5 +45,7 @@ "total_tokens": 16460.0 }, "tool_calls": 22, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-02/metadata.json index a919819..8a7e6f6 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 60.0, + "agent_setup": 3.8, + "environment_setup": 2.5, + "total": 143.0, + "verifier": 47.7 + }, "token_usage": { "final_input_tokens": 1109.0, "final_output_tokens": 169.0, @@ -27,5 +45,7 @@ "total_tokens": 18174.0 }, "tool_calls": 20, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-03/metadata.json index 2101b42..aa24382 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 4.2, + "environment_setup": 75.2, + "total": 240.0, + "verifier": 42.2 + }, "token_usage": { "final_input_tokens": 1090.0, "final_output_tokens": 181.0, @@ -27,5 +45,7 @@ "total_tokens": 24311.0 }, "tool_calls": 23, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/metadata.json index 6cb78b2..71988f1 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/metadata.json @@ -1,10 +1,15 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, - "finished_at": 1784093477.189311, + "finished_at": 1784140206.649261, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.5", "id": "gpt-5.5", "model": "azure/gpt-5.5", @@ -13,19 +18,36 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 3.6, + "environment_setup": 3.5, + "total": 161.1, + "verifier": 35.1 + }, "token_usage": { - "final_input_tokens": 0.0, - "final_output_tokens": 0.0, - "input_tokens": 0.0, - "output_tokens": 0.0, - "total_tokens": 0.0 + "final_input_tokens": 1158.0, + "final_output_tokens": 171.0, + "input_tokens": 1158.0, + "output_tokens": 171.0, + "total_tokens": 17713.0 }, - "tool_calls": 9, - "trial": 4 + "tool_calls": 24, + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/score.json index a6d99c3..64fa474 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-04/score.json @@ -11,37 +11,37 @@ { "bootstrap_steps": 0, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.15601597448739296, + "length": 549, + "mean_action_latency_ms": 0.00748113315138552, "policy_error": null, - "policy_steps": 2600, + "policy_steps": 549, "pre_policy_noops": 0, "return": 0.0, "seed": 10007, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "canonical" }, { "bootstrap_steps": 0, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.11907760976240613, + "length": 549, + "mean_action_latency_ms": 0.006186613857148992, "policy_error": null, - "policy_steps": 2600, + "policy_steps": 549, "pre_policy_noops": 0, "return": 0.0, "seed": 10103, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "canonical" } ], "invalid_episode_rate": 0.0, "max_steps": 2600, - "mean_action_latency_ms": 0.13754679212489954, + "mean_action_latency_ms": 0.006833873504267255, "mean_return": 0.0, "policy_path": "submitted_artifact", "recovery_success_rate": 0.0, @@ -59,37 +59,37 @@ { "bootstrap_steps": 0, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.16236112535166244, + "length": 553, + "mean_action_latency_ms": 0.006268672560426322, "policy_error": null, - "policy_steps": 2597, + "policy_steps": 550, "pre_policy_noops": 3, "return": 0.0, "seed": 10001, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "startup_noop_3" }, { "bootstrap_steps": 0, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.14516854209441635, + "length": 537, + "mean_action_latency_ms": 0.007700968626175673, "policy_error": null, - "policy_steps": 2587, + "policy_steps": 524, "pre_policy_noops": 13, "return": 0.0, "seed": 10001, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "startup_noop_13" } ], "invalid_episode_rate": 0.0, "max_steps": 2600, - "mean_action_latency_ms": 0.1537648337230394, + "mean_action_latency_ms": 0.006984820593300998, "mean_return": 0.0, "policy_path": "submitted_artifact", "recovery_success_rate": 0.0, @@ -107,67 +107,67 @@ { "bootstrap_steps": 512, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.0939244203780726, + "length": 1065, + "mean_action_latency_ms": 0.007438455591273438, "policy_error": null, - "policy_steps": 2084, + "policy_steps": 549, "pre_policy_noops": 4, "return": 0.0, "seed": 10001, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "recover_512_noop_4" }, { "bootstrap_steps": 512, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.09359156258422165, + "length": 1081, + "mean_action_latency_ms": 0.0065877919920359915, "policy_error": null, - "policy_steps": 2080, + "policy_steps": 561, "pre_policy_noops": 8, "return": 0.0, "seed": 10001, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "recover_512_noop_8" }, { "bootstrap_steps": 768, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.1827507849803062, + "length": 1269, + "mean_action_latency_ms": 0.008120874357307125, "policy_error": null, - "policy_steps": 1830, + "policy_steps": 499, "pre_policy_noops": 2, "return": 0.0, "seed": 10001, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "recover_768_noop_2" }, { "bootstrap_steps": 900, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.09532441782272037, + "length": 1341, + "mean_action_latency_ms": 0.01008263493459376, "policy_error": null, - "policy_steps": 1700, + "policy_steps": 441, "pre_policy_noops": 0, "return": 0.0, "seed": 10001, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "recover_900" } ], "invalid_episode_rate": 0.0, "max_steps": 2600, - "mean_action_latency_ms": 0.1163977964413302, + "mean_action_latency_ms": 0.00805743921880258, "mean_return": 0.0, "policy_path": "submitted_artifact", "recovery_success_rate": 0.0, @@ -185,22 +185,22 @@ { "bootstrap_steps": 0, "invalid_action": false, - "length": 2600, - "mean_action_latency_ms": 0.1588235599382852, + "length": 549, + "mean_action_latency_ms": 0.0074930138757048625, "policy_error": null, - "policy_steps": 2600, + "policy_steps": 549, "pre_policy_noops": 0, "return": 0.0, "seed": 10001, "target_reached": false, - "terminated": false, - "truncated": true, + "terminated": true, + "truncated": false, "variant": "canonical" } ], "invalid_episode_rate": 0.0, "max_steps": 2600, - "mean_action_latency_ms": 0.1588235599382852, + "mean_action_latency_ms": 0.0074930138757048625, "mean_return": 0.0, "policy_path": "submitted_artifact", "recovery_success_rate": 0.0, diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-05/metadata.json index 95cd0eb..2d7fd17 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_montezuma_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,13 +16,30 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.1, + "agent_setup": 3.3, + "environment_setup": 2.8, + "total": 228.2, + "verifier": 45.2 + }, "token_usage": null, "tool_calls": 40, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/metadata.json index 45d5555..e8704c1 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/metadata.json @@ -9,17 +9,17 @@ "provider": "azure", "provider_reasoning_effort": "xhigh" }, - "normalized_score": 49.1666666668, - "normalized_score_stddev": 40.44028519521613, + "normalized_score": 54.166666666800005, + "normalized_score_stddev": 38.188130791325946, "provider_reasoning_effort": "xhigh", - "raw_score": 8.8, + "raw_score": 10.0, "raw_score_imputation_count": 0, - "raw_score_stddev": 9.705668446840743, + "raw_score_stddev": 9.16515138991168, "source_run_ids": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "task": "simulation_heuristics_pong_ram_v1", diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/score.json index 0dd946e..e91bec7 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/score.json @@ -1,11 +1,11 @@ { "benchmark": "learning_beyond_gradients_article_suite", - "normalized_score": 49.1666666668, - "normalized_score_stddev": 40.44028519521613, + "normalized_score": 54.166666666800005, + "normalized_score_stddev": 38.188130791325946, "raw_score_imputation_count": 0, "reference_score": 21.0, - "score": 8.8, - "score_stddev": 9.705668446840743, + "score": 10.0, + "score_stddev": 9.16515138991168, "starter_score": -2.9999999999999996, "task": "simulation_heuristics_pong_ram_v1", "trial_count": 5, @@ -38,12 +38,12 @@ "trial": 3 }, { - "normalized_score": 25.0, - "observed_raw_score": 2.9999999999999996, - "raw_score": 2.9999999999999996, + "normalized_score": 50.0, + "observed_raw_score": 9.0, + "raw_score": 9.0, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "trial": 4 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-01/metadata.json index 192bbf0..c25c977 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784074045.3527198, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 41.666666667, "observed_raw_score": 7.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 7.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 120.0, + "agent_setup": 3.0, + "environment_setup": 2.1, + "total": 214.2, + "verifier": 66.8 + }, "token_usage": { "final_input_tokens": 2559.0, "final_output_tokens": 157.0, @@ -27,5 +45,7 @@ "total_tokens": 14492.0 }, "tool_calls": 16, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-02/metadata.json index c611355..f557745 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 79.166666667, "observed_raw_score": 16.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 16.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 330.1, + "agent_setup": 3.3, + "environment_setup": 5.0, + "total": 431.6, + "verifier": 65.3 + }, "token_usage": { "final_input_tokens": 777.0, "final_output_tokens": 255.0, @@ -27,5 +45,7 @@ "total_tokens": 17416.0 }, "tool_calls": 16, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-03/metadata.json index 39edb92..38c9ef6 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 180.1, + "agent_setup": 4.9, + "environment_setup": 6.5, + "total": 278.1, + "verifier": 60.2 + }, "token_usage": { "final_input_tokens": 1704.0, "final_output_tokens": 239.0, @@ -27,5 +45,7 @@ "total_tokens": 17303.0 }, "tool_calls": 19, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/metadata.json index 1199610..b8f4b95 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/metadata.json @@ -1,25 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, - "finished_at": 1784095422.450964, + "finished_at": 1784140206.649261, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.5", "id": "gpt-5.5", "model": "azure/gpt-5.5", "provider": "azure", "provider_reasoning_effort": "xhigh" }, - "normalized_score": 25.0, - "observed_raw_score": 2.9999999999999996, + "normalized_score": 50.0, + "observed_raw_score": 9.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", - "raw_score": 2.9999999999999996, + "published_protocol_version": "2.2", + "raw_score": 9.0, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", - "token_usage": null, - "tool_calls": 22, - "trial": 4 + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.1, + "agent_setup": 3.1, + "environment_setup": 2.5, + "total": 242.7, + "verifier": 63.5 + }, + "token_usage": { + "final_input_tokens": 907.0, + "final_output_tokens": 259.0, + "input_tokens": 907.0, + "output_tokens": 259.0, + "total_tokens": 19086.0 + }, + "tool_calls": 23, + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/score.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/score.json index 3d4bf6c..3e4cf1a 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-04/score.json @@ -1,7 +1,7 @@ { - "normalized_score": 25.0, + "normalized_score": 50.0, "reference_score": 21.0, - "score": 2.9999999999999996, + "score": 9.0, "starter_score": -2.9999999999999996, "suite_weights": { "hidden_nominal": 0.7, @@ -13,11 +13,11 @@ { "invalid_action": false, "length": 27000, - "mean_action_latency_ms": 0.003091757802327198, - "points_against": 10, - "points_for": 13, + "mean_action_latency_ms": 0.003342468515728359, + "points_against": 0, + "points_for": 9, "policy_error": null, - "score": 3.0, + "score": 9.0, "seed": 101, "target_reached": false, "terminated": false, @@ -28,11 +28,11 @@ { "invalid_action": false, "length": 27000, - "mean_action_latency_ms": 0.003134694601478124, - "points_against": 10, - "points_for": 13, + "mean_action_latency_ms": 0.0030724480317008717, + "points_against": 0, + "points_for": 9, "policy_error": null, - "score": 3.0, + "score": 9.0, "seed": 211, "target_reached": false, "terminated": false, @@ -43,11 +43,11 @@ { "invalid_action": false, "length": 27000, - "mean_action_latency_ms": 0.0029207480622938384, - "points_against": 10, - "points_for": 13, + "mean_action_latency_ms": 0.0031461192346695396, + "points_against": 0, + "points_for": 9, "policy_error": null, - "score": 3.0, + "score": 9.0, "seed": 307, "target_reached": false, "terminated": false, @@ -57,13 +57,13 @@ } ], "invalid_episode_rate": 0.0, - "max_score": 3.0, + "max_score": 9.0, "max_steps": 27000, - "mean_action_latency_ms": 0.0030490668220330536, - "mean_score": 3.0, - "min_score": 3.0, + "mean_action_latency_ms": 0.0031870119273662567, + "mean_score": 9.0, + "min_score": 9.0, "policy_path": "submitted_artifact", - "score": 3.0, + "score": 9.0, "target_score": 21.0, "target_score_rate": 0.0, "win_rate": 1.0 @@ -73,11 +73,11 @@ { "invalid_action": false, "length": 27000, - "mean_action_latency_ms": 0.0028929879794035245, - "points_against": 10, - "points_for": 13, + "mean_action_latency_ms": 0.003289461705005831, + "points_against": 0, + "points_for": 9, "policy_error": null, - "score": 3.0, + "score": 9.0, "seed": 503, "target_reached": false, "terminated": false, @@ -88,11 +88,11 @@ { "invalid_action": false, "length": 27000, - "mean_action_latency_ms": 0.003008703041511277, - "points_against": 10, - "points_for": 13, + "mean_action_latency_ms": 0.002842728514224291, + "points_against": 0, + "points_for": 9, "policy_error": null, - "score": 3.0, + "score": 9.0, "seed": 601, "target_reached": false, "terminated": false, @@ -103,11 +103,11 @@ { "invalid_action": false, "length": 27000, - "mean_action_latency_ms": 0.0030335993031729703, - "points_against": 10, - "points_for": 13, + "mean_action_latency_ms": 0.003087215336833011, + "points_against": 0, + "points_for": 9, "policy_error": null, - "score": 3.0, + "score": 9.0, "seed": 701, "target_reached": false, "terminated": false, @@ -117,13 +117,13 @@ } ], "invalid_episode_rate": 0.0, - "max_score": 3.0, + "max_score": 9.0, "max_steps": 27000, - "mean_action_latency_ms": 0.0029784301080292576, - "mean_score": 3.0, - "min_score": 3.0, + "mean_action_latency_ms": 0.0030731351853543775, + "mean_score": 9.0, + "min_score": 9.0, "policy_path": "submitted_artifact", - "score": 3.0, + "score": 9.0, "target_score": 21.0, "target_score_rate": 0.0, "win_rate": 1.0 diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-05/metadata.json index 2535570..ae4b731 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_pong_ram_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,13 +16,30 @@ }, "normalized_score": 0.0, "observed_raw_score": -2.9999999999999996, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -2.9999999999999996, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 120.0, + "agent_setup": 3.3, + "environment_setup": 2.7, + "total": 208.5, + "verifier": 60.2 + }, "token_usage": null, "tool_calls": 20, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json index 76be973..f17bbf9 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784074045.3527198, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": -117.814998349, "observed_raw_score": -0.0016000167280435552, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": -0.0016000167280435552, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 120.0, + "agent_setup": 4.5, + "environment_setup": 3.1, + "total": 204.2, + "verifier": 36.1 + }, "token_usage": { "final_input_tokens": 1573.0, "final_output_tokens": 195.0, @@ -27,5 +45,7 @@ "total_tokens": 19176.0 }, "tool_calls": 19, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json index f7ad2c1..3d3f539 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 85.486445859, "observed_raw_score": 0.7632199863344431, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.7632199863344431, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 122.9, + "agent_setup": 5.1, + "environment_setup": 3.5, + "total": 220.5, + "verifier": 43.7 + }, "token_usage": { "final_input_tokens": 1510.0, "final_output_tokens": 193.0, @@ -27,5 +45,7 @@ "total_tokens": 18087.0 }, "tool_calls": 18, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json index 6f6efff..dbb041b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 104.662416785, "observed_raw_score": 0.8353599861264229, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.8353599861264229, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.0, + "agent_setup": 6.0, + "environment_setup": 5.3, + "total": 189.4, + "verifier": 44.3 + }, "token_usage": { "final_input_tokens": 1355.0, "final_output_tokens": 198.0, @@ -27,5 +45,7 @@ "total_tokens": 15377.0 }, "tool_calls": 14, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json index fd858fa..6e4281f 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784093477.189311, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 111.839447567, "observed_raw_score": 0.8623599748685956, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.8623599748685956, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 442.7, + "agent_setup": 6.6, + "environment_setup": 8.9, + "total": 4171.5, + "verifier": 45.0 + }, "token_usage": { "final_input_tokens": 434.0, "final_output_tokens": 192.0, @@ -27,5 +45,7 @@ "total_tokens": 17010.0 }, "tool_calls": 18, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json index 6108b07..5a5cee0 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,13 +16,30 @@ }, "normalized_score": 52.519933736, "observed_raw_score": 0.6391999725997448, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 0.6391999725997448, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.0, + "agent_setup": 4.5, + "environment_setup": 3.0, + "total": 243.7, + "verifier": 45.4 + }, "token_usage": null, "tool_calls": 22, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json index aa559d9..2ee1e99 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784074045.3527198, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 22.326203209, "observed_raw_score": 98.69999999999999, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 98.69999999999999, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 270.5, + "agent_setup": 3.7, + "environment_setup": 2.5, + "total": 354.8, + "verifier": 49.1 + }, "token_usage": { "final_input_tokens": 2072.0, "final_output_tokens": 198.0, @@ -27,5 +45,7 @@ "total_tokens": 23774.0 }, "tool_calls": 18, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json index 6122ae2..3333681 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784076544.549837, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 94.585561497, "observed_raw_score": 314.9, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 314.9, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 210.1, + "agent_setup": 3.7, + "environment_setup": 2.5, + "total": 308.5, + "verifier": 62.8 + }, "token_usage": { "final_input_tokens": 1955.0, "final_output_tokens": 215.0, @@ -27,5 +45,7 @@ "total_tokens": 19066.0 }, "tool_calls": 14, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json index 33ef5ba..4e7918b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784080509.825609, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": -0.601604278, "observed_raw_score": 30.099999999999998, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 30.099999999999998, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 180.0, + "agent_setup": 3.8, + "environment_setup": 2.6, + "total": 253.6, + "verifier": 37.2 + }, "token_usage": { "final_input_tokens": 2152.0, "final_output_tokens": 211.0, @@ -27,5 +45,7 @@ "total_tokens": 24379.0 }, "tool_calls": 20, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json index f42a7c6..c1b6e26 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784093477.189311, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 31.9, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 31.9, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 300.1, + "agent_setup": 3.5, + "environment_setup": 2.3, + "total": 371.9, + "verifier": 38.6 + }, "token_usage": { "final_input_tokens": 1882.0, "final_output_tokens": 131.0, @@ -27,5 +45,7 @@ "total_tokens": 25053.0 }, "tool_calls": 19, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json index a89ac3d..b2777cc 100644 --- a/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.5/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.5", "id": "gpt-5.5", @@ -13,13 +16,30 @@ }, "normalized_score": 0.0, "observed_raw_score": 31.9, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "xhigh", + "published_protocol_version": "2.2", "raw_score": 31.9, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 153.2, + "agent_setup": 3.4, + "environment_setup": 2.6, + "total": 223.1, + "verifier": 35.7 + }, "token_usage": null, "tool_calls": 16, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/metadata.json index f674300..b783e1f 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/metadata.json @@ -9,15 +9,15 @@ "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": -4.9009087386, - "normalized_score_stddev": 11.991465810903719, + "normalized_score": -6.081625694, + "normalized_score_stddev": 12.597925216329989, "provider_reasoning_effort": "max", - "raw_score": 2272.356693724495, + "raw_score": 2231.0491055302227, "raw_score_imputation_count": 0, - "raw_score_stddev": 419.52351859557746, + "raw_score_stddev": 440.7406064520731, "source_run_ids": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/score.json index 61ed20f..940595e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/score.json @@ -1,11 +1,11 @@ { "benchmark": "learning_beyond_gradients_article_suite", - "normalized_score": -4.9009087386, - "normalized_score_stddev": 11.991465810903719, + "normalized_score": -6.081625694, + "normalized_score_stddev": 12.597925216329989, "raw_score_imputation_count": 0, "reference_score": 5942.333244755838, - "score": 2272.356693724495, - "score_stddev": 419.52351859557746, + "score": 2231.0491055302227, + "score_stddev": 440.7406064520731, "starter_score": 2443.815838992282, "task": "simulation_heuristics_ant_v1", "trial_count": 5, @@ -20,12 +20,12 @@ "trial": 1 }, { - "normalized_score": -7.591479, - "observed_raw_score": 2178.226624807645, - "raw_score": 2178.226624807645, + "normalized_score": -13.495063777, + "observed_raw_score": 1971.688683836282, + "raw_score": 1971.688683836282, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "trial": 2 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-01/metadata.json index 25ac64b..6e2262a 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784075101.71677, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 9.173888471, "observed_raw_score": 2764.765923919426, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 2764.765923919426, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 425.6, + "agent_setup": 3.3, + "environment_setup": 34.3, + "total": 4667.2, + "verifier": 4179.6 + }, "token_usage": { "final_input_tokens": 374.0, "final_output_tokens": 210.0, @@ -27,5 +45,7 @@ "total_tokens": 21576.0 }, "tool_calls": 27, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/metadata.json index ceaec6a..ec1babf 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/metadata.json @@ -1,31 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, - "finished_at": 1784078883.680485, + "finished_at": 1784142166.67042, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", "model": "azure/gpt-5.6-sol", "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": -7.591479, - "observed_raw_score": 2178.226624807645, + "normalized_score": -13.495063777, + "observed_raw_score": 1971.688683836282, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", - "raw_score": 2178.226624807645, + "published_protocol_version": "2.2", + "raw_score": 1971.688683836282, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1452.7, + "agent_setup": 3.2, + "environment_setup": 2.3, + "total": 4253.2, + "verifier": 2768.5 + }, "token_usage": { - "final_input_tokens": 539.0, - "final_output_tokens": 172.0, - "input_tokens": 539.0, - "output_tokens": 172.0, - "total_tokens": 38087.0 + "final_input_tokens": 467.0, + "final_output_tokens": 223.0, + "input_tokens": 467.0, + "output_tokens": 223.0, + "total_tokens": 31410.0 }, - "tool_calls": 37, - "trial": 2 + "tool_calls": 39, + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/score.json index 0a2274f..b587333 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-02/score.json @@ -4,66 +4,66 @@ { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.02975405380129814, + "mean_action_latency_ms": 121.38340470008552, "policy_error": null, - "return": 2000.6286232348173, + "return": 2434.700543256514, "seed": 101, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 53.80113931757859, - "x_velocity": 2.1311493378351543 + "x_position": 78.69224638995453, + "x_velocity": -0.11651200505440329 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.028919238597154617, + "mean_action_latency_ms": 92.90469318255782, "policy_error": null, - "return": 2569.3091871731217, + "return": 1721.0506182539702, "seed": 211, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 82.17737194908375, - "x_velocity": 2.354063933416626 + "x_position": 41.950037975835265, + "x_velocity": -0.01165465803836696 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.03253542631864548, + "mean_action_latency_ms": 82.84715152718127, "policy_error": null, - "return": 2385.6946277480392, + "return": 1274.243078859664, "seed": 307, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 73.04151306245869, - "x_velocity": 1.1981546903319895 + "x_position": 19.299807645061282, + "x_velocity": 0.13435547853099195 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.03688942268490791, + "mean_action_latency_ms": 105.26210810802877, "policy_error": null, - "return": 2368.655642993632, + "return": 1839.658259625324, "seed": 401, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 72.19982332507476, - "x_velocity": 1.1704044847220985 + "x_position": 48.0578512776704, + "x_velocity": 0.1615724828003806 } ], "fall_rate": 0.0, "invalid_episode_rate": 0.0, - "max_return": 2569.3091871731217, + "max_return": 2434.700543256514, "max_steps": 1000, - "mean_action_latency_ms": 0.03202453535050154, - "mean_return": 2331.0720202874027, - "mean_x_position": 70.30496191354895, - "min_return": 2000.6286232348173, + "mean_action_latency_ms": 100.59933937946334, + "mean_return": 1817.413124998868, + "mean_x_position": 46.99998582213038, + "min_return": 1274.243078859664, "policy_path": "submitted_artifact", - "score": 2331.0720202874027 + "score": 1817.413124998868 }, "hidden_nominal_weight": 0.7, "hidden_robustness": { @@ -71,71 +71,71 @@ { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.02802887186408043, + "mean_action_latency_ms": 136.40258463472128, "policy_error": null, - "return": 2798.32044246107, + "return": 2735.341707835714, "seed": 503, "terminated": false, "truncated": true, "variant": "light_low_friction", - "x_position": 93.54063165430142, - "x_velocity": 1.2473198151258202 + "x_position": 93.81933784511419, + "x_velocity": -0.0518694415265486 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.03125184401869774, + "mean_action_latency_ms": 82.31060975603759, "policy_error": null, - "return": 358.1267611570017, + "return": 1262.6491013540442, "seed": 601, "terminated": false, "truncated": true, "variant": "light_low_friction", - "x_position": -28.309368557338463, - "x_velocity": -1.156389118354042 + "x_position": 18.604123447729453, + "x_velocity": -0.12545521329045073 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.027832284569740295, + "mean_action_latency_ms": 160.63754370249808, "policy_error": null, - "return": 1812.2106051749915, + "return": 3548.3174197337967, "seed": 503, "terminated": false, "truncated": true, "variant": "heavy_weak_actuators", - "x_position": 44.58481769343802, - "x_velocity": -1.9142496670555431 + "x_position": 135.9268991594252, + "x_velocity": 1.7938654634463091 }, { "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 0.034463025629520416, + "mean_action_latency_ms": 103.39761203154922, "policy_error": null, - "return": 2317.6916659597805, + "return": 1780.3517222374426, "seed": 601, "terminated": false, "truncated": true, "variant": "heavy_weak_actuators", - "x_position": 69.94711534445288, - "x_velocity": 0.8705855141025154 + "x_position": 45.547677151747905, + "x_velocity": 0.34760196226713447 } ], "fall_rate": 0.0, "invalid_episode_rate": 0.0, - "max_return": 2798.32044246107, + "max_return": 3548.3174197337967, "max_steps": 1000, - "mean_action_latency_ms": 0.03039400652050972, - "mean_return": 1821.587368688211, - "mean_x_position": 44.94079903371346, - "min_return": 358.1267611570017, + "mean_action_latency_ms": 120.68708753120154, + "mean_return": 2331.664987790249, + "mean_x_position": 73.47450940100418, + "min_return": 1262.6491013540442, "policy_path": "submitted_artifact", - "score": 1821.587368688211 + "score": 2331.664987790249 }, "hidden_robustness_weight": 0.3, - "normalized_score": -7.591479, + "normalized_score": -13.495063777, "reference_score": 5942.333244755838, - "score": 2178.226624807645, + "score": 1971.688683836282, "starter_score": 2443.815838992282, "unique_policy_evaluations": 3 } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-03/metadata.json index dcfa35e..22e9fc7 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784092762.840873, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": -11.713338376, "observed_raw_score": 2034.0226571116652, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 2034.0226571116652, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 391.1, + "agent_setup": 5.1, + "environment_setup": 4.0, + "total": 2427.9, + "verifier": 1985.6 + }, "token_usage": { "final_input_tokens": 719.0, "final_output_tokens": 129.0, @@ -27,5 +45,7 @@ "total_tokens": 21840.0 }, "tool_calls": 34, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-04/metadata.json index 7db93be..67b9485 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784100567.131854, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": -19.696444908, "observed_raw_score": 1754.7322855694497, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 1754.7322855694497, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 360.1, + "agent_setup": 3.0, + "environment_setup": 4.2, + "total": 3314.1, + "verifier": 2923.1 + }, "token_usage": { "final_input_tokens": 601.0, "final_output_tokens": 185.0, @@ -27,5 +45,7 @@ "total_tokens": 22290.0 }, "tool_calls": 28, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-05/metadata.json index dac34cf..c9323aa 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_ant_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 5.32283012, "observed_raw_score": 2630.03597721429, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 2630.03597721429, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_ant_v1", "task_digest": "sha256:c338003eab742d732cfd8b1cb13fe3d6a5b8fdd6f58842f6ba2991314d6ab974", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 361.4, + "agent_setup": 5.0, + "environment_setup": 12.2, + "total": 2403.1, + "verifier": 1984.8 + }, "token_usage": { "final_input_tokens": 399.0, "final_output_tokens": 146.0, @@ -27,5 +45,7 @@ "total_tokens": 32289.0 }, "tool_calls": 44, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-01/metadata.json index e91beb3..cf975f1 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784075101.71677, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 213.2, + "agent_setup": 3.0, + "environment_setup": 3.3, + "total": 257.9, + "verifier": 15.0 + }, "token_usage": { "final_input_tokens": 708.0, "final_output_tokens": 204.0, @@ -27,5 +45,7 @@ "total_tokens": 45456.0 }, "tool_calls": 29, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-02/metadata.json index cc2f047..f67c973 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784078883.680485, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.2, + "agent_setup": 3.0, + "environment_setup": 5.1, + "total": 196.1, + "verifier": 11.9 + }, "token_usage": { "final_input_tokens": 805.0, "final_output_tokens": 197.0, @@ -27,5 +45,7 @@ "total_tokens": 28138.0 }, "tool_calls": 30, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-03/metadata.json index 179e322..f24a194 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784092762.840873, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 212.3, + "agent_setup": 3.1, + "environment_setup": 2.9, + "total": 262.2, + "verifier": 13.5 + }, "token_usage": { "final_input_tokens": 603.0, "final_output_tokens": 219.0, @@ -27,5 +45,7 @@ "total_tokens": 33590.0 }, "tool_calls": 25, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-04/metadata.json index d2f0725..6d53d92 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,13 +16,30 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 122.5, + "agent_setup": 3.0, + "environment_setup": 2.7, + "total": 159.1, + "verifier": 10.0 + }, "token_usage": null, "tool_calls": 29, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-05/metadata.json index 823fee7..8710df7 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_atari57_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 1814400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_atari57_v1", "task_digest": "sha256:b3b516b56ca8c457a9abd3b7628d00741db0f22dc261cb174df008fafad35b8c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 210.9, + "agent_setup": 2.8, + "environment_setup": 2.9, + "total": 248.4, + "verifier": 10.7 + }, "token_usage": { "final_input_tokens": 806.0, "final_output_tokens": 202.0, @@ -27,5 +45,7 @@ "total_tokens": 32752.0 }, "tool_calls": 25, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/metadata.json index ff610e7..8548c8c 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/metadata.json @@ -9,17 +9,17 @@ "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 43.899371069400004, - "normalized_score_stddev": 47.34804435501101, + "normalized_score": 60.209643606, + "normalized_score_stddev": 50.34075134460858, "provider_reasoning_effort": "max", - "raw_score": 596.4, + "raw_score": 674.2, "raw_score_imputation_count": 0, - "raw_score_stddev": 225.8501715739884, + "raw_score_stddev": 240.1253839143209, "source_run_ids": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "task": "simulation_heuristics_breakout_ram_v1", diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/score.json index b3465d4..06a7c25 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/score.json @@ -1,11 +1,11 @@ { "benchmark": "learning_beyond_gradients_article_suite", - "normalized_score": 43.899371069400004, - "normalized_score_stddev": 47.34804435501101, + "normalized_score": 60.209643606, + "normalized_score_stddev": 50.34075134460858, "raw_score_imputation_count": 0, "reference_score": 864.0, - "score": 596.4, - "score_stddev": 225.8501715739884, + "score": 674.2, + "score_stddev": 240.1253839143209, "starter_score": 387.0, "task": "simulation_heuristics_breakout_ram_v1", "trial_count": 5, @@ -38,12 +38,12 @@ "trial": 3 }, { - "normalized_score": 18.448637317, - "observed_raw_score": 475.0, - "raw_score": 475.0, + "normalized_score": 100.0, + "observed_raw_score": 864.0, + "raw_score": 864.0, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "trial": 4 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json index 896bd23..dbd6a6d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784075101.71677, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 864.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 864.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 514.2, + "agent_setup": 3.0, + "environment_setup": 34.8, + "total": 712.0, + "verifier": 136.8 + }, "token_usage": { "final_input_tokens": 623.0, "final_output_tokens": 154.0, @@ -27,5 +45,7 @@ "total_tokens": 43785.0 }, "tool_calls": 59, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json index 054b318..66f1739 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784078883.680485, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 6.918238994, "observed_raw_score": 420.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 420.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 120.1, + "agent_setup": 3.1, + "environment_setup": 2.7, + "total": 253.7, + "verifier": 104.9 + }, "token_usage": { "final_input_tokens": 388.0, "final_output_tokens": 117.0, @@ -27,5 +45,7 @@ "total_tokens": 23545.0 }, "tool_calls": 39, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json index 8198f19..597b94e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784092762.840873, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 3.5639413, "observed_raw_score": 404.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 404.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 270.9, + "agent_setup": 3.6, + "environment_setup": 7.1, + "total": 475.7, + "verifier": 167.9 + }, "token_usage": { "final_input_tokens": 470.0, "final_output_tokens": 139.0, @@ -27,5 +45,7 @@ "total_tokens": 37473.0 }, "tool_calls": 64, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json index 0123d69..9b433db 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/metadata.json @@ -1,25 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, - "finished_at": 1784095422.450964, + "finished_at": 1784138571.705982, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", "model": "azure/gpt-5.6-sol", "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 18.448637317, - "observed_raw_score": 475.0, + "normalized_score": 100.0, + "observed_raw_score": 864.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", - "raw_score": 475.0, + "published_protocol_version": "2.2", + "raw_score": 864.0, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", - "token_usage": null, - "tool_calls": 95, - "trial": 4 + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 429.7, + "agent_setup": 3.5, + "environment_setup": 17.1, + "total": 655.2, + "verifier": 175.6 + }, + "token_usage": { + "final_input_tokens": 395.0, + "final_output_tokens": 173.0, + "input_tokens": 395.0, + "output_tokens": 173.0, + "total_tokens": 43576.0 + }, + "tool_calls": 42, + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/score.json index c802e30..8f51b6e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-04/score.json @@ -1,80 +1,80 @@ { - "normalized_score": 18.448637317, + "normalized_score": 100.0, "reference_score": 864.0, - "score": 475.0, + "score": 864.0, "starter_score": 387.0, "suites": { "hidden_nominal": { - "completion_rate": 0.0, + "completion_rate": 1.0, "episodes": [ { - "final_lives": 0, + "final_lives": 4, "invalid_action": false, - "length": 23884, - "mean_action_latency_ms": 0.005037961625969005, + "length": 108000, + "mean_action_latency_ms": 0.012770433841204201, "observation_mode": "ram", "policy_error": null, - "return": 475.0, + "return": 864.0, "seed": 101, - "terminated": true, - "truncated": false, + "terminated": false, + "truncated": true, "variant": "nominal", "warmup_steps": 0 } ], "invalid_episode_rate": 0.0, - "max_return": 475.0, + "max_return": 864.0, "max_steps": 108000, - "mean_action_latency_ms": 0.005037961625969005, - "mean_return": 475.0, - "min_return": 475.0, + "mean_action_latency_ms": 0.012770433841204201, + "mean_return": 864.0, + "min_return": 864.0, "observation_mode": "ram", "policy_path": "submitted_artifact", - "score": 475.0 + "score": 864.0 }, "hidden_shifted_start": { - "completion_rate": 0.0, + "completion_rate": 1.0, "episodes": [ { - "final_lives": 0, + "final_lives": 4, "invalid_action": false, - "length": 23878, - "mean_action_latency_ms": 0.005554480693310147, + "length": 108000, + "mean_action_latency_ms": 0.013449122585977117, "observation_mode": "ram", "policy_error": null, - "return": 475.0, + "return": 864.0, "seed": 211, - "terminated": true, - "truncated": false, + "terminated": false, + "truncated": true, "variant": "left_6", "warmup_steps": 6 }, { - "final_lives": 0, + "final_lives": 4, "invalid_action": false, - "length": 23878, - "mean_action_latency_ms": 0.005160711673857908, + "length": 108000, + "mean_action_latency_ms": 0.012494602430335901, "observation_mode": "ram", "policy_error": null, - "return": 475.0, + "return": 864.0, "seed": 211, - "terminated": true, - "truncated": false, + "terminated": false, + "truncated": true, "variant": "right_6", "warmup_steps": 6 } ], "invalid_episode_rate": 0.0, - "max_return": 475.0, + "max_return": 864.0, "max_steps": 108000, - "mean_action_latency_ms": 0.005357596183584027, - "mean_return": 475.0, - "min_return": 475.0, + "mean_action_latency_ms": 0.012971862508156509, + "mean_return": 864.0, + "min_return": 864.0, "observation_mode": "ram", "policy_path": "submitted_artifact", - "score": 475.0 + "score": 864.0 } }, - "target_gap": -389.0, + "target_gap": 0.0, "target_score": 864.0 } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json index c840d3a..e61db7d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_ram_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 90.566037736, "observed_raw_score": 819.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 819.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_breakout_ram_v1", "task_digest": "sha256:12eb4ca8814c602a2fbf709f0472c90c8758f3c1ffd0131329b734ca1d8e6b20", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 331.7, + "agent_setup": 2.6, + "environment_setup": 2.7, + "total": 444.4, + "verifier": 86.7 + }, "token_usage": { "final_input_tokens": 663.0, "final_output_tokens": 137.0, @@ -27,5 +45,7 @@ "total_tokens": 41760.0 }, "tool_calls": 62, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json index a2552db..87b4d85 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784075101.71677, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 362.4, + "agent_setup": 2.9, + "environment_setup": 33.8, + "total": 513.5, + "verifier": 91.2 + }, "token_usage": { "final_input_tokens": 745.0, "final_output_tokens": 114.0, @@ -27,5 +45,7 @@ "total_tokens": 29531.0 }, "tool_calls": 62, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json index fbf6fb4..cf18749 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784078883.680485, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 300.1, + "agent_setup": 3.4, + "environment_setup": 3.4, + "total": 423.2, + "verifier": 92.2 + }, "token_usage": { "final_input_tokens": 741.0, "final_output_tokens": 135.0, @@ -27,5 +45,7 @@ "total_tokens": 24428.0 }, "tool_calls": 41, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json index 0f27660..ec90f6c 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784092762.840873, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 367.5, + "agent_setup": 3.2, + "environment_setup": 2.7, + "total": 491.4, + "verifier": 93.9 + }, "token_usage": { "final_input_tokens": 557.0, "final_output_tokens": 134.0, @@ -27,5 +45,7 @@ "total_tokens": 32435.0 }, "tool_calls": 64, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json index ff38a32..42da334 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,13 +16,30 @@ }, "normalized_score": 70.758122744, "observed_raw_score": 702.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 702.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 390.4, + "agent_setup": 2.9, + "environment_setup": 6.3, + "total": 510.6, + "verifier": 89.1 + }, "token_usage": null, "tool_calls": 66, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json index 5920402..0ab18a3 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_breakout_rgb_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 21.299638989, "observed_raw_score": 428.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 428.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_breakout_rgb_v1", "task_digest": "sha256:dae0fbd989a518bd7dbdcd1c40d9a347bb6f835ac9cbc50957e7c5a63ec1db1e", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 210.4, + "agent_setup": 3.0, + "environment_setup": 2.8, + "total": 330.6, + "verifier": 93.1 + }, "token_usage": { "final_input_tokens": 23803.0, "final_output_tokens": 110.0, @@ -27,5 +45,7 @@ "total_tokens": 23913.0 }, "tool_calls": 45, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/metadata.json index 536ab9c..baf411e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/metadata.json @@ -9,14 +9,14 @@ "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 31.8442100886, - "normalized_score_stddev": 3.8873875677992533, + "normalized_score": 28.1034211084, + "normalized_score_stddev": 11.772801175021295, "provider_reasoning_effort": "max", - "raw_score": 6385.135940620764, + "raw_score": 6105.10168077257, "raw_score_imputation_count": 0, - "raw_score_stddev": 291.0085829715705, + "raw_score_stddev": 881.3081092982512, "source_run_ids": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/score.json index 522af16..70bbca5 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/score.json @@ -1,22 +1,22 @@ { "benchmark": "learning_beyond_gradients_article_suite", - "normalized_score": 31.8442100886, - "normalized_score_stddev": 3.8873875677992533, + "normalized_score": 28.1034211084, + "normalized_score_stddev": 11.772801175021295, "raw_score_imputation_count": 0, "reference_score": 11487.256451471967, - "score": 6385.135940620764, - "score_stddev": 291.0085829715705, + "score": 6105.10168077257, + "score_stddev": 881.3081092982512, "starter_score": 4001.2886158917113, "task": "simulation_heuristics_halfcheetah_v1", "trial_count": 5, "trials": [ { - "normalized_score": 26.12140084, - "observed_raw_score": 5956.728280971269, - "raw_score": 5956.728280971269, + "normalized_score": 7.417455939, + "observed_raw_score": 4556.556981730299, + "raw_score": 4556.556981730299, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "trial": 1 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json index 5f51c1b..60c3e59 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/metadata.json @@ -1,31 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, - "finished_at": 1784075101.71677, + "finished_at": 1784141320.1244009, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", "model": "azure/gpt-5.6-sol", "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 26.12140084, - "observed_raw_score": 5956.728280971269, + "normalized_score": 7.417455939, + "observed_raw_score": 4556.556981730299, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", - "raw_score": 5956.728280971269, + "published_protocol_version": "2.2", + "raw_score": 4556.556981730299, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 807.0, + "agent_setup": 4.5, + "environment_setup": 12.5, + "total": 3407.5, + "verifier": 2543.4 + }, "token_usage": { - "final_input_tokens": 666.0, - "final_output_tokens": 258.0, - "input_tokens": 666.0, - "output_tokens": 258.0, - "total_tokens": 18844.0 + "final_input_tokens": 543.0, + "final_output_tokens": 170.0, + "input_tokens": 543.0, + "output_tokens": 170.0, + "total_tokens": 27849.0 }, - "tool_calls": 22, - "trial": 1 + "tool_calls": 42, + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/score.json index a1559fb..72ec2de 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-01/score.json @@ -2,97 +2,97 @@ "hidden_nominal": { "episodes": [ { - "control_cost": 415.5962042212486, - "forward_reward": 6502.7550215291085, + "control_cost": 416.43198899924755, + "forward_reward": 5074.159036536156, "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 11.819874366978183, + "mean_action_latency_ms": 87.15905960649252, "policy_error": null, - "return": 6087.15881730786, + "return": 4657.727047536909, "seed": 211, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 325.1270932474631, - "x_velocity": 7.454664376824296 + "x_position": 253.69729399781534, + "x_velocity": 5.726530505354624 }, { - "control_cost": 415.76147669553757, - "forward_reward": 6405.91186391007, + "control_cost": 418.38612481951714, + "forward_reward": 5107.433908633669, "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 11.884192822035402, + "mean_action_latency_ms": 87.84144629724324, "policy_error": null, - "return": 5990.15038721453, + "return": 4689.047783814148, "seed": 307, "terminated": false, "truncated": true, "variant": "nominal", - "x_position": 320.3221772614403, - "x_velocity": 7.396573180181561 + "x_position": 255.39827949762008, + "x_velocity": 5.697664317698729 } ], "invalid_episode_rate": 0.0, - "max_return": 6087.15881730786, + "max_return": 4689.047783814148, "max_steps": 1000, - "mean_action_latency_ms": 11.852033594506793, - "mean_control_cost": 415.6788404583931, - "mean_forward_reward": 6454.33344271959, - "mean_return": 6038.654602261195, - "mean_x_position": 322.7246352544517, - "min_return": 5990.15038721453, + "mean_action_latency_ms": 87.50025295186788, + "mean_control_cost": 417.40905690938234, + "mean_forward_reward": 5090.796472584912, + "mean_return": 4673.387415675528, + "mean_x_position": 254.5477867477177, + "min_return": 4657.727047536909, "policy_path": "submitted_artifact", - "score": 6038.654602261195 + "score": 4673.387415675528 }, "hidden_nominal_weight": 0.7, "hidden_robustness": { "episodes": [ { - "control_cost": 406.1905238777399, - "forward_reward": 7183.572041837751, + "control_cost": 419.35061928629875, + "forward_reward": 5447.14772432608, "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 11.910023966571316, + "mean_action_latency_ms": 87.68435064703226, "policy_error": null, - "return": 6777.381517960015, + "return": 5027.797105039781, "seed": 503, "terminated": false, "truncated": true, "variant": "light_low_friction", - "x_position": 359.16092371449446, - "x_velocity": 6.435041780863084 + "x_position": 272.33970783891135, + "x_velocity": 5.425350565562265 }, { - "control_cost": 419.91816034913063, - "forward_reward": 5173.670371645333, + "control_cost": 416.9603366702795, + "forward_reward": 3957.068503346699, "invalid_action": false, "length": 1000, - "mean_action_latency_ms": 11.844803968910128, + "mean_action_latency_ms": 89.01173623092473, "policy_error": null, - "return": 4753.752211296202, + "return": 3540.1081666764185, "seed": 503, "terminated": false, "truncated": true, "variant": "heavy_weak_actuators", - "x_position": 258.6658402048741, - "x_velocity": 5.467666635602541 + "x_position": 197.83574678994236, + "x_velocity": 4.581104579131372 } ], "invalid_episode_rate": 0.0, - "max_return": 6777.381517960015, + "max_return": 5027.797105039781, "max_steps": 1000, - "mean_action_latency_ms": 11.877413967740722, - "mean_control_cost": 413.05434211343527, - "mean_forward_reward": 6178.621206741542, - "mean_return": 5765.566864628108, - "mean_x_position": 308.9133819596843, - "min_return": 4753.752211296202, + "mean_action_latency_ms": 88.3480434389785, + "mean_control_cost": 418.1554779782891, + "mean_forward_reward": 4702.10811383639, + "mean_return": 4283.9526358581, + "mean_x_position": 235.08772731442684, + "min_return": 3540.1081666764185, "policy_path": "submitted_artifact", - "score": 5765.566864628108 + "score": 4283.9526358581 }, "hidden_robustness_weight": 0.3, - "normalized_score": 26.12140084, + "normalized_score": 7.417455939, "reference_score": 11487.256451471967, - "score": 5956.728280971269, + "score": 4556.556981730299, "starter_score": 4001.2886158917113 } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json index 5df3418..6a94c15 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784078883.680485, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 33.28729024, "observed_raw_score": 6493.16445658394, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 6493.16445658394, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 660.3, + "agent_setup": 3.1, + "environment_setup": 2.7, + "total": 3139.6, + "verifier": 2443.8 + }, "token_usage": { "final_input_tokens": 622.0, "final_output_tokens": 230.0, @@ -27,5 +45,7 @@ "total_tokens": 29012.0 }, "tool_calls": 38, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json index 89047d6..ce6209d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784092762.840873, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 36.605449762, "observed_raw_score": 6741.560811161888, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 6741.560811161888, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1460.3, + "agent_setup": 3.3, + "environment_setup": 4.6, + "total": 7715.8, + "verifier": 6223.6 + }, "token_usage": { "final_input_tokens": 19401.0, "final_output_tokens": 208.0, @@ -27,5 +45,7 @@ "total_tokens": 19609.0 }, "tool_calls": 28, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json index 2e2bfc1..7a99225 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784100567.131854, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 32.792734613, "observed_raw_score": 6456.142181402096, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 6456.142181402096, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1260.4, + "agent_setup": 4.5, + "environment_setup": 39.9, + "total": 4912.0, + "verifier": 3569.6 + }, "token_usage": { "final_input_tokens": 345.0, "final_output_tokens": 211.0, @@ -27,5 +45,7 @@ "total_tokens": 19500.0 }, "tool_calls": 28, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json index 2732c2a..b588475 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_halfcheetah_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 10800, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 30.414174988, "observed_raw_score": 6278.083972984627, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 6278.083972984627, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_halfcheetah_v1", "task_digest": "sha256:aab8a7d4a73765939684373673d03a7811f56e06c4160beee887b3e3e6743d3b", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 255.4, + "agent_setup": 3.9, + "environment_setup": 2.6, + "total": 2466.1, + "verifier": 2169.4 + }, "token_usage": { "final_input_tokens": 2487.0, "final_output_tokens": 213.0, @@ -27,5 +45,7 @@ "total_tokens": 22668.0 }, "tool_calls": 27, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-01/metadata.json index 6065114..4e7d509 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784075101.71677, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 0.0, "observed_raw_score": 0.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.1, + "agent_setup": 3.4, + "environment_setup": 2.1, + "total": 228.1, + "verifier": 45.3 + }, "token_usage": { "final_input_tokens": 13326.0, "final_output_tokens": 32.0, @@ -27,5 +45,7 @@ "total_tokens": 41006.0 }, "tool_calls": 49, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-02/metadata.json index 5722190..b5837dc 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784078883.680485, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 400.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 400.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 210.1, + "agent_setup": 3.7, + "environment_setup": 2.4, + "total": 292.0, + "verifier": 46.9 + }, "token_usage": { "final_input_tokens": 753.0, "final_output_tokens": 138.0, @@ -27,5 +45,7 @@ "total_tokens": 41851.0 }, "tool_calls": 49, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-03/metadata.json index 05b24fe..f908c07 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784092762.840873, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 400.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 400.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1080.7, + "agent_setup": 3.9, + "environment_setup": 2.8, + "total": 1299.1, + "verifier": 43.8 + }, "token_usage": { "final_input_tokens": 671.0, "final_output_tokens": 170.0, @@ -27,5 +45,7 @@ "total_tokens": 29001.0 }, "tool_calls": 60, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-04/metadata.json index 11f68f4..517f615 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,13 +16,30 @@ }, "normalized_score": 100.0, "observed_raw_score": 400.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 400.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 210.1, + "agent_setup": 3.6, + "environment_setup": 2.8, + "total": 283.4, + "verifier": 39.9 + }, "token_usage": null, "tool_calls": 47, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-05/metadata.json index 71fde67..c48a4b3 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_montezuma_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 72.5, "observed_raw_score": 290.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 290.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_montezuma_v1", "task_digest": "sha256:518afe1cd1a1794ad99057a695c46008f38ebee53f9ed7e2dce017cdecc08dca", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 151.9, + "agent_setup": 3.7, + "environment_setup": 2.3, + "total": 241.4, + "verifier": 50.0 + }, "token_usage": { "final_input_tokens": 651.0, "final_output_tokens": 236.0, @@ -27,5 +45,7 @@ "total_tokens": 58743.0 }, "tool_calls": 64, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-01/metadata.json index 370e532..c90d91d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784075101.71677, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 79.166666667, "observed_raw_score": 16.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 16.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.1, + "agent_setup": 3.4, + "environment_setup": 5.4, + "total": 259.2, + "verifier": 73.3 + }, "token_usage": { "final_input_tokens": 789.0, "final_output_tokens": 132.0, @@ -27,5 +45,7 @@ "total_tokens": 33689.0 }, "tool_calls": 39, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-02/metadata.json index 9b4ad1c..8284a7b 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784078883.680485, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.5, + "agent_setup": 3.4, + "environment_setup": 3.3, + "total": 244.0, + "verifier": 62.3 + }, "token_usage": { "final_input_tokens": 541.0, "final_output_tokens": 166.0, @@ -27,5 +45,7 @@ "total_tokens": 26307.0 }, "tool_calls": 29, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-03/metadata.json index 6d93fc6..37c355d 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-03/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784092762.840873, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 83.333333333, "observed_raw_score": 17.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 17.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 1191.2, + "agent_setup": 3.7, + "environment_setup": 2.4, + "total": 1294.3, + "verifier": 55.4 + }, "token_usage": { "final_input_tokens": 492.0, "final_output_tokens": 162.0, @@ -27,5 +45,7 @@ "total_tokens": 39566.0 }, "tool_calls": 57, - "trial": 3 + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-04/metadata.json index dc48679..3dd4718 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,13 +16,30 @@ }, "normalized_score": 95.833333333, "observed_raw_score": 20.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 20.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 202.5, + "agent_setup": 3.0, + "environment_setup": 4.4, + "total": 291.9, + "verifier": 59.2 + }, "token_usage": null, "tool_calls": 37, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-05/metadata.json index fefdec2..e948cd5 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_pong_ram_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 100.0, "observed_raw_score": 21.0, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 21.0, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_pong_ram_v1", "task_digest": "sha256:23dc66f3d54d291790ea8a04459e3848966a82854ac3bea5107b0b5240e74457", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 90.1, + "agent_setup": 9.1, + "environment_setup": 2.3, + "total": 181.7, + "verifier": 51.4 + }, "token_usage": { "final_input_tokens": 811.0, "final_output_tokens": 194.0, @@ -27,5 +45,7 @@ "total_tokens": 22509.0 }, "tool_calls": 20, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/metadata.json index 05ddaab..412af81 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/metadata.json @@ -9,16 +9,16 @@ "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 56.834663982799995, - "normalized_score_stddev": 33.111283305681766, + "normalized_score": 73.23338822320001, + "normalized_score_stddev": 37.18119323399795, "provider_reasoning_effort": "max", - "raw_score": 0.6554319871515035, + "raw_score": 0.7171239853203296, "raw_score_imputation_count": 0, - "raw_score_stddev": 0.12456464290218099, + "raw_score_stddev": 0.13987564345039064, "source_run_ids": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/score.json index 0ebcf21..34fcd72 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/score.json @@ -1,11 +1,11 @@ { "benchmark": "learning_beyond_gradients_article_suite", - "normalized_score": 56.834663982799995, - "normalized_score_stddev": 33.111283305681766, + "normalized_score": 73.23338822320001, + "normalized_score_stddev": 37.18119323399795, "raw_score_imputation_count": 0, "reference_score": 0.8178199748694898, - "score": 0.6554319871515035, - "score_stddev": 0.12456464290218099, + "score": 0.7171239853203296, + "score_stddev": 0.13987564345039064, "starter_score": 0.44161998964846133, "task": "simulation_heuristics_vizdoom_d1_v1", "trial_count": 5, @@ -29,12 +29,12 @@ "trial": 2 }, { - "normalized_score": 31.015418049, - "observed_raw_score": 0.5582999877631664, - "raw_score": 0.5582999877631664, + "normalized_score": 113.009039251, + "observed_raw_score": 0.866759978607297, + "raw_score": 0.866759978607297, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "trial": 3 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json index 6ebc6f5..81ce28e 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-01/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784075101.71677, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 42.828284045, "observed_raw_score": 0.6027399878948927, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.6027399878948927, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 300.1, + "agent_setup": 4.7, + "environment_setup": 2.9, + "total": 389.9, + "verifier": 40.1 + }, "token_usage": { "final_input_tokens": 675.0, "final_output_tokens": 123.0, @@ -27,5 +45,7 @@ "total_tokens": 23326.0 }, "tool_calls": 28, - "trial": 1 + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json index bfd409f..983ae26 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784078883.680485, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 95.055823417, "observed_raw_score": 0.7992199832946062, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.7992199832946062, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.1, + "agent_setup": 5.5, + "environment_setup": 3.8, + "total": 251.9, + "verifier": 43.9 + }, "token_usage": { "final_input_tokens": 491.0, "final_output_tokens": 191.0, @@ -27,5 +45,7 @@ "total_tokens": 22186.0 }, "tool_calls": 24, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json index 6bef491..a07336c 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/metadata.json @@ -1,31 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, - "finished_at": 1784092762.840873, + "finished_at": 1784138333.944502, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", "model": "azure/gpt-5.6-sol", "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 31.015418049, - "observed_raw_score": 0.5582999877631664, + "normalized_score": 113.009039251, + "observed_raw_score": 0.866759978607297, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", - "raw_score": 0.5582999877631664, + "published_protocol_version": "2.2", + "raw_score": 0.866759978607297, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.3, + "agent_setup": 4.2, + "environment_setup": 3.5, + "total": 234.7, + "verifier": 38.4 + }, "token_usage": { - "final_input_tokens": 0.0, - "final_output_tokens": 0.0, - "input_tokens": 0.0, - "output_tokens": 0.0, - "total_tokens": 0.0 + "final_input_tokens": 655.0, + "final_output_tokens": 177.0, + "input_tokens": 655.0, + "output_tokens": 177.0, + "total_tokens": 21312.0 }, - "tool_calls": 1, - "trial": 3 + "tool_calls": 25, + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/score.json index 2693627..91abe79 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-03/score.json @@ -9,7 +9,7 @@ "suite_process_count": 2, "unique_working_directories": true }, - "normalized_score": 31.015418049, + "normalized_score": 113.009039251, "reference_score": 0.8178199748694897, "resource_cleanup": { "removed_bytes": 0, @@ -28,7 +28,7 @@ ], "terminated_processes": 0 }, - "score": 0.5582999877631664, + "score": 0.866759978607297, "starter_score": 0.44161998964846133, "suites": { "hidden_nominal": { @@ -37,70 +37,70 @@ "episodes": [ { "final_variables": { - "HEALTH": 68.0 + "HEALTH": 60.0 }, "invalid_action": false, "lane": 0, "length": 2100, - "mean_action_latency_ms": 0.09957417490936461, + "mean_action_latency_ms": 0.22186305728696643, "policy_error": null, - "return": 0.9199999831616879, + "return": 0.8799999766051769, "seed": 101, "terminated": false, "truncated": true }, { "final_variables": { - "HEALTH": 6.0 + "HEALTH": 43.0 }, "invalid_action": false, "lane": 1, - "length": 1216, - "mean_action_latency_ms": 0.1304904075560013, + "length": 2100, + "mean_action_latency_ms": 0.1953048977468695, "policy_error": null, - "return": 0.16199998930096626, + "return": 0.7949999757111073, "seed": 102, - "terminated": true, - "truncated": false + "terminated": false, + "truncated": true }, { "final_variables": { - "HEALTH": 58.0 + "HEALTH": 46.0 }, "invalid_action": false, "lane": 2, "length": 2100, - "mean_action_latency_ms": 0.08596905906285558, + "mean_action_latency_ms": 0.19933032847586132, "policy_error": null, - "return": 0.8699999824166298, + "return": 0.8099999837577343, "seed": 103, "terminated": false, "truncated": true }, { "final_variables": { - "HEALTH": 6.0 + "HEALTH": 78.0 }, "invalid_action": false, "lane": 3, - "length": 1376, - "mean_action_latency_ms": 0.11495853433125587, + "length": 2100, + "mean_action_latency_ms": 0.1936875833641915, "policy_error": null, - "return": 0.24199999123811722, + "return": 0.9699999801814556, "seed": 104, - "terminated": true, - "truncated": false + "terminated": false, + "truncated": true }, { "final_variables": { - "HEALTH": 69.0 + "HEALTH": 38.0 }, "invalid_action": false, "lane": 4, "length": 2100, - "mean_action_latency_ms": 0.08597019705034438, + "mean_action_latency_ms": 0.1954846740478561, "policy_error": null, - "return": 0.8929999880492687, + "return": 0.7539999708533287, "seed": 105, "terminated": false, "truncated": true @@ -108,16 +108,16 @@ ], "frame_skip": 1, "invalid_episode_rate": 0.0, - "max_return": 0.9199999831616879, + "max_return": 0.9699999801814556, "max_steps": 2100, - "mean_action_latency_ms": 0.10339247458196435, - "mean_return": 0.617399986833334, - "min_return": 0.16199998930096626, + "mean_action_latency_ms": 0.20113410818434901, + "mean_return": 0.8417999774217606, + "min_return": 0.7539999708533287, "policy_path": "submitted_artifact", "render_height": 180, "render_width": 240, "scenario": "d1", - "score": 0.617399986833334 + "score": 0.8417999774217606 }, "hidden_secondary": { "batch_seed": 607, @@ -125,87 +125,87 @@ "episodes": [ { "final_variables": { - "HEALTH": 7.0 + "HEALTH": 76.0 }, "invalid_action": false, "lane": 0, - "length": 672, - "mean_action_latency_ms": 0.2247366133988613, + "length": 2100, + "mean_action_latency_ms": 0.22642912609236582, "policy_error": null, - "return": -0.10500000417232513, + "return": 0.9599999822676182, "seed": 607, - "terminated": true, - "truncated": false + "terminated": false, + "truncated": true }, { "final_variables": { - "HEALTH": 69.0 + "HEALTH": 70.0 }, "invalid_action": false, "lane": 1, "length": 2100, - "mean_action_latency_ms": 0.13988662688505082, + "mean_action_latency_ms": 0.2081541752531415, "policy_error": null, - "return": 0.9089999869465828, + "return": 0.9299999810755253, "seed": 608, "terminated": false, "truncated": true }, { "final_variables": { - "HEALTH": 68.0 + "HEALTH": 60.0 }, "invalid_action": false, "lane": 2, "length": 2100, - "mean_action_latency_ms": 0.07088755479171163, + "mean_action_latency_ms": 0.20032513709295363, "policy_error": null, - "return": 0.9199999831616879, + "return": 0.8799999766051769, "seed": 609, "terminated": false, "truncated": true }, { "final_variables": { - "HEALTH": 3.0 + "HEALTH": 47.0 }, "invalid_action": false, "lane": 3, - "length": 1888, - "mean_action_latency_ms": 0.1139322570051556, + "length": 2100, + "mean_action_latency_ms": 0.19666522918712526, "policy_error": null, - "return": 0.4829999879002571, + "return": 0.8149999864399433, "seed": 610, - "terminated": true, - "truncated": false + "terminated": false, + "truncated": true }, { "final_variables": { - "HEALTH": 7.0 + "HEALTH": 92.0 }, "invalid_action": false, "lane": 4, - "length": 672, - "mean_action_latency_ms": 0.13459125988274104, + "length": 2100, + "mean_action_latency_ms": 0.18393707328609057, "policy_error": null, - "return": -0.10500000417232513, + "return": 1.0399999804794788, "seed": 611, - "terminated": true, - "truncated": false + "terminated": false, + "truncated": true } ], "frame_skip": 1, "invalid_episode_rate": 0.0, - "max_return": 0.9199999831616879, + "max_return": 1.0399999804794788, "max_steps": 2100, - "mean_action_latency_ms": 0.13680686239270407, - "mean_return": 0.4203999899327755, - "min_return": -0.10500000417232513, + "mean_action_latency_ms": 0.20310214818233535, + "mean_return": 0.9249999813735486, + "min_return": 0.8149999864399433, "policy_path": "submitted_artifact", "render_height": 180, "render_width": 240, "scenario": "d1", - "score": 0.4203999899327755 + "score": 0.9249999813735486 } } } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json index b2749b2..5fa4138 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,13 +16,30 @@ }, "normalized_score": 89.73418684, "observed_raw_score": 0.7791999872773886, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.7791999872773886, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 169.0, + "agent_setup": 4.3, + "environment_setup": 2.7, + "total": 262.9, + "verifier": 48.1 + }, "token_usage": null, "tool_calls": 23, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json index eed9519..bdba9e9 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d1_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 25.539607563, "observed_raw_score": 0.5376999895274639, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 0.5376999895274639, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_vizdoom_d1_v1", "task_digest": "sha256:ea30e0252670bc77855fc732868f29c080f8773c1e0f9b72180289c568c59238", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 150.0, + "agent_setup": 3.9, + "environment_setup": 8.1, + "total": 241.1, + "verifier": 39.2 + }, "token_usage": { "final_input_tokens": 817.0, "final_output_tokens": 159.0, @@ -27,5 +45,7 @@ "total_tokens": 19408.0 }, "tool_calls": 20, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/metadata.json index b16b3de..6cdcd83 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/metadata.json @@ -9,16 +9,16 @@ "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 16.3034759358, - "normalized_score_stddev": 23.995771403261536, + "normalized_score": 13.850267379600002, + "normalized_score_stddev": 20.096747543003065, "provider_reasoning_effort": "max", - "raw_score": 80.67999999999999, + "raw_score": 73.34, "raw_score_imputation_count": 0, - "raw_score_stddev": 71.79534803871347, + "raw_score_stddev": 60.12946864890792, "source_run_ids": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/score.json index c1a882f..ae2d3da 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/score.json @@ -1,22 +1,22 @@ { "benchmark": "learning_beyond_gradients_article_suite", - "normalized_score": 16.3034759358, - "normalized_score_stddev": 23.995771403261536, + "normalized_score": 13.850267379600002, + "normalized_score_stddev": 20.096747543003065, "raw_score_imputation_count": 0, "reference_score": 331.1, - "score": 80.67999999999999, - "score_stddev": 71.79534803871347, + "score": 73.34, + "score_stddev": 60.12946864890792, "starter_score": 31.9, "task": "simulation_heuristics_vizdoom_d3_v1", "trial_count": 5, "trials": [ { - "normalized_score": 40.106951872, - "observed_raw_score": 151.89999999999998, - "raw_score": 151.89999999999998, + "normalized_score": 4.879679144, + "observed_raw_score": 46.5, + "raw_score": 46.5, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "trial": 1 }, { @@ -29,12 +29,12 @@ "trial": 2 }, { - "normalized_score": 0.0, - "observed_raw_score": 31.9, - "raw_score": 31.9, + "normalized_score": 22.961229947, + "observed_raw_score": 100.6, + "raw_score": 100.6, "raw_score_source": "observed", "score_path": "leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/score.json", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "trial": 3 }, { diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json index 22dea8f..ba270f9 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/metadata.json @@ -1,31 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, - "finished_at": 1784075101.71677, + "finished_at": 1784141320.1244009, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", "model": "azure/gpt-5.6-sol", "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 40.106951872, - "observed_raw_score": 151.89999999999998, + "normalized_score": 4.879679144, + "observed_raw_score": 46.5, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", - "raw_score": 151.89999999999998, + "published_protocol_version": "2.2", + "raw_score": 46.5, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 254.7, + "agent_setup": 3.7, + "environment_setup": 2.6, + "total": 327.8, + "verifier": 38.0 + }, "token_usage": { - "final_input_tokens": 854.0, - "final_output_tokens": 171.0, - "input_tokens": 854.0, - "output_tokens": 171.0, - "total_tokens": 30721.0 + "final_input_tokens": 725.0, + "final_output_tokens": 169.0, + "input_tokens": 725.0, + "output_tokens": 169.0, + "total_tokens": 31102.0 }, - "tool_calls": 35, - "trial": 1 + "tool_calls": 32, + "transport_error_info": null, + "trial": 1, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/score.json index dfd9ffa..94d15c2 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-01/score.json @@ -4,9 +4,9 @@ "suite_process_count": 2, "unique_working_directories": true }, - "normalized_score": 40.106951872, + "normalized_score": 4.879679144, "reference_score": 331.1, - "score": 151.89999999999998, + "score": 46.5, "starter_score": 31.9, "suites": { "hidden_nominal": { @@ -15,36 +15,36 @@ "episodes": [ { "final_variables": { - "AMMO2": 0.0, - "DAMAGECOUNT": 20.0, + "AMMO2": 12.0, + "DAMAGECOUNT": 5.0, "HEALTH": 10.0, - "HITCOUNT": 4.0, - "KILLCOUNT": 4.0 + "HITCOUNT": 1.0, + "KILLCOUNT": 1.0 }, "invalid_action": false, "lane": 0, - "length": 754, - "mean_action_latency_ms": 3.0050282255407987, + "length": 181, + "mean_action_latency_ms": 3.355628754223249, "policy_error": null, - "return": 60.0, + "return": 15.0, "seed": 101, "terminated": true, "truncated": false }, { "final_variables": { - "AMMO2": 0.0, - "DAMAGECOUNT": 60.0, - "HEALTH": 4.0, - "HITCOUNT": 5.0, - "KILLCOUNT": 5.0 + "AMMO2": 18.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 7.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 }, "invalid_action": false, "lane": 1, - "length": 736, - "mean_action_latency_ms": 3.0707310804206394, + "length": 197, + "mean_action_latency_ms": 3.3963004004229145, "policy_error": null, - "return": 110.0, + "return": 0.0, "seed": 102, "terminated": true, "truncated": false @@ -52,35 +52,35 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 70.0, - "HEALTH": 2.0, - "HITCOUNT": 9.0, + "DAMAGECOUNT": 105.0, + "HEALTH": 20.0, + "HITCOUNT": 10.0, "KILLCOUNT": 9.0 }, "invalid_action": false, "lane": 2, - "length": 737, - "mean_action_latency_ms": 3.0704634200928167, + "length": 523, + "mean_action_latency_ms": 3.858295612991654, "policy_error": null, - "return": 160.0, + "return": 195.0, "seed": 103, "terminated": true, "truncated": false }, { "final_variables": { - "AMMO2": 42.0, - "DAMAGECOUNT": 95.0, - "HEALTH": 1.0, - "HITCOUNT": 10.0, - "KILLCOUNT": 9.0 + "AMMO2": 0.0, + "DAMAGECOUNT": 15.0, + "HEALTH": 6.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 }, "invalid_action": false, "lane": 3, - "length": 968, - "mean_action_latency_ms": 2.989207848537067, + "length": 442, + "mean_action_latency_ms": 3.423510743010098, "policy_error": null, - "return": 185.0, + "return": 35.0, "seed": 104, "terminated": true, "truncated": false @@ -88,17 +88,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 30.0, + "DAMAGECOUNT": 20.0, "HEALTH": 10.0, - "HITCOUNT": 3.0, - "KILLCOUNT": 3.0 + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 }, "invalid_action": false, "lane": 4, - "length": 596, - "mean_action_latency_ms": 3.0182157299362573, + "length": 414, + "mean_action_latency_ms": 3.4997149276128714, "policy_error": null, - "return": 60.0, + "return": 40.0, "seed": 105, "terminated": true, "truncated": false @@ -106,16 +106,16 @@ ], "frame_skip": 2, "invalid_episode_rate": 0.0, - "max_return": 185.0, + "max_return": 195.0, "max_steps": 1050, - "mean_action_latency_ms": 3.030729260905516, - "mean_return": 115.0, - "min_return": 60.0, + "mean_action_latency_ms": 3.5066900876521574, + "mean_return": 57.0, + "min_return": 0.0, "policy_path": "submitted_artifact", "render_height": 480, "render_width": 640, "scenario": "d3", - "score": 115.0 + "score": 57.0 }, "hidden_secondary": { "batch_seed": 607, @@ -123,107 +123,107 @@ "episodes": [ { "final_variables": { - "AMMO2": 2.0, - "DAMAGECOUNT": 165.0, - "HEALTH": 100.0, - "HITCOUNT": 16.0, - "KILLCOUNT": 14.0 + "AMMO2": 6.0, + "DAMAGECOUNT": 10.0, + "HEALTH": 13.0, + "HITCOUNT": 1.0, + "KILLCOUNT": 1.0 }, "invalid_action": false, "lane": 0, - "length": 1050, - "mean_action_latency_ms": 3.06140120895136, + "length": 167, + "mean_action_latency_ms": 3.5258141864916523, "policy_error": null, - "return": 305.0, + "return": 20.0, "seed": 607, - "terminated": false, - "truncated": true + "terminated": true, + "truncated": false }, { "final_variables": { - "AMMO2": 26.0, - "DAMAGECOUNT": 90.0, - "HEALTH": 22.0, - "HITCOUNT": 7.0, - "KILLCOUNT": 7.0 + "AMMO2": 0.0, + "DAMAGECOUNT": 15.0, + "HEALTH": 6.0, + "HITCOUNT": 1.0, + "KILLCOUNT": 1.0 }, "invalid_action": false, "lane": 1, - "length": 466, - "mean_action_latency_ms": 2.9988041699188464, + "length": 465, + "mean_action_latency_ms": 3.698079288005829, "policy_error": null, - "return": 160.0, + "return": 25.0, "seed": 608, "terminated": true, "truncated": false }, { "final_variables": { - "AMMO2": 46.0, - "DAMAGECOUNT": 95.0, - "HEALTH": 16.0, - "HITCOUNT": 9.0, - "KILLCOUNT": 9.0 + "AMMO2": 0.0, + "DAMAGECOUNT": 0.0, + "HEALTH": 22.0, + "HITCOUNT": 0.0, + "KILLCOUNT": 0.0 }, "invalid_action": false, "lane": 2, - "length": 1021, - "mean_action_latency_ms": 3.027502381690686, + "length": 389, + "mean_action_latency_ms": 3.359957792075557, "policy_error": null, - "return": 185.0, + "return": 0.0, "seed": 609, "terminated": true, "truncated": false }, { "final_variables": { - "AMMO2": 18.0, - "DAMAGECOUNT": 50.0, - "HEALTH": 1.0, - "HITCOUNT": 5.0, - "KILLCOUNT": 5.0 + "AMMO2": 3.0, + "DAMAGECOUNT": 10.0, + "HEALTH": 19.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 }, "invalid_action": false, "lane": 3, - "length": 916, - "mean_action_latency_ms": 3.0297691645060043, + "length": 436, + "mean_action_latency_ms": 3.4050547833973113, "policy_error": null, - "return": 100.0, + "return": 30.0, "seed": 610, "terminated": true, "truncated": false }, { "final_variables": { - "AMMO2": 20.0, - "DAMAGECOUNT": 220.0, - "HEALTH": 100.0, - "HITCOUNT": 25.0, - "KILLCOUNT": 22.0 + "AMMO2": 0.0, + "DAMAGECOUNT": 15.0, + "HEALTH": 7.0, + "HITCOUNT": 2.0, + "KILLCOUNT": 2.0 }, "invalid_action": false, "lane": 4, - "length": 1050, - "mean_action_latency_ms": 3.0611898253361383, + "length": 373, + "mean_action_latency_ms": 3.432666638181932, "policy_error": null, - "return": 440.0, + "return": 35.0, "seed": 611, - "terminated": false, - "truncated": true + "terminated": true, + "truncated": false } ], "frame_skip": 2, "invalid_episode_rate": 0.0, - "max_return": 440.0, + "max_return": 35.0, "max_steps": 1050, - "mean_action_latency_ms": 3.0357333500806067, - "mean_return": 238.0, - "min_return": 100.0, + "mean_action_latency_ms": 3.484314537630456, + "mean_return": 22.0, + "min_return": 0.0, "policy_path": "submitted_artifact", "render_height": 480, "render_width": 640, "scenario": "d3", - "score": 238.0 + "score": 22.0 } } } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json index 91e7836..ab9d674 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-02/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784078883.680485, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": -3.442513369, "observed_raw_score": 21.599999999999998, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 21.599999999999998, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 211.9, + "agent_setup": 3.9, + "environment_setup": 2.5, + "total": 290.1, + "verifier": 41.1 + }, "token_usage": { "final_input_tokens": 379.0, "final_output_tokens": 167.0, @@ -27,5 +45,7 @@ "total_tokens": 22562.0 }, "tool_calls": 22, - "trial": 2 + "transport_error_info": null, + "trial": 2, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json index 8eeddb8..1924efa 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/metadata.json @@ -1,31 +1,53 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, - "finished_at": 1784092762.840873, + "finished_at": 1784138333.944502, "harness": "opencode", + "idle_timeout_info": null, "model": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", "model": "azure/gpt-5.6-sol", "provider": "azure", "provider_reasoning_effort": "max" }, - "normalized_score": 0.0, - "observed_raw_score": 31.9, + "normalized_score": 22.961229947, + "observed_raw_score": 100.6, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", - "raw_score": 31.9, + "published_protocol_version": "2.2", + "raw_score": 100.6, "raw_score_source": "observed", - "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "source_agent_idle_timeout_sec": 3600, + "source_daytona_pty_readline_timeout_sec": 3600, + "source_protocol_version": "2.2", + "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 330.3, + "agent_setup": 3.7, + "environment_setup": 2.6, + "total": 419.7, + "verifier": 54.4 + }, "token_usage": { - "final_input_tokens": 516.0, - "final_output_tokens": 100.0, - "input_tokens": 516.0, - "output_tokens": 100.0, - "total_tokens": 26216.0 + "final_input_tokens": 743.0, + "final_output_tokens": 164.0, + "input_tokens": 743.0, + "output_tokens": 164.0, + "total_tokens": 25995.0 }, - "tool_calls": 23, - "trial": 3 + "tool_calls": 30, + "transport_error_info": null, + "trial": 3, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/score.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/score.json index 6aaa0ce..3eaaa4f 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/score.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-03/score.json @@ -4,9 +4,9 @@ "suite_process_count": 2, "unique_working_directories": true }, - "normalized_score": 0.0, + "normalized_score": 22.961229947, "reference_score": 331.1, - "score": 31.9, + "score": 100.6, "starter_score": 31.9, "suites": { "hidden_nominal": { @@ -16,17 +16,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 10.0, + "DAMAGECOUNT": 20.0, "HEALTH": 10.0, "HITCOUNT": 2.0, "KILLCOUNT": 2.0 }, "invalid_action": false, "lane": 0, - "length": 711, - "mean_action_latency_ms": 2.0836446952970724, + "length": 416, + "mean_action_latency_ms": 5.102991484678709, "policy_error": null, - "return": 30.0, + "return": 40.0, "seed": 101, "terminated": true, "truncated": false @@ -34,17 +34,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 20.0, - "HEALTH": 12.0, - "HITCOUNT": 2.0, - "KILLCOUNT": 2.0 + "DAMAGECOUNT": 130.0, + "HEALTH": 13.0, + "HITCOUNT": 14.0, + "KILLCOUNT": 13.0 }, "invalid_action": false, "lane": 1, - "length": 980, - "mean_action_latency_ms": 2.0975400332589538, + "length": 925, + "mean_action_latency_ms": 4.9426648302658185, "policy_error": null, - "return": 40.0, + "return": 260.0, "seed": 102, "terminated": true, "truncated": false @@ -52,17 +52,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 10.0, - "HEALTH": 16.0, + "DAMAGECOUNT": 15.0, + "HEALTH": 13.0, "HITCOUNT": 2.0, "KILLCOUNT": 2.0 }, "invalid_action": false, "lane": 2, - "length": 317, - "mean_action_latency_ms": 2.065451903862156, + "length": 505, + "mean_action_latency_ms": 5.006042963797503, "policy_error": null, - "return": 30.0, + "return": 35.0, "seed": 103, "terminated": true, "truncated": false @@ -70,17 +70,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 0.0, - "HEALTH": 13.0, - "HITCOUNT": 0.0, - "KILLCOUNT": 0.0 + "DAMAGECOUNT": 85.0, + "HEALTH": 1.0, + "HITCOUNT": 10.0, + "KILLCOUNT": 9.0 }, "invalid_action": false, "lane": 3, - "length": 179, - "mean_action_latency_ms": 2.033258387662845, + "length": 846, + "mean_action_latency_ms": 4.950328934805613, "policy_error": null, - "return": 0.0, + "return": 175.0, "seed": 104, "terminated": true, "truncated": false @@ -88,17 +88,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 20.0, - "HEALTH": 10.0, - "HITCOUNT": 2.0, - "KILLCOUNT": 2.0 + "DAMAGECOUNT": 35.0, + "HEALTH": 4.0, + "HITCOUNT": 3.0, + "KILLCOUNT": 3.0 }, "invalid_action": false, "lane": 4, - "length": 306, - "mean_action_latency_ms": 2.0744595676660538, + "length": 294, + "mean_action_latency_ms": 5.3924833912225, "policy_error": null, - "return": 40.0, + "return": 65.0, "seed": 105, "terminated": true, "truncated": false @@ -106,16 +106,16 @@ ], "frame_skip": 2, "invalid_episode_rate": 0.0, - "max_return": 40.0, + "max_return": 260.0, "max_steps": 1050, - "mean_action_latency_ms": 2.070870917549416, - "mean_return": 28.0, - "min_return": 0.0, + "mean_action_latency_ms": 5.078902320954029, + "mean_return": 115.0, + "min_return": 35.0, "policy_path": "submitted_artifact", "render_height": 480, "render_width": 640, "scenario": "d3", - "score": 28.0 + "score": 115.0 }, "hidden_secondary": { "batch_seed": 607, @@ -124,35 +124,35 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 35.0, - "HEALTH": 90.0, - "HITCOUNT": 4.0, - "KILLCOUNT": 4.0 + "DAMAGECOUNT": 65.0, + "HEALTH": 7.0, + "HITCOUNT": 6.0, + "KILLCOUNT": 6.0 }, "invalid_action": false, "lane": 0, - "length": 1050, - "mean_action_latency_ms": 1.997888364962169, + "length": 410, + "mean_action_latency_ms": 5.356633408767421, "policy_error": null, - "return": 75.0, + "return": 125.0, "seed": 607, - "terminated": false, - "truncated": true + "terminated": true, + "truncated": false }, { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 10.0, - "HEALTH": 10.0, + "DAMAGECOUNT": 15.0, + "HEALTH": 2.0, "HITCOUNT": 1.0, "KILLCOUNT": 1.0 }, "invalid_action": false, "lane": 1, - "length": 262, - "mean_action_latency_ms": 2.115637646945378, + "length": 388, + "mean_action_latency_ms": 5.24872732492759, "policy_error": null, - "return": 20.0, + "return": 25.0, "seed": 608, "terminated": true, "truncated": false @@ -160,17 +160,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 0.0, - "HEALTH": 4.0, - "HITCOUNT": 0.0, - "KILLCOUNT": 0.0 + "DAMAGECOUNT": 30.0, + "HEALTH": 15.0, + "HITCOUNT": 4.0, + "KILLCOUNT": 4.0 }, "invalid_action": false, "lane": 2, - "length": 473, - "mean_action_latency_ms": 2.0683587353526915, + "length": 533, + "mean_action_latency_ms": 5.308813990336645, "policy_error": null, - "return": 0.0, + "return": 70.0, "seed": 609, "terminated": true, "truncated": false @@ -178,17 +178,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 0.0, - "HEALTH": 19.0, - "HITCOUNT": 0.0, - "KILLCOUNT": 0.0 + "DAMAGECOUNT": 5.0, + "HEALTH": 13.0, + "HITCOUNT": 1.0, + "KILLCOUNT": 1.0 }, "invalid_action": false, "lane": 3, - "length": 400, - "mean_action_latency_ms": 2.0626725628972054, + "length": 489, + "mean_action_latency_ms": 5.064522980851386, "policy_error": null, - "return": 0.0, + "return": 15.0, "seed": 610, "terminated": true, "truncated": false @@ -196,17 +196,17 @@ { "final_variables": { "AMMO2": 0.0, - "DAMAGECOUNT": 60.0, - "HEALTH": 19.0, + "DAMAGECOUNT": 50.0, + "HEALTH": 7.0, "HITCOUNT": 5.0, "KILLCOUNT": 5.0 }, "invalid_action": false, "lane": 4, - "length": 373, - "mean_action_latency_ms": 2.069716169511345, + "length": 386, + "mean_action_latency_ms": 5.069627122557843, "policy_error": null, - "return": 110.0, + "return": 100.0, "seed": 611, "terminated": true, "truncated": false @@ -214,16 +214,16 @@ ], "frame_skip": 2, "invalid_episode_rate": 0.0, - "max_return": 110.0, + "max_return": 125.0, "max_steps": 1050, - "mean_action_latency_ms": 2.0628546959337575, - "mean_return": 41.0, - "min_return": 0.0, + "mean_action_latency_ms": 5.2096649654881775, + "mean_return": 67.0, + "min_return": 15.0, "policy_path": "submitted_artifact", "render_height": 480, "render_width": 640, "scenario": "d3", - "score": 41.0 + "score": 67.0 } } } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json index f531486..8937311 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-04/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784095422.450964, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,13 +16,30 @@ }, "normalized_score": 0.0, "observed_raw_score": 31.9, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 31.9, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 210.6, + "agent_setup": 6.3, + "environment_setup": 4.3, + "total": 482.0, + "verifier": 224.6 + }, "token_usage": null, "tool_calls": 23, - "trial": 4 + "transport_error_info": null, + "trial": 4, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json index d2453ec..8b0ed76 100644 --- a/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json +++ b/leaderboard/article_suite_submissions/gpt-5.6-sol/simulation_heuristics_vizdoom_d3_v1/trial-05/metadata.json @@ -1,9 +1,12 @@ { + "agent_timeout_info": null, + "agent_timeout_sec": 5400, "benchmark": "learning_beyond_gradients_article_suite", "current_task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", "digest_compatibility_note": null, "finished_at": 1784098371.002007, "harness": "opencode", + "idle_timeout_info": null, "model": { "display_name": "GPT-5.6 Sol", "id": "gpt-5.6-sol", @@ -13,12 +16,27 @@ }, "normalized_score": 44.852941176, "observed_raw_score": 166.1, + "protocol_agent_idle_timeout_sec": 3600, + "protocol_daytona_pty_readline_timeout_sec": 3600, "provider_reasoning_effort": "max", + "published_protocol_version": "2.2", "raw_score": 166.1, "raw_score_source": "observed", + "source_agent_idle_timeout_sec": 600, + "source_daytona_pty_readline_timeout_sec": 900, + "source_protocol_version": "2.1", "source_run_id": "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05", "task": "simulation_heuristics_vizdoom_d3_v1", "task_digest": "sha256:a1da044451acdb98fa331f2742def1693b3233098571b4d5aee5ecd43124617c", + "terminal_error": null, + "terminal_error_category": null, + "timing": { + "agent_execution": 240.1, + "agent_setup": 3.6, + "environment_setup": 2.1, + "total": 316.9, + "verifier": 45.2 + }, "token_usage": { "final_input_tokens": 543.0, "final_output_tokens": 169.0, @@ -27,5 +45,7 @@ "total_tokens": 22728.0 }, "tool_calls": 22, - "trial": 5 + "transport_error_info": null, + "trial": 5, + "verifier_timeout_info": null } diff --git a/leaderboard/article_suite_task_leaderboards.png b/leaderboard/article_suite_task_leaderboards.png index 18766cf..1bb88e0 100644 Binary files a/leaderboard/article_suite_task_leaderboards.png and b/leaderboard/article_suite_task_leaderboards.png differ diff --git a/leaderboard/article_suite_timeout_fairness_audit.json b/leaderboard/article_suite_timeout_fairness_audit.json new file mode 100644 index 0000000..d65ae28 --- /dev/null +++ b/leaderboard/article_suite_timeout_fairness_audit.json @@ -0,0 +1,186 @@ +{ + "expected_cells": 180, + "fairness": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, + "interpretation": "historically affected cells rerun under standardized safeguards; unaffected source cells retained only when no idle or PTY timeout influenced the selected result" + }, + "generated_at": "2026-07-15T19:13:43.174081+00:00", + "historically_affected_cells": [ + { + "model_id": "gpt-5.5", + "reason": "pty_900s", + "task": "simulation_heuristics_breakout_rgb_v1", + "trial": 1 + }, + { + "model_id": "gpt-5.5", + "reason": "pty_900s", + "task": "simulation_heuristics_ant_v1", + "trial": 4 + }, + { + "model_id": "gpt-5.5", + "reason": "idle_600s", + "task": "simulation_heuristics_montezuma_v1", + "trial": 4 + }, + { + "model_id": "gpt-5.5", + "reason": "pty_900s", + "task": "simulation_heuristics_pong_ram_v1", + "trial": 4 + }, + { + "model_id": "gpt-5.6-sol", + "reason": "pty_900s", + "task": "simulation_heuristics_halfcheetah_v1", + "trial": 1 + }, + { + "model_id": "gpt-5.6-sol", + "reason": "pty_900s", + "task": "simulation_heuristics_vizdoom_d3_v1", + "trial": 1 + }, + { + "model_id": "gpt-5.6-sol", + "reason": "pty_900s", + "task": "simulation_heuristics_ant_v1", + "trial": 2 + }, + { + "model_id": "gpt-5.6-sol", + "reason": "idle_600s", + "task": "simulation_heuristics_vizdoom_d1_v1", + "trial": 3 + }, + { + "model_id": "gpt-5.6-sol", + "reason": "pty_900s", + "task": "simulation_heuristics_vizdoom_d3_v1", + "trial": 3 + }, + { + "model_id": "gpt-5.6-sol", + "reason": "pty_900s", + "task": "simulation_heuristics_breakout_ram_v1", + "trial": 4 + } + ], + "model_iqm_changes": [ + { + "after_iqm": 13.868245803826087, + "before_iqm": 10.662970787652174, + "delta": 3.2052750161739123, + "model_id": "gpt-5.5" + }, + { + "after_iqm": 40.67930709160869, + "before_iqm": 36.651764607913044, + "delta": 4.0275424836956475, + "model_id": "gpt-5.6-sol" + } + ], + "protocol_version": "2.2", + "rerun_cell_count": 10, + "rerun_score_changes": [ + { + "after_normalized_score": 99.277978339, + "before_normalized_score": 21.299638989, + "delta": 77.97833935, + "model_id": "gpt-5.5", + "task": "simulation_heuristics_breakout_rgb_v1", + "trial": 1 + }, + { + "after_normalized_score": 0.0, + "before_normalized_score": -1.398078834, + "delta": 1.398078834, + "model_id": "gpt-5.5", + "task": "simulation_heuristics_ant_v1", + "trial": 4 + }, + { + "after_normalized_score": 0.0, + "before_normalized_score": 0.0, + "delta": 0.0, + "model_id": "gpt-5.5", + "task": "simulation_heuristics_montezuma_v1", + "trial": 4 + }, + { + "after_normalized_score": 50.0, + "before_normalized_score": 25.0, + "delta": 25.0, + "model_id": "gpt-5.5", + "task": "simulation_heuristics_pong_ram_v1", + "trial": 4 + }, + { + "after_normalized_score": 7.417455939, + "before_normalized_score": 26.12140084, + "delta": -18.703944901, + "model_id": "gpt-5.6-sol", + "task": "simulation_heuristics_halfcheetah_v1", + "trial": 1 + }, + { + "after_normalized_score": 4.879679144, + "before_normalized_score": 40.106951872, + "delta": -35.227272728, + "model_id": "gpt-5.6-sol", + "task": "simulation_heuristics_vizdoom_d3_v1", + "trial": 1 + }, + { + "after_normalized_score": -13.495063777, + "before_normalized_score": -7.591479, + "delta": -5.903584777000001, + "model_id": "gpt-5.6-sol", + "task": "simulation_heuristics_ant_v1", + "trial": 2 + }, + { + "after_normalized_score": 113.009039251, + "before_normalized_score": 31.015418049, + "delta": 81.993621202, + "model_id": "gpt-5.6-sol", + "task": "simulation_heuristics_vizdoom_d1_v1", + "trial": 3 + }, + { + "after_normalized_score": 22.961229947, + "before_normalized_score": 0.0, + "delta": 22.961229947, + "model_id": "gpt-5.6-sol", + "task": "simulation_heuristics_vizdoom_d3_v1", + "trial": 3 + }, + { + "after_normalized_score": 100.0, + "before_normalized_score": 18.448637317, + "delta": 81.55136268300001, + "model_id": "gpt-5.6-sol", + "task": "simulation_heuristics_breakout_ram_v1", + "trial": 4 + } + ], + "selected_cells": 180, + "selected_timeout_influenced_cells": [], + "shared_wall_clock_timeout_cells": [ + { + "model_id": "claude-opus-4.8", + "task": "simulation_heuristics_ant_v1", + "timeout_sec": 5400.0, + "trial": 1 + }, + { + "model_id": "claude-opus-4.8", + "task": "simulation_heuristics_ant_v1", + "timeout_sec": 5400.0, + "trial": 2 + } + ], + "status": "ok" +} diff --git a/scripts/audit_article_suite_timeout_fairness.py b/scripts/audit_article_suite_timeout_fairness.py new file mode 100644 index 0000000..705a02f --- /dev/null +++ b/scripts/audit_article_suite_timeout_fairness.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import math +import tomllib +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[1] +PROTOCOL_PATH = REPO_ROOT / "experiments" / "article_suite" / "protocol.toml" +SUBMISSIONS_ROOT = REPO_ROOT / "leaderboard" / "article_suite_submissions" +DEFAULT_OUTPUT = ( + REPO_ROOT / "leaderboard" / "article_suite_timeout_fairness_audit.json" +) + +HISTORICALLY_AFFECTED_CELLS = ( + ("gpt-5.5", "simulation_heuristics_breakout_rgb_v1", 1, "pty_900s"), + ("gpt-5.5", "simulation_heuristics_ant_v1", 4, "pty_900s"), + ("gpt-5.5", "simulation_heuristics_montezuma_v1", 4, "idle_600s"), + ("gpt-5.5", "simulation_heuristics_pong_ram_v1", 4, "pty_900s"), + ("gpt-5.6-sol", "simulation_heuristics_halfcheetah_v1", 1, "pty_900s"), + ("gpt-5.6-sol", "simulation_heuristics_vizdoom_d3_v1", 1, "pty_900s"), + ("gpt-5.6-sol", "simulation_heuristics_ant_v1", 2, "pty_900s"), + ("gpt-5.6-sol", "simulation_heuristics_vizdoom_d1_v1", 3, "idle_600s"), + ("gpt-5.6-sol", "simulation_heuristics_vizdoom_d3_v1", 3, "pty_900s"), + ("gpt-5.6-sol", "simulation_heuristics_breakout_ram_v1", 4, "pty_900s"), +) + +RERUN_SCORE_CHANGES = ( + ("gpt-5.5", "simulation_heuristics_breakout_rgb_v1", 1, 21.299638989, 99.277978339), + ("gpt-5.5", "simulation_heuristics_ant_v1", 4, -1.398078834, 0.0), + ("gpt-5.5", "simulation_heuristics_montezuma_v1", 4, 0.0, 0.0), + ("gpt-5.5", "simulation_heuristics_pong_ram_v1", 4, 25.0, 50.0), + ("gpt-5.6-sol", "simulation_heuristics_halfcheetah_v1", 1, 26.12140084, 7.417455939), + ("gpt-5.6-sol", "simulation_heuristics_vizdoom_d3_v1", 1, 40.106951872, 4.879679144), + ("gpt-5.6-sol", "simulation_heuristics_ant_v1", 2, -7.591479, -13.495063777), + ("gpt-5.6-sol", "simulation_heuristics_vizdoom_d1_v1", 3, 31.015418049, 113.009039251), + ("gpt-5.6-sol", "simulation_heuristics_vizdoom_d3_v1", 3, 0.0, 22.961229947), + ("gpt-5.6-sol", "simulation_heuristics_breakout_ram_v1", 4, 18.448637317, 100.0), +) + +MODEL_IQM_CHANGES = ( + ("gpt-5.5", 10.662970787652174, 13.868245803826087), + ("gpt-5.6-sol", 36.651764607913044, 40.67930709160869), +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Audit timeout fairness of published article-suite cells." + ) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + return parser.parse_args() + + +def _task_document(task: str) -> dict[str, Any]: + path = REPO_ROOT / "tasks" / task / "task.md" + return yaml.safe_load(path.read_text().split("---", 2)[1]) + + +def main() -> None: + args = parse_args() + protocol = tomllib.loads(PROTOCOL_PATH.read_text()) + fairness = protocol["fairness"] + expected_idle = int(fairness["agent_idle_timeout_sec"]) + expected_pty = int(fairness["daytona_pty_readline_timeout_sec"]) + metadata_paths = sorted( + SUBMISSIONS_ROOT.glob("*/*/trial-*/metadata.json") + ) + + issues: list[dict[str, Any]] = [] + cells: list[dict[str, Any]] = [] + wall_clock_timeout_cells: list[dict[str, Any]] = [] + for path in metadata_paths: + metadata = json.loads(path.read_text()) + model_id = metadata["model"]["id"] + task = metadata["task"] + trial = int(metadata["trial"]) + task_document = _task_document(task) + expected_agent = int(task_document["agent"]["timeout_sec"]) + expected_verifier = int(task_document["verifier"]["timeout_sec"]) + expected_build = int( + task_document["environment"]["build_timeout_sec"] + ) + + cell = { + "model_id": model_id, + "task": task, + "trial": trial, + "published_protocol_version": metadata.get( + "published_protocol_version" + ), + "source_protocol_version": metadata.get("source_protocol_version"), + "agent_timeout_sec": metadata.get("agent_timeout_sec"), + "source_agent_idle_timeout_sec": metadata.get( + "source_agent_idle_timeout_sec" + ), + "source_daytona_pty_readline_timeout_sec": metadata.get( + "source_daytona_pty_readline_timeout_sec" + ), + "protocol_agent_idle_timeout_sec": metadata.get( + "protocol_agent_idle_timeout_sec" + ), + "protocol_daytona_pty_readline_timeout_sec": metadata.get( + "protocol_daytona_pty_readline_timeout_sec" + ), + "verifier_timeout_sec": expected_verifier, + "build_timeout_sec": expected_build, + "terminal_error_category": metadata.get( + "terminal_error_category" + ), + "agent_timeout_info": metadata.get("agent_timeout_info"), + "idle_timeout_info": metadata.get("idle_timeout_info"), + "transport_error_info": metadata.get("transport_error_info"), + "verifier_timeout_info": metadata.get("verifier_timeout_info"), + "source_run_id": metadata.get("source_run_id"), + } + cells.append(cell) + + checks = ( + ( + "protocol_version", + metadata.get("published_protocol_version"), + protocol["version"], + ), + ( + "agent_timeout_sec", + metadata.get("agent_timeout_sec"), + expected_agent, + ), + ( + "protocol_agent_idle_timeout_sec", + metadata.get("protocol_agent_idle_timeout_sec"), + expected_idle, + ), + ( + "protocol_daytona_pty_readline_timeout_sec", + metadata.get("protocol_daytona_pty_readline_timeout_sec"), + expected_pty, + ), + ) + for field, actual, expected in checks: + if actual != expected: + issues.append( + { + "model_id": model_id, + "task": task, + "trial": trial, + "field": field, + "actual": actual, + "expected": expected, + } + ) + + historical_key = (model_id, task, trial) + affected_keys = { + (affected_model, affected_task, affected_trial) + for ( + affected_model, + affected_task, + affected_trial, + _, + ) in HISTORICALLY_AFFECTED_CELLS + } + if historical_key in affected_keys: + for field, actual, expected in ( + ( + "source_protocol_version", + metadata.get("source_protocol_version"), + protocol["version"], + ), + ( + "source_agent_idle_timeout_sec", + metadata.get("source_agent_idle_timeout_sec"), + expected_idle, + ), + ( + "source_daytona_pty_readline_timeout_sec", + metadata.get("source_daytona_pty_readline_timeout_sec"), + expected_pty, + ), + ): + if actual != expected: + issues.append( + { + "model_id": model_id, + "task": task, + "trial": trial, + "field": field, + "actual": actual, + "expected": expected, + } + ) + + if metadata.get("idle_timeout_info") is not None: + issues.append( + { + "model_id": model_id, + "task": task, + "trial": trial, + "field": "idle_timeout_info", + "actual": metadata["idle_timeout_info"], + "expected": None, + } + ) + terminal_error = metadata.get("terminal_error") or "" + if "PTY readline timeout" in terminal_error: + issues.append( + { + "model_id": model_id, + "task": task, + "trial": trial, + "field": "terminal_error", + "actual": terminal_error, + "expected": "no PTY timeout", + } + ) + + agent_timeout = metadata.get("agent_timeout_info") + if agent_timeout is not None: + timeout_sec = agent_timeout.get("timeout_sec") + if ( + agent_timeout.get("reason") != "wall_clock_timeout" + or not isinstance(timeout_sec, int | float) + or not math.isclose(float(timeout_sec), expected_agent) + ): + issues.append( + { + "model_id": model_id, + "task": task, + "trial": trial, + "field": "agent_timeout_info", + "actual": agent_timeout, + "expected": { + "reason": "wall_clock_timeout", + "timeout_sec": expected_agent, + }, + } + ) + else: + wall_clock_timeout_cells.append( + { + "model_id": model_id, + "task": task, + "trial": trial, + "timeout_sec": timeout_sec, + } + ) + + expected_cells = 4 * 9 * int(protocol["trials"]) + if len(cells) != expected_cells: + issues.append( + { + "field": "selected_cell_count", + "actual": len(cells), + "expected": expected_cells, + } + ) + + report = { + "status": "ok" if not issues else "failed", + "generated_at": datetime.now(UTC).isoformat(), + "protocol_version": protocol["version"], + "expected_cells": expected_cells, + "selected_cells": len(cells), + "fairness": { + "agent_idle_timeout_sec": expected_idle, + "daytona_pty_readline_timeout_sec": expected_pty, + "interpretation": ( + "historically affected cells rerun under standardized " + "safeguards; unaffected source cells retained only when no " + "idle or PTY timeout influenced the selected result" + ), + }, + "historically_affected_cells": [ + { + "model_id": model_id, + "task": task, + "trial": trial, + "reason": reason, + } + for model_id, task, trial, reason in HISTORICALLY_AFFECTED_CELLS + ], + "rerun_cell_count": len(HISTORICALLY_AFFECTED_CELLS), + "rerun_score_changes": [ + { + "model_id": model_id, + "task": task, + "trial": trial, + "before_normalized_score": before, + "after_normalized_score": after, + "delta": after - before, + } + for model_id, task, trial, before, after in RERUN_SCORE_CHANGES + ], + "model_iqm_changes": [ + { + "model_id": model_id, + "before_iqm": before, + "after_iqm": after, + "delta": after - before, + } + for model_id, before, after in MODEL_IQM_CHANGES + ], + "selected_timeout_influenced_cells": issues, + "shared_wall_clock_timeout_cells": wall_clock_timeout_cells, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps(report, indent=2, sort_keys=True)) + if issues: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_article_suite_leaderboard.py b/scripts/build_article_suite_leaderboard.py index f1741d7..7322f6a 100644 --- a/scripts/build_article_suite_leaderboard.py +++ b/scripts/build_article_suite_leaderboard.py @@ -674,6 +674,12 @@ def _render_article_suite_markdown( "documented in " "[`docs/article-suite-scoring.md`](../docs/article-suite-scoring.md).", "", + "The selected trajectory timeout audit is available in " + "[`article_suite_timeout_fairness_audit.json`]" + "(article_suite_timeout_fairness_audit.json) and " + "[`docs/article-suite-timeout-fairness.md`]" + "(../docs/article-suite-timeout-fairness.md).", + "", ] ) @@ -974,14 +980,28 @@ def main() -> None: source_score = ( rollout_dir / "verifier" / "genesis-score.json" ) + source_result = rollout_dir / "result.json" + source_config = rollout_dir / "config.json" if not source_score.is_file(): raise RuntimeError( f"{model_id}/{task}/trial-{trial:02d} has no " "verifier genesis-score.json" ) + if not source_result.is_file(): + raise RuntimeError( + f"{model_id}/{task}/trial-{trial:02d} has no " + "result.json" + ) + if not source_config.is_file(): + raise RuntimeError( + f"{model_id}/{task}/trial-{trial:02d} has no " + "config.json" + ) sanitized_score = _sanitize_score_paths( json.loads(source_score.read_text()) ) + result_payload = json.loads(source_result.read_text()) + config_payload = json.loads(source_config.read_text()) normalized = _normalized_task_score(result_row) try: ( @@ -1025,6 +1045,28 @@ def main() -> None: "provider_reasoning_effort": task_metadata[ "provider_reasoning_effort" ], + "published_protocol_version": protocol["version"], + "source_protocol_version": task_metadata.get( + "protocol", {} + ).get("version"), + "agent_timeout_sec": task_metadata[ + "task_agent_timeout_sec" + ][task], + "source_agent_idle_timeout_sec": config_payload.get( + "agent_idle_timeout_sec" + ), + "source_daytona_pty_readline_timeout_sec": ( + task_metadata["model"].get( + "daytona_pty_readline_timeout_sec", + 900, + ) + ), + "protocol_agent_idle_timeout_sec": protocol[ + "fairness" + ]["agent_idle_timeout_sec"], + "protocol_daytona_pty_readline_timeout_sec": protocol[ + "fairness" + ]["daytona_pty_readline_timeout_sec"], "normalized_score": normalized, "raw_score": float(raw_score), "observed_raw_score": observed_raw_score, @@ -1036,6 +1078,23 @@ def main() -> None: result_row.get("total_tool_calls"), ), "token_usage": result_row.get("token_usage"), + "terminal_error_category": result_payload.get( + "error_category" + ), + "terminal_error": result_payload.get("error"), + "agent_timeout_info": result_payload.get( + "agent_timeout_info" + ), + "idle_timeout_info": result_payload.get( + "idle_timeout_info" + ), + "transport_error_info": result_payload.get( + "transport_error_info" + ), + "verifier_timeout_info": result_payload.get( + "verifier_timeout_info" + ), + "timing": result_payload.get("timing"), } (trial_destination / "metadata.json").write_text( json.dumps( diff --git a/scripts/run_article_suite.py b/scripts/run_article_suite.py index d5ea14e..f8faf78 100644 --- a/scripts/run_article_suite.py +++ b/scripts/run_article_suite.py @@ -204,6 +204,24 @@ def _validate_protocol(tasks: tuple[str, ...], trials: int) -> dict[str, Any]: return protocol +def _validate_model_timeouts( + models: list[dict[str, Any]], + protocol: dict[str, Any], +) -> None: + fairness = protocol["fairness"] + expected_idle = int(fairness["agent_idle_timeout_sec"]) + expected_pty = int(fairness["daytona_pty_readline_timeout_sec"]) + for model in models: + idle = model.get("agent_idle_timeout_sec") + pty = model.get("daytona_pty_readline_timeout_sec") + if idle != expected_idle or pty != expected_pty: + raise RuntimeError( + f"{model['id']} timeout safeguards must match protocol.toml: " + f"idle={expected_idle}, Daytona PTY={expected_pty}; " + f"got idle={idle!r}, Daytona PTY={pty!r}" + ) + + def _selected_trials( requested: list[int] | None, *, @@ -453,6 +471,7 @@ def main() -> None: protocol = _validate_protocol(tasks, args.trials) selected_trials = _selected_trials(args.trial, trial_count=args.trials) models = _select_models(args.model, args.all_models) + _validate_model_timeouts(models, protocol) provider_env = os.environ.copy() provider_env.update(_read_env(args.env_file.expanduser())) if args.sandbox == "daytona": diff --git a/tests/test_article_suite_tooling.py b/tests/test_article_suite_tooling.py index 0d40804..58e7a1b 100644 --- a/tests/test_article_suite_tooling.py +++ b/tests/test_article_suite_tooling.py @@ -22,9 +22,13 @@ def test_article_suite_declares_exactly_nine_unique_tasks() -> None: def test_article_suite_protocol_uses_five_trials_and_triple_timeouts() -> None: protocol = runner._protocol() - assert protocol["version"] == "2.1" + assert protocol["version"] == "2.2" assert protocol["trials"] == 5 assert protocol["agent_timeout_multiplier"] == 3 + assert protocol["fairness"] == { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600, + } runner._validate_protocol(runner.TASKS, protocol["trials"]) with pytest.raises(ValueError, match="must match protocol.toml"): runner._validate_protocol(runner.TASKS, 4) @@ -42,6 +46,19 @@ def test_article_suite_defaults_to_daytona( assert runner.parse_args().sandbox == "daytona" +def test_all_models_share_fair_idle_and_pty_timeouts() -> None: + models = runner._models() + protocol = runner._protocol() + + runner._validate_model_timeouts(models, protocol) + assert { + model["agent_idle_timeout_sec"] for model in models + } == {3600} + assert { + model["daytona_pty_readline_timeout_sec"] for model in models + } == {3600} + + def test_execution_protocol_ignores_posthoc_aggregation_changes() -> None: current = runner._protocol() prior_scoring = { diff --git a/tests/test_leaderboard_artifacts.py b/tests/test_leaderboard_artifacts.py index fca47c9..fe81ee2 100644 --- a/tests/test_leaderboard_artifacts.py +++ b/tests/test_leaderboard_artifacts.py @@ -15,6 +15,9 @@ ROOT_README = REPO_ROOT / "README.md" WEBSITE = REPO_ROOT / "website" / "index.html" WEBSITE_DATA = REPO_ROOT / "website" / "assets" / "article_suite.json" +TIMEOUT_AUDIT = ( + REPO_ROOT / "leaderboard" / "article_suite_timeout_fairness_audit.json" +) TASK_IMAGE = REPO_ROOT / "leaderboard" / "article_suite_task_leaderboards.png" FINAL_IMAGE = REPO_ROOT / "leaderboard" / "article_suite_final_leaderboard.png" @@ -64,7 +67,7 @@ def test_article_suite_leaderboard_is_complete_and_self_contained() -> None: assert payload["aggregation"]["primary_metric"] == "interquartile_mean" assert payload["aggregation"]["trim_fraction_per_tail"] == 0.25 if "protocol" in payload: - assert payload["protocol"]["version"] == "2.1" + assert payload["protocol"]["version"] == "2.2" assert payload["protocol"]["trials"] == 5 assert payload["protocol"]["agent_timeout_multiplier"] == 3 assert payload["execution"] == { @@ -303,3 +306,16 @@ def assert_no_absolute_artifact_paths(value: object) -> None: width, height = _png_dimensions(source) assert width >= 1500 assert height >= 800 + + +def test_timeout_fairness_audit_is_clean() -> None: + payload = json.loads(TIMEOUT_AUDIT.read_text()) + + assert payload["status"] == "ok" + assert payload["protocol_version"] == "2.2" + assert payload["selected_cells"] == 180 + assert payload["expected_cells"] == 180 + assert payload["rerun_cell_count"] == 10 + assert payload["selected_timeout_influenced_cells"] == [] + assert payload["fairness"]["agent_idle_timeout_sec"] == 3600 + assert payload["fairness"]["daytona_pty_readline_timeout_sec"] == 3600 diff --git a/website/assets/article_suite.json b/website/assets/article_suite.json index 37b8aa1..a358ed9 100644 --- a/website/assets/article_suite.json +++ b/website/assets/article_suite.json @@ -39,7 +39,7 @@ "daytona" ] }, - "generated_at": "2026-07-15T12:56:18.690710+00:00", + "generated_at": "2026-07-15T19:02:46.670420+00:00", "inference_settings": { "cross_provider_comparability": "labels_are_not_a_shared_numeric_compute_scale", "field": "provider_reasoning_effort", @@ -130,20 +130,20 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": -4.9009087386, - "normalized_score_stddev": 11.991465810903719, + "normalized_score": -6.081625694, + "normalized_score_stddev": 12.597925216329989, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 2, - "raw_score": 2272.356693724495, + "raw_score": 2231.0491055302227, "raw_score_imputation_count": 0, - "raw_score_stddev": 419.52351859557746, + "raw_score_stddev": 440.7406064520731, "reference_score": 5942.333244755838, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" @@ -157,22 +157,22 @@ "model": "GPT-5.5", "model_id": "gpt-5.5", "model_route": "azure/gpt-5.5", - "normalized_score": -12.5116242462, - "normalized_score_stddev": 8.756962566015595, + "normalized_score": -12.2320084794, + "normalized_score_stddev": 9.211087046248206, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "xhigh", "rank": 3, - "raw_score": 2006.0944869926293, + "raw_score": 2015.8768932664839, "raw_score_imputation_count": 0, - "raw_score_stddev": 306.36385957656836, + "raw_score_stddev": 322.25148356656894, "reference_score": 5942.333244755838, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4" ], "starter_score": 2443.815838992282, @@ -276,22 +276,22 @@ "model": "GPT-5.5", "model_id": "gpt-5.5", "model_route": "azure/gpt-5.5", - "normalized_score": 49.1666666668, - "normalized_score_stddev": 40.44028519521613, + "normalized_score": 54.166666666800005, + "normalized_score_stddev": 38.188130791325946, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "xhigh", "rank": 3, - "raw_score": 8.8, + "raw_score": 10.0, "raw_score_imputation_count": 0, - "raw_score_stddev": 9.705668446840743, + "raw_score_stddev": 9.16515138991168, "reference_score": 21.0, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "starter_score": -2.9999999999999996, @@ -368,22 +368,22 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": 43.899371069400004, - "normalized_score_stddev": 47.34804435501101, + "normalized_score": 60.209643606, + "normalized_score_stddev": 50.34075134460858, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 2, - "raw_score": 596.4, + "raw_score": 674.2, "raw_score_imputation_count": 0, - "raw_score_stddev": 225.8501715739884, + "raw_score_stddev": 240.1253839143209, "reference_score": 864.0, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "starter_score": 387.0, @@ -487,19 +487,19 @@ "model": "GPT-5.5", "model_id": "gpt-5.5", "model_route": "azure/gpt-5.5", - "normalized_score": 60.86642599300001, - "normalized_score_stddev": 22.118506348049813, + "normalized_score": 76.462093863, + "normalized_score_stddev": 12.754467163779541, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "xhigh", "rank": 2, - "raw_score": 647.2, + "raw_score": 733.6, "raw_score_imputation_count": 0, - "raw_score_stddev": 122.53652516698847, + "raw_score_stddev": 70.65974808899335, "reference_score": 864.0, "sandbox": "daytona", "source_run": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", @@ -519,7 +519,7 @@ "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", - "rank": 2, + "rank": 3, "raw_score": 647.2, "raw_score_imputation_count": 0, "raw_score_stddev": 122.53652516698847, @@ -606,19 +606,19 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": 31.8442100886, - "normalized_score_stddev": 3.8873875677992533, + "normalized_score": 28.1034211084, + "normalized_score_stddev": 11.772801175021295, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 2, - "raw_score": 6385.135940620764, + "raw_score": 6105.10168077257, "raw_score_imputation_count": 0, - "raw_score_stddev": 291.0085829715705, + "raw_score_stddev": 881.3081092982512, "reference_score": 11487.256451471967, "sandbox": "daytona", "source_run": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", @@ -725,21 +725,21 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": 56.834663982799995, - "normalized_score_stddev": 33.111283305681766, + "normalized_score": 73.23338822320001, + "normalized_score_stddev": 37.18119323399795, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 2, - "raw_score": 0.6554319871515035, + "raw_score": 0.7171239853203296, "raw_score_imputation_count": 0, - "raw_score_stddev": 0.12456464290218099, + "raw_score_stddev": 0.13987564345039064, "reference_score": 0.8178199748694898, "sandbox": "daytona", "source_run": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], @@ -898,21 +898,21 @@ "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "normalized_score": 16.3034759358, - "normalized_score_stddev": 23.995771403261536, + "normalized_score": 13.850267379600002, + "normalized_score_stddev": 20.096747543003065, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", "rank": 4, - "raw_score": 80.67999999999999, + "raw_score": 73.34, "raw_score_imputation_count": 0, - "raw_score_stddev": 71.79534803871347, + "raw_score_stddev": 60.12946864890792, "reference_score": 331.1, "sandbox": "daytona", "source_run": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], @@ -1151,7 +1151,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "starter_score": 0.0, @@ -1192,17 +1192,17 @@ "trial_count": 5 }, { - "arithmetic_mean_normalized_score": 41.223767221955555, - "average_normalized_score": 41.223767221955555, - "final_normalized_score": 36.651764607913044, - "final_normalized_score_stddev": 5.524217450335815, + "arithmetic_mean_normalized_score": 44.038687475866666, + "average_normalized_score": 44.038687475866666, + "final_normalized_score": 40.67930709160869, + "final_normalized_score_stddev": 11.031024092730283, "harness": "opencode", - "mean_trial_iqm_normalized_score": 37.73164397808, - "median_normalized_score": 32.792734613, + "mean_trial_iqm_normalized_score": 41.6809489228, + "median_normalized_score": 33.28729024, "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", - "positive_display_score": 136.65176460791304, + "positive_display_score": 140.67930709160868, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "max", @@ -1211,17 +1211,17 @@ "trial_count": 5 }, { - "arithmetic_mean_normalized_score": 22.06816884377778, - "average_normalized_score": 22.06816884377778, - "final_normalized_score": 10.662970787652174, - "final_normalized_score_stddev": 18.67984640949238, + "arithmetic_mean_normalized_score": 24.387644803422226, + "average_normalized_score": 24.387644803422226, + "final_normalized_score": 13.868245803826087, + "final_normalized_score_stddev": 18.296053164135497, "harness": "opencode", - "mean_trial_iqm_normalized_score": 16.7775760408, + "mean_trial_iqm_normalized_score": 17.8186386096, "median_normalized_score": 5.870020964, "model": "GPT-5.5", "model_id": "gpt-5.5", "model_route": "azure/gpt-5.5", - "positive_display_score": 110.66297078765217, + "positive_display_score": 113.86824580382608, "provider": "azure", "provider_label": "Azure direct", "provider_reasoning_effort": "xhigh", @@ -1271,8 +1271,12 @@ "simulation_heuristics_vizdoom_d1_v1": 1800, "simulation_heuristics_vizdoom_d3_v1": 1800 }, + "fairness": { + "agent_idle_timeout_sec": 3600, + "daytona_pty_readline_timeout_sec": 3600 + }, "trials": 5, - "version": "2.1" + "version": "2.2" }, "rows": [ { @@ -1527,13 +1531,13 @@ } }, { - "arithmetic_mean_normalized_score": 41.223767221955555, - "average_normalized_score": 41.223767221955555, - "final_normalized_score": 36.651764607913044, - "final_normalized_score_stddev": 5.524217450335815, + "arithmetic_mean_normalized_score": 44.038687475866666, + "average_normalized_score": 44.038687475866666, + "final_normalized_score": 40.67930709160869, + "final_normalized_score_stddev": 11.031024092730283, "harness": "opencode", - "mean_trial_iqm_normalized_score": 37.73164397808, - "median_normalized_score": 32.792734613, + "mean_trial_iqm_normalized_score": 41.6809489228, + "median_normalized_score": 33.28729024, "model": "GPT-5.6 Sol", "model_id": "gpt-5.6-sol", "model_route": "azure/gpt-5.6-sol", @@ -1553,32 +1557,32 @@ "simulation_heuristics_vizdoom_d3_v1": 0 }, "raw_task_score_stddevs": { - "simulation_heuristics_ant_v1": 419.52351859557746, + "simulation_heuristics_ant_v1": 440.7406064520731, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 225.8501715739884, + "simulation_heuristics_breakout_ram_v1": 240.1253839143209, "simulation_heuristics_breakout_rgb_v1": 122.53652516698847, - "simulation_heuristics_halfcheetah_v1": 291.0085829715705, + "simulation_heuristics_halfcheetah_v1": 881.3081092982512, "simulation_heuristics_montezuma_v1": 173.26280616450836, "simulation_heuristics_pong_ram_v1": 2.345207879911715, - "simulation_heuristics_vizdoom_d1_v1": 0.12456464290218099, - "simulation_heuristics_vizdoom_d3_v1": 71.79534803871347 + "simulation_heuristics_vizdoom_d1_v1": 0.13987564345039064, + "simulation_heuristics_vizdoom_d3_v1": 60.12946864890792 }, "raw_task_scores": { - "simulation_heuristics_ant_v1": 2272.356693724495, + "simulation_heuristics_ant_v1": 2231.0491055302227, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 596.4, + "simulation_heuristics_breakout_ram_v1": 674.2, "simulation_heuristics_breakout_rgb_v1": 647.2, - "simulation_heuristics_halfcheetah_v1": 6385.135940620764, + "simulation_heuristics_halfcheetah_v1": 6105.10168077257, "simulation_heuristics_montezuma_v1": 298.0, "simulation_heuristics_pong_ram_v1": 19.0, - "simulation_heuristics_vizdoom_d1_v1": 0.6554319871515035, - "simulation_heuristics_vizdoom_d3_v1": 80.67999999999999 + "simulation_heuristics_vizdoom_d1_v1": 0.7171239853203296, + "simulation_heuristics_vizdoom_d3_v1": 73.34 }, "sandbox": "daytona", "source_runs": { "simulation_heuristics_ant_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" @@ -1594,7 +1598,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_breakout_rgb_v1": [ @@ -1605,7 +1609,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_halfcheetah_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", @@ -1628,14 +1632,14 @@ "simulation_heuristics_vizdoom_d1_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_vizdoom_d3_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ] @@ -1690,33 +1694,33 @@ } }, "task_score_stddevs": { - "simulation_heuristics_ant_v1": 11.991465810903719, + "simulation_heuristics_ant_v1": 12.597925216329989, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 47.34804435501101, + "simulation_heuristics_breakout_ram_v1": 50.34075134460858, "simulation_heuristics_breakout_rgb_v1": 22.118506348049813, - "simulation_heuristics_halfcheetah_v1": 3.8873875677992533, + "simulation_heuristics_halfcheetah_v1": 11.772801175021295, "simulation_heuristics_montezuma_v1": 43.31570154112709, "simulation_heuristics_pong_ram_v1": 9.771699499561077, - "simulation_heuristics_vizdoom_d1_v1": 33.111283305681766, - "simulation_heuristics_vizdoom_d3_v1": 23.995771403261536 + "simulation_heuristics_vizdoom_d1_v1": 37.18119323399795, + "simulation_heuristics_vizdoom_d3_v1": 20.096747543003065 }, "task_scores": { - "simulation_heuristics_ant_v1": -4.9009087386, + "simulation_heuristics_ant_v1": -6.081625694, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 43.899371069400004, + "simulation_heuristics_breakout_ram_v1": 60.209643606, "simulation_heuristics_breakout_rgb_v1": 60.86642599300001, - "simulation_heuristics_halfcheetah_v1": 31.8442100886, + "simulation_heuristics_halfcheetah_v1": 28.1034211084, "simulation_heuristics_montezuma_v1": 74.5, "simulation_heuristics_pong_ram_v1": 91.6666666666, - "simulation_heuristics_vizdoom_d1_v1": 56.834663982799995, - "simulation_heuristics_vizdoom_d3_v1": 16.3034759358 + "simulation_heuristics_vizdoom_d1_v1": 73.23338822320001, + "simulation_heuristics_vizdoom_d3_v1": 13.850267379600002 }, "trial_count": 5, "trial_final_normalized_scores": { - "1": 37.797729594399996, + "1": 27.0114860686, "2": 41.203895079, - "3": 28.388586371000002, - "4": 42.346736302800004, + "3": 43.444415417200005, + "4": 57.82367550600001, "5": 38.921272543200004 }, "trial_task_normalized_scores": { @@ -1725,14 +1729,14 @@ "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 100.0, "simulation_heuristics_breakout_rgb_v1": 70.758122744, - "simulation_heuristics_halfcheetah_v1": 26.12140084, + "simulation_heuristics_halfcheetah_v1": 7.417455939, "simulation_heuristics_montezuma_v1": 0.0, "simulation_heuristics_pong_ram_v1": 79.166666667, "simulation_heuristics_vizdoom_d1_v1": 42.828284045, - "simulation_heuristics_vizdoom_d3_v1": 40.106951872 + "simulation_heuristics_vizdoom_d3_v1": 4.879679144 }, "2": { - "simulation_heuristics_ant_v1": -7.591479, + "simulation_heuristics_ant_v1": -13.495063777, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 6.918238994, "simulation_heuristics_breakout_rgb_v1": 70.758122744, @@ -1750,13 +1754,13 @@ "simulation_heuristics_halfcheetah_v1": 36.605449762, "simulation_heuristics_montezuma_v1": 100.0, "simulation_heuristics_pong_ram_v1": 83.333333333, - "simulation_heuristics_vizdoom_d1_v1": 31.015418049, - "simulation_heuristics_vizdoom_d3_v1": 0.0 + "simulation_heuristics_vizdoom_d1_v1": 113.009039251, + "simulation_heuristics_vizdoom_d3_v1": 22.961229947 }, "4": { "simulation_heuristics_ant_v1": -19.696444908, "simulation_heuristics_atari57_v1": 0.0, - "simulation_heuristics_breakout_ram_v1": 18.448637317, + "simulation_heuristics_breakout_ram_v1": 100.0, "simulation_heuristics_breakout_rgb_v1": 70.758122744, "simulation_heuristics_halfcheetah_v1": 32.792734613, "simulation_heuristics_montezuma_v1": 100.0, @@ -1778,12 +1782,12 @@ } }, { - "arithmetic_mean_normalized_score": 22.06816884377778, - "average_normalized_score": 22.06816884377778, - "final_normalized_score": 10.662970787652174, - "final_normalized_score_stddev": 18.67984640949238, + "arithmetic_mean_normalized_score": 24.387644803422226, + "average_normalized_score": 24.387644803422226, + "final_normalized_score": 13.868245803826087, + "final_normalized_score_stddev": 18.296053164135497, "harness": "opencode", - "mean_trial_iqm_normalized_score": 16.7775760408, + "mean_trial_iqm_normalized_score": 17.8186386096, "median_normalized_score": 5.870020964, "model": "GPT-5.5", "model_id": "gpt-5.5", @@ -1804,24 +1808,24 @@ "simulation_heuristics_vizdoom_d3_v1": 0 }, "raw_task_score_stddevs": { - "simulation_heuristics_ant_v1": 306.36385957656836, + "simulation_heuristics_ant_v1": 322.25148356656894, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 131.153345363357, - "simulation_heuristics_breakout_rgb_v1": 122.53652516698847, + "simulation_heuristics_breakout_rgb_v1": 70.65974808899335, "simulation_heuristics_halfcheetah_v1": 698.5070322948952, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 9.705668446840743, + "simulation_heuristics_pong_ram_v1": 9.16515138991168, "simulation_heuristics_vizdoom_d1_v1": 0.3579053781713541, "simulation_heuristics_vizdoom_d3_v1": 122.81457568220475 }, "raw_task_scores": { - "simulation_heuristics_ant_v1": 2006.0944869926293, + "simulation_heuristics_ant_v1": 2015.8768932664839, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 486.8, - "simulation_heuristics_breakout_rgb_v1": 647.2, + "simulation_heuristics_breakout_rgb_v1": 733.6, "simulation_heuristics_halfcheetah_v1": 4717.6162449605235, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 8.8, + "simulation_heuristics_pong_ram_v1": 10.0, "simulation_heuristics_vizdoom_d1_v1": 0.6197079806402326, "simulation_heuristics_vizdoom_d3_v1": 101.49999999999999 }, @@ -1831,7 +1835,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4" ], "simulation_heuristics_atari57_v1": [ @@ -1849,7 +1853,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_breakout_rgb_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", @@ -1866,14 +1870,14 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_pong_ram_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_vizdoom_d1_v1": [ @@ -1941,33 +1945,33 @@ } }, "task_score_stddevs": { - "simulation_heuristics_ant_v1": 8.756962566015595, + "simulation_heuristics_ant_v1": 9.211087046248206, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 27.49546024414613, - "simulation_heuristics_breakout_rgb_v1": 22.118506348049813, + "simulation_heuristics_breakout_rgb_v1": 12.754467163779541, "simulation_heuristics_halfcheetah_v1": 9.330884765209642, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 40.44028519521613, + "simulation_heuristics_pong_ram_v1": 38.188130791325946, "simulation_heuristics_vizdoom_d1_v1": 95.13699952993868, "simulation_heuristics_vizdoom_d3_v1": 41.04765230004701 }, "task_scores": { - "simulation_heuristics_ant_v1": -12.5116242462, + "simulation_heuristics_ant_v1": -12.2320084794, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 20.9224318658, - "simulation_heuristics_breakout_rgb_v1": 60.86642599300001, + "simulation_heuristics_breakout_rgb_v1": 76.462093863, "simulation_heuristics_halfcheetah_v1": 9.5689381094, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 49.1666666668, + "simulation_heuristics_pong_ram_v1": 54.166666666800005, "simulation_heuristics_vizdoom_d1_v1": 47.33864911959999, "simulation_heuristics_vizdoom_d3_v1": 23.2620320856 }, "trial_count": 5, "trial_final_normalized_scores": { - "1": 5.4339319906, + "1": 5.6392448345999995, "2": 48.2436373978, "3": 18.718893057000002, - "4": 9.604625305799999, + "4": 14.604625305799999, "5": 1.8867924528 }, "trial_task_normalized_scores": { @@ -1975,7 +1979,7 @@ "simulation_heuristics_ant_v1": -6.720960755, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 5.870020964, - "simulation_heuristics_breakout_rgb_v1": 21.299638989, + "simulation_heuristics_breakout_rgb_v1": 99.277978339, "simulation_heuristics_halfcheetah_v1": 0.0, "simulation_heuristics_montezuma_v1": 0.0, "simulation_heuristics_pong_ram_v1": 41.666666667, @@ -2005,13 +2009,13 @@ "simulation_heuristics_vizdoom_d3_v1": -0.601604278 }, "4": { - "simulation_heuristics_ant_v1": -1.398078834, + "simulation_heuristics_ant_v1": 0.0, "simulation_heuristics_atari57_v1": 0.0, "simulation_heuristics_breakout_ram_v1": 9.224318658, "simulation_heuristics_breakout_rgb_v1": 70.758122744, "simulation_heuristics_halfcheetah_v1": 13.798807871, "simulation_heuristics_montezuma_v1": 0.0, - "simulation_heuristics_pong_ram_v1": 25.0, + "simulation_heuristics_pong_ram_v1": 50.0, "simulation_heuristics_vizdoom_d1_v1": 111.839447567, "simulation_heuristics_vizdoom_d3_v1": 0.0 }, @@ -2416,7 +2420,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05/tasks-81ec8d98a4" ], "simulation_heuristics_atari57_v1": [ @@ -2434,7 +2438,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_breakout_rgb_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01/simulation_heuristics_breakout_rgb_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", @@ -2451,14 +2455,14 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_pong_ram_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-885696cfa4", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-04/tasks-e2ced31f9c", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt55/gpt-5.5/trial-05" ], "simulation_heuristics_vizdoom_d1_v1": [ @@ -2479,7 +2483,7 @@ "gpt-5.6-sol": { "simulation_heuristics_ant_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02/simulation_heuristics_ant_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" @@ -2495,7 +2499,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/simulation_heuristics_breakout_ram_v1", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_breakout_rgb_v1": [ @@ -2506,7 +2510,7 @@ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_halfcheetah_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04/tasks-81ec8d98a4", @@ -2529,14 +2533,14 @@ "simulation_heuristics_vizdoom_d1_v1": [ "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ], "simulation_heuristics_vizdoom_d3_v1": [ - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-01/tasks-786d2dde21", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-02", - "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03", + "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-03/tasks-8869084d04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-04", "leaderboard/runs/article_suite/article-suite-v2-daytona-20260714-gpt56/gpt-5.6-sol/trial-05" ] diff --git a/website/index.html b/website/index.html index 37b2b53..397a43a 100644 --- a/website/index.html +++ b/website/index.html @@ -515,6 +515,7 @@

Final normalized score

Detailed leaderboard artifacts ↗ Scoring methodology ↗ + Timeout fairness audit ↗