From 13e52d0af23345c7f971da52bce095b63d9e7297 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Tue, 14 Jul 2026 11:50:57 -0700 Subject: [PATCH 01/15] feat: add five-trial article suite protocol --- README.md | 12 +- docs/article-suite-scoring.md | 40 +- docs/learning-beyond-gradients-suite.md | 10 +- ...imulation-heuristics-ant-v1-task-design.md | 3 + experiments/article_suite/README.md | 58 +- experiments/article_suite/protocol.toml | 21 + scripts/build_article_suite_leaderboard.py | 526 ++++++++++++++---- scripts/plot_article_suite_leaderboards.py | 66 ++- scripts/run_article_suite.py | 373 ++++++++++--- tasks/simulation_heuristics_ant_v1/task.md | 2 +- .../simulation_heuristics_atari57_v1/task.md | 2 +- .../task.md | 2 +- .../task.md | 2 +- .../task.md | 2 +- .../task.md | 2 +- .../simulation_heuristics_pong_ram_v1/task.md | 2 +- .../task.md | 2 +- .../task.md | 2 +- tests/test_article_suite_tooling.py | 131 ++++- website/README.md | 7 +- website/index.html | 10 +- 21 files changed, 1004 insertions(+), 271 deletions(-) create mode 100644 experiments/article_suite/protocol.toml diff --git a/README.md b/README.md index 824a1a3..9a5497e 100644 --- a/README.md +++ b/README.md @@ -128,10 +128,12 @@ Scores are unbounded normalized values: `0` matches the public starter and `100` matches the trusted article-level reference. Negative scores are genuine regressions; scores above `100` exceed the reference. -The final ranking uses the interquartile mean (IQM) of the nine task scores: -sort them, remove the lowest two and highest two, then average the middle five. -The chart uses a plot-only positive index equal to `IQM + 100`; the raw IQM -remains the ranking metric and is retained in the JSON. +Each model runs five independent trials. Within each trial, the final ranking +uses the interquartile mean (IQM) of the nine task scores: sort them, remove the +lowest two and highest two, then average the middle five. The published score +is the mean of the five trial IQMs and includes sample standard deviation. The +chart uses a plot-only positive index equal to `IQM + 100`; raw IQM remains in +the JSON. Inference settings are provider-specific: GPT-5.6 Sol and Claude Opus 4.8 use `max`; GPT-5.5 and GPT-5.4 Mini use `xhigh`. These labels come from different @@ -140,7 +142,7 @@ scale. Exact routes are documented in [`docs/learning-beyond-gradients-suite.md`](docs/learning-beyond-gradients-suite.md). See [`docs/article-suite-scoring.md`](docs/article-suite-scoring.md) for the -formula, research precedent, and current single-run limitation. +formula, research precedent, and statistical limitations. ## Contribute a Task diff --git a/docs/article-suite-scoring.md b/docs/article-suite-scoring.md index 9ad74f4..e7f8209 100644 --- a/docs/article-suite-scoring.md +++ b/docs/article-suite-scoring.md @@ -23,20 +23,33 @@ This follows the same family of baseline normalization used by benchmarks such as [D4RL](https://arxiv.org/abs/2004.07219), which normalizes returns between task-specific lower and upper reference scores. +## Five-trial estimator + +Each model runs the complete nine-task suite five independent times. For task +`t`, the published task score is the arithmetic mean of its five normalized +trial scores. The task leaderboard also publishes their sample standard +deviation. + ## Final normalized score -The primary final metric is the interquartile mean (IQM), implemented as the -25% trimmed mean used by +Within each trial, the primary cross-task metric is the interquartile mean +(IQM), implemented as the 25% trimmed mean used by [RLiable](https://github.com/google-research/rliable): ```text -scores = sort(the nine normalized task scores) +scores = sort(the nine normalized task scores for one trial) trim_count = floor(0.25 * 9) = 2 -final_normalized_score = mean(scores[2:7]) +trial_iqm = mean(scores[2:7]) ``` -In words: remove the lowest two and highest two task scores and average the -middle five. +The final model score is: + +```text +final_normalized_score = mean(trial_iqm[1:5]) +``` + +The leaderboard also publishes the sample standard deviation of the five +trial-level IQMs. The JSON also publishes: @@ -88,13 +101,10 @@ endpoints and would rewrite every displayed score whenever the comparison set changes. Clipping negative IQM values to zero would erase meaningful differences. -## Current statistical limitation - -The published sweep currently has one independent agent run per model/task -pair. That is sufficient to recompute deterministic aggregate metrics, but not -to estimate statistically meaningful bootstrap confidence intervals or -probability of improvement between models. +## Statistical limitation -A future multi-run release should retain IQM and additionally report -stratified-bootstrap confidence intervals, performance profiles, and pairwise -probability of improvement following the RLiable protocol. +Five trials expose run-to-run variability and are materially stronger than a +single run, but they remain a small sample. The leaderboard reports sample +standard deviation rather than claiming narrow confidence intervals. A larger +future release can add stratified-bootstrap confidence intervals, performance +profiles, and pairwise probability of improvement following RLiable. diff --git a/docs/learning-beyond-gradients-suite.md b/docs/learning-beyond-gradients-suite.md index 17e9e30..fc2a5de 100644 --- a/docs/learning-beyond-gradients-suite.md +++ b/docs/learning-beyond-gradients-suite.md @@ -79,10 +79,12 @@ Each task first receives an unbounded anchor-normalized score: task score = 100 * (candidate - starter) / (reference - starter) ``` -The primary final score is the 25% trimmed interquartile mean (IQM). With nine -tasks, the two lowest and two highest normalized scores are removed and the -middle five are averaged. Arithmetic mean and median remain secondary -diagnostics. +Each model runs five independent trials. Task leaderboards report mean ± sample +standard deviation across those trials. Within each trial, the primary +cross-task score is the 25% trimmed interquartile mean (IQM): remove the two +lowest and two highest normalized task scores and average the middle five. The +final model score is the mean ± sample standard deviation of the five +trial-level IQMs. The repository README intentionally shows only the final leaderboard image. The nine task panels use each environment's native raw score. The final chart diff --git a/docs/simulation-heuristics-ant-v1-task-design.md b/docs/simulation-heuristics-ant-v1-task-design.md index 4516a87..4c9a1b3 100644 --- a/docs/simulation-heuristics-ant-v1-task-design.md +++ b/docs/simulation-heuristics-ant-v1-task-design.md @@ -176,6 +176,9 @@ The first published sweep uses: - isolated Docker workspaces; - each model's highest supported reasoning level. +The current article-suite protocol triples the Ant agent timeout to 90 minutes +and reports mean ± sample standard deviation across five independent trials. + Simulation Heuristics Ant v1 currently enforces wall-clock time but does not centrally meter every simulator step. Future hosted versions should add an authoritative interaction meter before making sample-efficiency claims across diff --git a/experiments/article_suite/README.md b/experiments/article_suite/README.md index cdc3125..623db9b 100644 --- a/experiments/article_suite/README.md +++ b/experiments/article_suite/README.md @@ -16,6 +16,33 @@ the nine task packages derived from the article: Every run uses BenchFlow's registered `opencode` ACP harness. OpenHands is not part of this suite. +## Experiment protocol + +The current leaderboard protocol is defined in `protocol.toml`: + +- five independent full-suite trials per model; +- three times the original agent wall-clock timeout for every task; +- task scores reported as mean ± sample standard deviation across trials; +- one nine-task IQM computed per trial; +- final score reported as mean ± sample standard deviation across the five + trial-level IQMs. + +The resulting matrix is: + +```text +4 models × 5 trials × 9 tasks = 180 model-task runs +``` + +| Tasks | Previous cap | Current cap | +| --- | ---: | ---: | +| Ant, Pong, both Breakouts, Montezuma, VizDoom D1/D3 | 30 min | 90 min | +| HalfCheetah | 60 min | 180 min | +| Atari57 aggregate search | 7 days | 21 days | + +Atari57's cap is exceptional because one task contains 342 accounted search +slots. It can be scheduled separately when operating capacity is limited; the +cap is a maximum, not an expected runtime. + ## Inference settings | Model | Exact route | Provider-specific reasoning setting | @@ -51,7 +78,8 @@ CLAUDE_CODE_OAUTH_TOKEN ## Run Docker must be running because the authoritative suite uses isolated BenchFlow -task environments: +task environments. Local Docker runs default to calibrated `linux/amd64` +images, including on Apple Silicon: ```bash uv run python scripts/run_article_suite.py \ @@ -59,6 +87,10 @@ uv run python scripts/run_article_suite.py \ --model gpt-5.6-sol ``` +Use `--docker-platform` only when a task's verifier has a matching trusted +platform calibration. The current VizDoom reproducibility anchors support +`linux-x86_64`, so uncalibrated `linux-arm64` runs fail closed. + Atari57 requests more CPU, memory, and storage than the current Daytona account allows. The all-nine commands therefore use the default local Docker sandbox. Use Daytona only for selected non-Atari tasks. @@ -68,21 +100,27 @@ Run all four models sequentially: ```bash uv run python scripts/run_article_suite.py \ --env-file /path/to/credentials.env \ - --all-models + --all-models \ + --trials 5 \ + --batch-id article-suite-v2 ``` -Long runs are resumable at task granularity. For example: +Runs are resumable at model-trial granularity. Reusing `--batch-id` skips +completed trials. Use `--trial` to schedule or repair selected trials: ```bash uv run python scripts/run_article_suite.py \ --env-file /path/to/credentials.env \ --model gpt-5.6-sol \ --task simulation_heuristics_halfcheetah_v1 \ + --trial 3 \ + --batch-id article-suite-v2 \ --sandbox daytona ``` -The leaderboard builder selects the latest successful result for every -model/task pair across all run batches. +The leaderboard builder selects the latest complete batch matching +`protocol.toml`; it never mixes older single-run results into the five-trial +leaderboard. GPT-5.6 Sol is routed directly through OpenCode's Azure Responses-API provider with reasoning effort `max`. BenchFlow still owns sandboxing, task staging, @@ -109,8 +147,8 @@ The builder writes: ranking displayed as `IQM + 100` so every current plotted value is positive. Each task maps its starter policy to `0` and its trusted article-level reference -to `100`. The final primary aggregate is the interquartile mean: remove the two -lowest and two highest of the nine normalized scores, then average the middle -five. Arithmetic mean and median remain secondary diagnostics. The top-level -repository README shows only the final image. The additive display offset does -not affect ranking or score gaps; raw IQM remains available in the JSON. +to `100`. For each trial, remove the two lowest and two highest of the nine +normalized scores and average the middle five. The final leaderboard averages +the five trial-level IQMs. Per-task and final sample standard deviations remain +visible diagnostics. The additive display offset does not affect ranking or +score gaps; raw IQM remains available in the JSON. diff --git a/experiments/article_suite/protocol.toml b/experiments/article_suite/protocol.toml new file mode 100644 index 0000000..134202c --- /dev/null +++ b/experiments/article_suite/protocol.toml @@ -0,0 +1,21 @@ +version = "2.0" +trials = 5 +agent_timeout_multiplier = 3 + +[baseline_agent_timeout_sec] +simulation_heuristics_ant_v1 = 1800 +simulation_heuristics_pong_ram_v1 = 1800 +simulation_heuristics_breakout_ram_v1 = 1800 +simulation_heuristics_breakout_rgb_v1 = 1800 +simulation_heuristics_halfcheetah_v1 = 3600 +simulation_heuristics_vizdoom_d1_v1 = 1800 +simulation_heuristics_vizdoom_d3_v1 = 1800 +simulation_heuristics_atari57_v1 = 604800 +simulation_heuristics_montezuma_v1 = 1800 + +[aggregation] +task_score = "arithmetic_mean_across_trials" +task_variability = "sample_standard_deviation" +trial_score = "interquartile_mean_across_nine_tasks" +final_score = "arithmetic_mean_across_trial_scores" +final_variability = "sample_standard_deviation" diff --git a/scripts/build_article_suite_leaderboard.py b/scripts/build_article_suite_leaderboard.py index 16ff80e..abae929 100644 --- a/scripts/build_article_suite_leaderboard.py +++ b/scripts/build_article_suite_leaderboard.py @@ -16,6 +16,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] MODELS_PATH = REPO_ROOT / "experiments" / "article_suite" / "models.toml" +PROTOCOL_PATH = REPO_ROOT / "experiments" / "article_suite" / "protocol.toml" TASKS = ( "simulation_heuristics_ant_v1", "simulation_heuristics_pong_ram_v1", @@ -251,6 +252,10 @@ def _expected_models() -> dict[str, dict[str, Any]]: return {model["id"]: model for model in payload["models"]} +def _protocol() -> dict[str, Any]: + return tomllib.loads(PROTOCOL_PATH.read_text()) + + def _rank_rows( rows: list[dict[str, Any]], *, @@ -285,13 +290,44 @@ def _interquartile_mean(scores: list[float]) -> float: return math.fsum(retained) / len(retained) -def _aggregate_task_scores(task_scores: dict[str, float]) -> dict[str, float]: +def _aggregate_task_scores( + task_scores: dict[str, float], + *, + trial_task_scores: dict[int, dict[str, float]] | None = None, +) -> dict[str, Any]: scores = [float(task_scores[task]) for task in TASKS] - return { + aggregate: dict[str, Any] = { "final_normalized_score": _interquartile_mean(scores), "arithmetic_mean_normalized_score": math.fsum(scores) / len(scores), "median_normalized_score": float(statistics.median(scores)), } + if trial_task_scores: + trial_final_scores = { + trial: _interquartile_mean( + [float(task_scores_for_trial[task]) for task in TASKS] + ) + for trial, task_scores_for_trial in sorted( + trial_task_scores.items() + ) + } + values = list(trial_final_scores.values()) + aggregate.update( + { + "final_normalized_score": statistics.fmean(values), + "final_normalized_score_stddev": statistics.stdev(values), + "trial_final_normalized_scores": trial_final_scores, + } + ) + else: + aggregate.update( + { + "final_normalized_score_stddev": 0.0, + "trial_final_normalized_scores": { + 1: aggregate["final_normalized_score"] + }, + } + ) + return aggregate def _build_leaderboards( @@ -311,7 +347,12 @@ def _build_leaderboards( "provider_reasoning_effort" ], "normalized_score": row["task_scores"][task], + "normalized_score_stddev": row[ + "task_score_stddevs" + ][task], "raw_score": row["raw_task_scores"][task], + "raw_score_stddev": row["raw_task_score_stddevs"][task], + "trial_count": row["trial_count"], "starter_score": row["task_anchors"][task][ "starter_score" ], @@ -352,6 +393,10 @@ def _build_leaderboards( "arithmetic_mean_normalized_score" ], "median_normalized_score": row["median_normalized_score"], + "final_normalized_score_stddev": row[ + "final_normalized_score_stddev" + ], + "trial_count": row["trial_count"], "positive_display_score": row["final_normalized_score"] + FINAL_DISPLAY_OFFSET, "average_normalized_score": row["average_normalized_score"], @@ -388,6 +433,13 @@ def _leaderboard_relative_path(path: str) -> str: return detail_path.as_posix() +def _published_artifact_path(path: Path, *, output_parent: Path) -> str: + try: + return str(path.relative_to(REPO_ROOT)) + except ValueError: + return str(path.relative_to(output_parent)) + + def _render_article_suite_markdown( leaderboards: list[dict[str, Any]], ) -> str: @@ -401,8 +453,8 @@ def _render_article_suite_markdown( "The first image contains the nine independently ranked task " "leaderboards. The second image contains the final cross-task ranking.", "", - "The nine task panels use each environment's native raw score. " - "The final scientific metric remains unbounded IQM.", + "The nine task panels report five-trial mean ± sample standard " + "deviation for each environment's native raw score.", "", "## Nine task leaderboards", "", @@ -411,10 +463,11 @@ def _render_article_suite_markdown( "", "## Final normalized score", "", - "The primary score is the interquartile mean (IQM): sort the nine task " + "Each trial computes an interquartile mean (IQM): sort the nine task " "scores, remove the lowest two and highest two, then average the middle " - "five. The image uses a plot-only positive display index equal to " - "`IQM + 100`; raw IQM, arithmetic mean, and median remain in the JSON.", + "five. The final score is mean ± sample standard deviation across five " + "trial IQMs. The image uses a plot-only positive display index equal " + "to `IQM + 100`; raw metrics remain in the JSON.", "", "![Final GenesisBench article-suite leaderboard]" "(article_suite_final_leaderboard.png)", @@ -508,146 +561,375 @@ def _latest_task_results( return selected +def _trial_batch_results( + batch_root: Path, + *, + expected_models: set[str], + expected_trials: int, +) -> dict[ + str, + dict[str, dict[int, tuple[dict[str, Any], dict[str, Any], Path, str]]], +]: + selected: dict[ + str, + dict[str, dict[int, tuple[dict[str, Any], dict[str, Any], Path, str]]], + ] = {} + metadata_paths = [ + *batch_root.glob("*/trial-*/run_metadata.json"), + *batch_root.glob("*/trial-*/*/run_metadata.json"), + ] + for metadata_path in metadata_paths: + metadata = json.loads(metadata_path.read_text()) + model_id = metadata.get("model", {}).get("id") + trial = metadata.get("trial") + configured_tasks = metadata.get("tasks") + if ( + model_id not in expected_models + or not isinstance(trial, int) + or not 1 <= trial <= expected_trials + or not isinstance(configured_tasks, list) + or not configured_tasks + or not all(task in TASKS for task in configured_tasks) + or metadata.get("status") != "completed" + or metadata.get("return_code") != 0 + or metadata.get("dry_run") + ): + continue + model_root = metadata_path.parent + manifest_path = model_root / "task_manifest.json" + if not manifest_path.is_file(): + continue + manifest = json.loads(manifest_path.read_text()) + manifest_digests = { + item.get("task_id"): item.get("digest") + for item in manifest.get("tasks", []) + if isinstance(item, dict) + } + try: + rows = _load_results( + model_root, + expected_tasks=tuple(configured_tasks), + ) + except RuntimeError: + continue + for task, row in rows.items(): + digest = manifest_digests.get(task) + if not isinstance(digest, str): + continue + task_trials = selected.setdefault(model_id, {}).setdefault( + task, + {}, + ) + current = task_trials.get(trial) + candidate = ( + metadata, + row, + model_root, + digest, + ) + if ( + current is None + or float(metadata["finished_at"]) + > float(current[0]["finished_at"]) + ): + task_trials[trial] = candidate + return selected + + +def _trial_batch_missing( + results: dict[ + str, + dict[str, dict[int, tuple[dict[str, Any], dict[str, Any], Path, str]]], + ], + *, + expected_models: set[str], + expected_trials: int, +) -> list[str]: + missing = [] + for model_id in sorted(expected_models): + for task in TASKS: + trials = results.get(model_id, {}).get(task, {}) + for trial in range(1, expected_trials + 1): + if trial not in trials: + missing.append(f"{model_id}/{task}/trial-{trial:02d}") + return missing + + +def _latest_complete_trial_batch( + runs_root: Path, + *, + expected_models: set[str], + protocol: dict[str, Any], +) -> tuple[ + Path, + dict[ + str, + dict[ + str, + dict[int, tuple[dict[str, Any], dict[str, Any], Path, str]], + ], + ], +]: + expected_trials = int(protocol["trials"]) + candidates = [] + incomplete: list[tuple[Path, int]] = [] + for manifest_path in runs_root.glob("*/batch_manifest.json"): + manifest = json.loads(manifest_path.read_text()) + if manifest.get("protocol") != protocol: + continue + batch_root = manifest_path.parent + results = _trial_batch_results( + batch_root, + expected_models=expected_models, + expected_trials=expected_trials, + ) + missing = _trial_batch_missing( + results, + expected_models=expected_models, + expected_trials=expected_trials, + ) + if missing: + incomplete.append((batch_root, len(missing))) + continue + finished_at = max( + entry[0]["finished_at"] + for model_results in results.values() + for task_results in model_results.values() + for entry in task_results.values() + ) + candidates.append((float(finished_at), batch_root, results)) + if not candidates: + detail = ( + ", ".join( + f"{path.name}: {missing_count} missing" + for path, missing_count in sorted(incomplete)[-3:] + ) + or "no matching protocol batches" + ) + raise RuntimeError( + f"No complete {expected_trials}-trial article-suite batch is " + "available: " + + detail + ) + _, batch_root, results = max(candidates, key=lambda item: item[0]) + return batch_root, results + + def main() -> None: args = parse_args() runs_root = args.runs_root.resolve() expected_models = _expected_models() - task_results = _latest_task_results(runs_root) - missing_models = sorted(set(expected_models) - set(task_results)) - if missing_models: - raise RuntimeError( - "Missing completed article-suite model runs: " - + ", ".join(missing_models) - ) + protocol = _protocol() + trial_count = int(protocol["trials"]) + batch_root, task_results = _latest_complete_trial_batch( + runs_root, + expected_models=set(expected_models), + protocol=protocol, + ) ranked: list[dict[str, Any]] = [] submissions_root = args.output.parent / "article_suite_submissions" for model_id in expected_models: model_results = task_results[model_id] - missing_tasks = sorted(set(TASKS) - set(model_results)) - if missing_tasks: - raise RuntimeError( - f"{model_id} is missing completed task results: " - + ", ".join(missing_tasks) - ) - stale_tasks = [ - task - for task in TASKS - if _digest_compatibility_note( - task, - model_results[task][3], - task_digest(REPO_ROOT / "tasks" / task), - ) - is None - and model_results[task][3] - != task_digest(REPO_ROOT / "tasks" / task) - ] - if stale_tasks: - raise RuntimeError( - f"{model_id} has stale task digests: " - + ", ".join(stale_tasks) - ) - latest_task = max( - model_results.values(), - key=lambda item: item[0], - ) - metadata = json.loads( - (latest_task[2] / "run_metadata.json").read_text() - ) - task_scores = { - task: _normalized_task_score(model_results[task][1]) - for task in TASKS - } + representative_metadata = model_results[TASKS[0]][1][0] + task_scores: dict[str, float] = {} + task_score_stddevs: dict[str, float] = {} raw_task_scores: dict[str, float] = {} + raw_task_score_stddevs: dict[str, float] = {} task_anchors: dict[str, dict[str, float]] = {} submission_details: dict[str, str] = {} + source_runs: dict[str, list[str]] = {} + trial_task_scores = { + trial: {} for trial in range(1, trial_count + 1) + } for task in TASKS: - _, result_row, model_root, digest = model_results[task] + trial_results = model_results[task] current_digest = task_digest(REPO_ROOT / "tasks" / task) - compatibility_note = _digest_compatibility_note( - task, - digest, - current_digest, - ) - task_metadata = json.loads( - (model_root / "run_metadata.json").read_text() - ) - rollout_dir = Path(result_row["info"]["rollout_dir"]) - source_score = rollout_dir / "verifier" / "genesis-score.json" - if not source_score.is_file(): - raise RuntimeError( - f"{model_id}/{task} has no verifier genesis-score.json" - ) destination = submissions_root / model_id / task + if destination.exists(): + shutil.rmtree(destination) destination.mkdir(parents=True, exist_ok=True) - sanitized_score = _sanitize_score_paths( - json.loads(source_score.read_text()) - ) - raw_score = sanitized_score.get("score") - starter_score = sanitized_score.get("starter_score") - reference_score = sanitized_score.get("reference_score") - for name, value in ( - ("score", raw_score), - ("starter_score", starter_score), - ("reference_score", reference_score), - ): - if ( - not isinstance(value, int | float) - or isinstance(value, bool) - or not math.isfinite(float(value)) - ): + normalized_values = [] + raw_values = [] + starter_values = [] + reference_values = [] + trial_records = [] + for trial in range(1, trial_count + 1): + task_metadata, result_row, model_root, digest = trial_results[ + trial + ] + compatibility_note = _digest_compatibility_note( + task, + digest, + current_digest, + ) + if digest != current_digest and compatibility_note is None: raise RuntimeError( - f"{model_id}/{task} has invalid {name} {value!r}" + f"{model_id}/{task}/trial-{trial:02d} has stale " + f"task digest {digest}" ) - raw_task_scores[task] = float(raw_score) + rollout_dir = Path(result_row["info"]["rollout_dir"]) + source_score = ( + rollout_dir / "verifier" / "genesis-score.json" + ) + if not source_score.is_file(): + raise RuntimeError( + f"{model_id}/{task}/trial-{trial:02d} has no " + "verifier genesis-score.json" + ) + sanitized_score = _sanitize_score_paths( + json.loads(source_score.read_text()) + ) + normalized = _normalized_task_score(result_row) + raw_score = sanitized_score.get("score") + starter_score = sanitized_score.get("starter_score") + reference_score = sanitized_score.get("reference_score") + for name, value in ( + ("score", raw_score), + ("starter_score", starter_score), + ("reference_score", reference_score), + ): + if ( + not isinstance(value, int | float) + or isinstance(value, bool) + or not math.isfinite(float(value)) + ): + raise RuntimeError( + f"{model_id}/{task}/trial-{trial:02d} has " + f"invalid {name} {value!r}" + ) + trial_destination = destination / f"trial-{trial:02d}" + trial_destination.mkdir(parents=True, exist_ok=True) + trial_score_path = trial_destination / "score.json" + trial_score_path.write_text( + json.dumps( + sanitized_score, + indent=2, + sort_keys=True, + ) + + "\n" + ) + source_run = str(model_root.relative_to(REPO_ROOT)) + trial_metadata = { + "benchmark": "learning_beyond_gradients_article_suite", + "task": task, + "trial": trial, + "task_digest": digest, + "current_task_digest": current_digest, + "digest_compatibility_note": compatibility_note, + "model": task_metadata["model"], + "harness": task_metadata["harness"], + "provider_reasoning_effort": task_metadata[ + "provider_reasoning_effort" + ], + "normalized_score": normalized, + "raw_score": float(raw_score), + "source_run_id": source_run, + "finished_at": task_metadata["finished_at"], + "tool_calls": result_row.get("metrics", {}).get( + "n_tool_calls", + result_row.get("total_tool_calls"), + ), + "token_usage": result_row.get("token_usage"), + } + (trial_destination / "metadata.json").write_text( + json.dumps( + trial_metadata, + indent=2, + sort_keys=True, + ) + + "\n" + ) + normalized_values.append(normalized) + raw_values.append(float(raw_score)) + starter_values.append(float(starter_score)) + reference_values.append(float(reference_score)) + trial_task_scores[trial][task] = normalized + trial_records.append( + { + "trial": trial, + "normalized_score": normalized, + "raw_score": float(raw_score), + "score_path": _published_artifact_path( + trial_score_path, + output_parent=args.output.parent, + ), + "source_run_id": source_run, + } + ) + + task_scores[task] = statistics.fmean(normalized_values) + task_score_stddevs[task] = statistics.stdev(normalized_values) + raw_task_scores[task] = statistics.fmean(raw_values) + raw_task_score_stddevs[task] = statistics.stdev(raw_values) task_anchors[task] = { - "starter_score": float(starter_score), - "reference_score": float(reference_score), + "starter_score": statistics.fmean(starter_values), + "reference_score": statistics.fmean(reference_values), + } + aggregate_score = { + "benchmark": "learning_beyond_gradients_article_suite", + "task": task, + "trial_count": trial_count, + "normalized_score": task_scores[task], + "normalized_score_stddev": task_score_stddevs[task], + "score": raw_task_scores[task], + "score_stddev": raw_task_score_stddevs[task], + "starter_score": task_anchors[task]["starter_score"], + "reference_score": task_anchors[task]["reference_score"], + "trials": trial_records, } - (destination / "score.json").write_text( - json.dumps(sanitized_score, indent=2, sort_keys=True) + "\n" + aggregate_score_path = destination / "score.json" + aggregate_score_path.write_text( + json.dumps(aggregate_score, indent=2, sort_keys=True) + "\n" ) - source_run = str(model_root.relative_to(REPO_ROOT)) metadata_payload = { "benchmark": "learning_beyond_gradients_article_suite", "task": task, - "task_digest": digest, + "trial_count": trial_count, "current_task_digest": current_digest, - "digest_compatibility_note": compatibility_note, - "model": task_metadata["model"], - "harness": task_metadata["harness"], - "provider_reasoning_effort": task_metadata[ + "model": representative_metadata["model"], + "harness": representative_metadata["harness"], + "provider_reasoning_effort": representative_metadata[ "provider_reasoning_effort" ], "normalized_score": task_scores[task], - "source_run_id": source_run, - "finished_at": task_metadata["finished_at"], - "tool_calls": result_row.get("metrics", {}).get( - "n_tool_calls", - result_row.get("total_tool_calls"), - ), - "token_usage": result_row.get("token_usage"), + "normalized_score_stddev": task_score_stddevs[task], + "raw_score": raw_task_scores[task], + "raw_score_stddev": raw_task_score_stddevs[task], + "source_run_ids": [ + record["source_run_id"] for record in trial_records + ], } (destination / "metadata.json").write_text( json.dumps(metadata_payload, indent=2, sort_keys=True) + "\n" ) submission_details[task] = str( - (destination / "score.json").relative_to(REPO_ROOT) + _published_artifact_path( + aggregate_score_path, + output_parent=args.output.parent, + ) ) - aggregate_scores = _aggregate_task_scores(task_scores) + source_runs[task] = [ + record["source_run_id"] for record in trial_records + ] + aggregate_scores = _aggregate_task_scores( + task_scores, + trial_task_scores=trial_task_scores, + ) ranked.append( { - "model_id": metadata["model"]["id"], - "model": metadata["model"]["display_name"], - "model_route": metadata["model"]["model"], - "provider": metadata["model"]["provider"], + "model_id": representative_metadata["model"]["id"], + "model": representative_metadata["model"]["display_name"], + "model_route": representative_metadata["model"]["model"], + "provider": representative_metadata["model"]["provider"], "provider_label": PROVIDER_LABELS.get( - metadata["model"]["provider"], - metadata["model"]["provider"], + representative_metadata["model"]["provider"], + representative_metadata["model"]["provider"], ), - "harness": metadata["harness"], - "provider_reasoning_effort": metadata[ + "harness": representative_metadata["harness"], + "provider_reasoning_effort": representative_metadata[ "provider_reasoning_effort" ], + "trial_count": trial_count, **aggregate_scores, # Backward-compatible alias for consumers of the first # published schema. This is not the primary ranking metric. @@ -655,15 +937,12 @@ def main() -> None: "arithmetic_mean_normalized_score" ], "task_scores": task_scores, + "task_score_stddevs": task_score_stddevs, "raw_task_scores": raw_task_scores, + "raw_task_score_stddevs": raw_task_score_stddevs, "task_anchors": task_anchors, "submission_details": submission_details, - "source_runs": { - task: str( - model_results[task][2].relative_to(REPO_ROOT) - ) - for task in TASKS - }, + "source_runs": source_runs, } ) ranked = _rank_rows( @@ -674,6 +953,8 @@ def main() -> None: payload = { "benchmark": "learning_beyond_gradients_article_suite", + "batch_id": batch_root.name, + "protocol": protocol, "task_count": len(TASKS), "leaderboard_count": len(leaderboards), "tasks": list(TASKS), @@ -691,7 +972,11 @@ def main() -> None: "arithmetic_mean_normalized_score", "median_normalized_score", ], - "uncertainty": "not_estimated_single_run_per_model_task", + "task_estimator": "mean_across_five_trials", + "task_variability": "sample_standard_deviation", + "trial_estimator": "iqm_across_nine_tasks", + "final_estimator": "mean_across_five_trial_iqms", + "final_variability": "sample_standard_deviation", "display_transform": { "type": "additive_offset", "offset": FINAL_DISPLAY_OFFSET, @@ -724,17 +1009,14 @@ def main() -> None: }, "task_digest_compatibility": TASK_DIGEST_COMPATIBILITY, "source_runs": { - model_id: { - task: str(task_results[model_id][task][2].relative_to(REPO_ROOT)) - for task in TASKS - } - for model_id in expected_models + row["model_id"]: row["source_runs"] for row in ranked }, "generated_at": datetime.fromtimestamp( max( - item[0] + entry[0]["finished_at"] for model_results in task_results.values() - for item in model_results.values() + for task_trials in model_results.values() + for entry in task_trials.values() ), UTC, ).isoformat(), diff --git a/scripts/plot_article_suite_leaderboards.py b/scripts/plot_article_suite_leaderboards.py index 06bcc1e..a2a90e8 100644 --- a/scripts/plot_article_suite_leaderboards.py +++ b/scripts/plot_article_suite_leaderboards.py @@ -87,9 +87,17 @@ def _plot_task_leaderboards( rows = board["rows"] names = [row["model"] for row in rows] scores = [float(row["raw_score"]) for row in rows] + score_stddevs = [ + float(row.get("raw_score_stddev", 0.0)) for row in rows + ] starter_score = float(rows[0]["starter_score"]) reference_score = float(rows[0]["reference_score"]) - all_values = [*scores, starter_score, reference_score] + all_values = [ + *(score - stddev for score, stddev in zip(scores, score_stddevs)), + *(score + stddev for score, stddev in zip(scores, score_stddevs)), + starter_score, + reference_score, + ] value_min = min(all_values) value_max = max(all_values) padding = max((value_max - value_min) * 0.16, abs(value_max) * 0.05, 1.0) @@ -99,7 +107,14 @@ def _plot_task_leaderboards( colors = [ MODEL_COLORS.get(row["model_id"], "#666666") for row in rows ] - bars = axis.barh(names, scores, color=colors, height=0.62) + bars = axis.barh( + names, + scores, + xerr=score_stddevs, + color=colors, + height=0.62, + error_kw={"ecolor": "#333333", "elinewidth": 0.8, "capsize": 2}, + ) axis.invert_yaxis() axis.axvline( starter_score, @@ -134,7 +149,12 @@ def _plot_task_leaderboards( axis.spines[["top", "right", "left"]].set_visible(False) axis.tick_params(axis="y", length=0, labelsize=9.5) axis.tick_params(axis="x", labelsize=8.5) - for bar, score in zip(bars, scores, strict=True): + for bar, score, score_stddev in zip( + bars, + scores, + score_stddevs, + strict=True, + ): if score < lower + span * 0.08: x = score + span * 0.012 alignment = "left" @@ -145,7 +165,7 @@ def _plot_task_leaderboards( axis.text( x, bar.get_y() + bar.get_height() / 2, - f"{score:.1f}", + f"{score:.1f} ± {score_stddev:.1f}", va="center", ha=alignment, fontsize=9, @@ -207,6 +227,9 @@ def _plot_final_leaderboard( rows = final_board["rows"] names = [f"{row['rank']} {row['model']}" for row in rows] scores = [float(row["positive_display_score"]) for row in rows] + score_stddevs = [ + float(row.get("final_normalized_score_stddev", 0.0)) for row in rows + ] colors = [ MODEL_COLORS.get(row["model_id"], "#666666") for row in rows ] @@ -214,10 +237,30 @@ def _plot_final_leaderboard( figure, axis = plt.subplots(figsize=(12, 7.2), dpi=160) figure.patch.set_facecolor("#fafafa") axis.set_facecolor("#fafafa") - bars = axis.barh(names, scores, color=colors, height=0.62) + bars = axis.barh( + names, + scores, + xerr=score_stddevs, + color=colors, + height=0.62, + error_kw={"ecolor": "#333333", "elinewidth": 1.0, "capsize": 3}, + ) axis.invert_yaxis() lower = 0.0 - upper = max(160.0, math.ceil((max(scores) + 10.0) / 10.0) * 10.0) + upper = max( + 160.0, + math.ceil( + ( + max( + score + stddev + for score, stddev in zip(scores, score_stddevs) + ) + + 10.0 + ) + / 10.0 + ) + * 10.0, + ) span = upper - lower axis.set_xlim(lower, upper) axis.axvline( @@ -233,14 +276,21 @@ def _plot_final_leaderboard( axis.tick_params(axis="x", labelsize=10) axis.set_xlabel("Positive display index (IQM + 100)", fontsize=11) - for bar, row, score in zip(bars, rows, scores, strict=True): + for bar, row, score, score_stddev in zip( + bars, + rows, + scores, + score_stddevs, + strict=True, + ): label_color = ( "#171717" if row["model_id"] == "gpt-5.4-mini" else "#ffffff" ) axis.text( score - span * 0.015, bar.get_y() + bar.get_height() / 2, - f"{score:.2f} (IQM {row['final_normalized_score']:.2f})", + f"{score:.2f} ± {score_stddev:.2f} " + f"(IQM {row['final_normalized_score']:.2f})", va="center", ha="right", fontsize=10.5, diff --git a/scripts/run_article_suite.py b/scripts/run_article_suite.py index 032286d..b69c1ad 100644 --- a/scripts/run_article_suite.py +++ b/scripts/run_article_suite.py @@ -2,8 +2,10 @@ from __future__ import annotations import argparse +import hashlib import json import os +import shutil import subprocess import time import tomllib @@ -15,6 +17,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] MODELS_PATH = REPO_ROOT / "experiments" / "article_suite" / "models.toml" +PROTOCOL_PATH = REPO_ROOT / "experiments" / "article_suite" / "protocol.toml" TASKS = ( "simulation_heuristics_ant_v1", "simulation_heuristics_pong_ram_v1", @@ -43,6 +46,7 @@ def _default_env_path() -> Path: def parse_args() -> argparse.Namespace: + protocol = _protocol() parser = argparse.ArgumentParser( description="Run the nine-task article suite through OpenCode." ) @@ -62,15 +66,50 @@ def parse_args() -> argparse.Namespace: default=REPO_ROOT / "leaderboard" / "runs" / "article_suite", ) parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument( + "--trials", + type=int, + default=int(protocol["trials"]), + help="Independent full-suite trials per model.", + ) + parser.add_argument( + "--trial", + action="append", + type=int, + help="Run only this 1-indexed trial; repeat for multiple trials.", + ) + parser.add_argument( + "--batch-id", + help="Stable batch directory name for resumable multi-invocation runs.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Delete and rerun selected trial directories even if completed.", + ) + parser.add_argument( + "--fail-fast", + action="store_true", + help="Stop after the first failed model trial instead of continuing.", + ) parser.add_argument( "--sandbox", choices=("docker", "daytona"), default="docker", ) + parser.add_argument( + "--docker-platform", + default="linux/amd64", + help="Calibrated Docker target used by authoritative local runs.", + ) parser.add_argument("--dry-run", action="store_true") return parser.parse_args() +def _protocol() -> dict[str, Any]: + return tomllib.loads(PROTOCOL_PATH.read_text()) + + def _read_env(path: Path) -> dict[str, str]: result: dict[str, str] = {} if not path.is_file(): @@ -119,6 +158,73 @@ def _validate_tasks(tasks: tuple[str, ...]) -> None: ) +def _task_agent_timeout(task: str) -> int: + import yaml + + task_path = REPO_ROOT / "tasks" / task / "task.md" + parts = task_path.read_text().split("---", 2) + if len(parts) != 3: + raise RuntimeError(f"{task} has no YAML front matter") + document = yaml.safe_load(parts[1]) + timeout = document.get("agent", {}).get("timeout_sec") + if not isinstance(timeout, int) or timeout <= 0: + raise RuntimeError(f"{task} has invalid agent.timeout_sec {timeout!r}") + return timeout + + +def _validate_protocol(tasks: tuple[str, ...], trials: int) -> dict[str, Any]: + protocol = _protocol() + if trials < 1: + raise ValueError("--trials must be >= 1") + multiplier = protocol["agent_timeout_multiplier"] + baselines = protocol["baseline_agent_timeout_sec"] + for task in tasks: + expected = int(baselines[task]) * int(multiplier) + actual = _task_agent_timeout(task) + if actual != expected: + raise RuntimeError( + f"{task} agent timeout {actual} does not match protocol " + f"{baselines[task]} x {multiplier} = {expected}" + ) + return protocol + + +def _selected_trials( + requested: list[int] | None, + *, + trial_count: int, +) -> tuple[int, ...]: + if requested is None: + return tuple(range(1, trial_count + 1)) + selected = tuple(dict.fromkeys(requested)) + invalid = [trial for trial in selected if not 1 <= trial <= trial_count] + if invalid: + raise ValueError( + f"--trial must be between 1 and {trial_count}: {invalid}" + ) + return selected + + +def _completed_trial(metadata_path: Path) -> bool: + if not metadata_path.is_file(): + return False + metadata = json.loads(metadata_path.read_text()) + return ( + metadata.get("status") == "completed" + and metadata.get("return_code") == 0 + and not metadata.get("dry_run") + ) + + +def _task_scope(tasks: tuple[str, ...]) -> str: + if tasks == TASKS: + return "full-suite" + if len(tasks) == 1: + return tasks[0] + digest = hashlib.sha256("\n".join(tasks).encode()).hexdigest()[:10] + return f"tasks-{digest}" + + def _require_credentials(model: dict[str, Any], env: dict[str, str]) -> None: required = { "azure": ("AZURE_API_ENDPOINT", "AZURE_API_KEY"), @@ -315,10 +421,22 @@ def _docker_ready() -> bool: return result.returncode == 0 +def _configure_docker_platform( + run_env: dict[str, str], + *, + platform: str, +) -> None: + if not platform.strip(): + raise ValueError("--docker-platform must not be empty") + run_env["DOCKER_DEFAULT_PLATFORM"] = platform + + def main() -> None: args = parse_args() tasks = tuple(args.task or TASKS) _validate_tasks(tasks) + protocol = _validate_protocol(tasks, args.trials) + selected_trials = _selected_trials(args.trial, trial_count=args.trials) models = _select_models(args.model, args.all_models) provider_env = os.environ.copy() provider_env.update(_read_env(args.env_file.expanduser())) @@ -326,8 +444,8 @@ def main() -> None: if not provider_env.get("DAYTONA_API_KEY"): raise RuntimeError("DAYTONA_API_KEY is required for --sandbox daytona") provider_env.setdefault("DAYTONA_TARGET", "us") - timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") - batch_root = args.output_root.resolve() / timestamp + batch_id = args.batch_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + batch_root = args.output_root.resolve() / batch_id batch_root.mkdir(parents=True, exist_ok=True) if args.sandbox == "docker" and not args.dry_run and not _docker_ready(): @@ -335,33 +453,54 @@ def main() -> None: "Docker is not ready. Start Docker Desktop before running the " "authoritative article-suite evaluation." ) + if args.sandbox == "docker": + _configure_docker_platform( + provider_env, + platform=args.docker_platform, + ) + manifest_path = batch_root / "batch_manifest.json" batch_manifest: dict[str, Any] = { "suite": "learning_beyond_gradients_article", + "batch_id": batch_id, + "protocol": protocol, "harness": "opencode", "tasks": list(tasks), "created_at": datetime.now(UTC).isoformat(), - "models": [], + "trial_count": args.trials, + "runs": [], } + if manifest_path.is_file(): + existing = json.loads(manifest_path.read_text()) + if existing.get("protocol") != protocol: + raise RuntimeError( + f"{batch_root} uses a different experiment protocol" + ) + batch_manifest["created_at"] = existing.get( + "created_at", + batch_manifest["created_at"], + ) + batch_manifest["runs"] = list(existing.get("runs", [])) + + run_entries = { + ( + entry["model_id"], + entry["trial"], + entry.get("task_scope", "full-suite"), + ): { + **entry, + "task_scope": entry.get("task_scope", "full-suite"), + } + for entry in batch_manifest["runs"] + } + failures: list[tuple[str, int, int]] = [] for model in models: _require_credentials(model, provider_env) - model_root = batch_root / model["id"] - jobs_dir = model_root / "jobs" - model_root.mkdir(parents=True, exist_ok=True) run_env = _scoped_provider_env(model, provider_env) azure_resource_name = None if model["provider"] == "azure": azure_resource_name = _azure_resource_name(run_env) run_env["AZURE_RESOURCE_NAME"] = azure_resource_name - command = build_command( - model=model, - jobs_dir=jobs_dir, - sandbox=args.sandbox, - concurrency=args.concurrency, - artifact_dir=model_root, - tasks=tasks, - azure_resource_name=azure_resource_name, - ) run_env["BENCHFLOW_REASONING_EFFORT"] = model[ "provider_reasoning_effort" ] @@ -369,86 +508,152 @@ def main() -> None: run_env["BENCHFLOW_DAYTONA_PTY_READLINE_TIMEOUT"] = str( model["daytona_pty_readline_timeout_sec"] ) - started_at = time.time() - metadata = { - "model": model, - "harness": "opencode", - "sandbox": args.sandbox, - "tasks": list(tasks), - "command": command, - "provider_env_keys": sorted( - key - for key in ( - "AZURE_API_ENDPOINT", - "AZURE_API_KEY", - "CLAUDE_CODE_OAUTH_TOKEN", + for trial in selected_trials: + trial_root = batch_root / model["id"] / f"trial-{trial:02d}" + task_scope = _task_scope(tasks) + run_root = ( + trial_root + if task_scope == "full-suite" + else trial_root / task_scope + ) + metadata_path = run_root / "run_metadata.json" + if args.force and run_root.exists() and not args.dry_run: + shutil.rmtree(run_root) + if not args.force and _completed_trial(metadata_path): + print( + f"skip completed {model['id']} trial {trial} " + f"scope {task_scope}" ) - if run_env.get(key) - ), - "provider_reasoning_effort": model["provider_reasoning_effort"], - "started_at": started_at, - "finished_at": None, - "elapsed_seconds": None, - "return_code": None, - "status": "running", - "dry_run": args.dry_run, - } - metadata_path = model_root / "run_metadata.json" - metadata_path.write_text( - json.dumps(metadata, indent=2, sort_keys=True) + "\n" - ) - batch_entry = { - "id": model["id"], - "run_metadata": str(metadata_path.relative_to(batch_root)), - "return_code": None, - "status": "running", - } - batch_manifest["models"].append(batch_entry) - (batch_root / "batch_manifest.json").write_text( - json.dumps(batch_manifest, indent=2, sort_keys=True) + "\n" - ) - return_code = 0 - status = "completed" - try: - if args.dry_run: - print(" ".join(command)) - else: - return_code = subprocess.run( - command, - cwd=REPO_ROOT, - env=run_env, - check=False, - ).returncode - if return_code != 0: - status = "failed" - except KeyboardInterrupt: - return_code = 130 - status = "interrupted" - raise - finally: - metadata.update( - { - "finished_at": time.time(), - "elapsed_seconds": time.time() - started_at, - "return_code": return_code, - "status": status, - } + continue + + jobs_dir = run_root / "jobs" + run_root.mkdir(parents=True, exist_ok=True) + command = build_command( + model=model, + jobs_dir=jobs_dir, + sandbox=args.sandbox, + concurrency=args.concurrency, + artifact_dir=run_root, + tasks=tasks, + azure_resource_name=azure_resource_name, ) + started_at = time.time() + metadata = { + "batch_id": batch_id, + "trial": trial, + "trial_count": args.trials, + "task_scope": task_scope, + "protocol": protocol, + "model": model, + "harness": "opencode", + "sandbox": args.sandbox, + "docker_platform": ( + args.docker_platform if args.sandbox == "docker" else None + ), + "tasks": list(tasks), + "task_agent_timeout_sec": { + task: _task_agent_timeout(task) for task in tasks + }, + "command": command, + "provider_env_keys": sorted( + key + for key in ( + "AZURE_API_ENDPOINT", + "AZURE_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + ) + if run_env.get(key) + ), + "provider_reasoning_effort": model[ + "provider_reasoning_effort" + ], + "started_at": started_at, + "finished_at": None, + "elapsed_seconds": None, + "return_code": None, + "status": "running", + "dry_run": args.dry_run, + } metadata_path.write_text( json.dumps(metadata, indent=2, sort_keys=True) + "\n" ) - batch_entry["return_code"] = return_code - batch_entry["status"] = status - (batch_root / "batch_manifest.json").write_text( + batch_entry = run_entries.setdefault( + (model["id"], trial, task_scope), + { + "model_id": model["id"], + "trial": trial, + "task_scope": task_scope, + "run_metadata": str( + metadata_path.relative_to(batch_root) + ), + }, + ) + batch_entry.update({"return_code": None, "status": "running"}) + batch_manifest["runs"] = sorted( + run_entries.values(), + key=lambda entry: ( + entry["model_id"], + entry["trial"], + entry["task_scope"], + ), + ) + manifest_path.write_text( json.dumps(batch_manifest, indent=2, sort_keys=True) + "\n" ) - if return_code != 0: - raise SystemExit(return_code) - (batch_root / "batch_manifest.json").write_text( + return_code = 0 + status = "completed" + try: + if args.dry_run: + print(" ".join(command)) + else: + return_code = subprocess.run( + command, + cwd=REPO_ROOT, + env=run_env, + check=False, + ).returncode + if return_code != 0: + status = "failed" + except KeyboardInterrupt: + return_code = 130 + status = "interrupted" + raise + finally: + metadata.update( + { + "finished_at": time.time(), + "elapsed_seconds": time.time() - started_at, + "return_code": return_code, + "status": status, + } + ) + metadata_path.write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n" + ) + batch_entry.update( + {"return_code": return_code, "status": status} + ) + manifest_path.write_text( + json.dumps(batch_manifest, indent=2, sort_keys=True) + "\n" + ) + if return_code != 0: + failures.append((model["id"], trial, return_code)) + if args.fail_fast: + raise SystemExit(return_code) + + batch_manifest["status"] = "failed" if failures else "completed" + batch_manifest["finished_at"] = datetime.now(UTC).isoformat() + batch_manifest["failures"] = [ + {"model_id": model_id, "trial": trial, "return_code": return_code} + for model_id, trial, return_code in failures + ] + manifest_path.write_text( json.dumps(batch_manifest, indent=2, sort_keys=True) + "\n" ) print(batch_root) + if failures: + raise SystemExit(1) if __name__ == "__main__": diff --git a/tasks/simulation_heuristics_ant_v1/task.md b/tasks/simulation_heuristics_ant_v1/task.md index 3f286a0..cad6b7a 100644 --- a/tasks/simulation_heuristics_ant_v1/task.md +++ b/tasks/simulation_heuristics_ant_v1/task.md @@ -37,7 +37,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 1800 + timeout_sec: 5400 user: agent network_mode: public verifier: diff --git a/tasks/simulation_heuristics_atari57_v1/task.md b/tasks/simulation_heuristics_atari57_v1/task.md index 06a536e..77f2835 100644 --- a/tasks/simulation_heuristics_atari57_v1/task.md +++ b/tasks/simulation_heuristics_atari57_v1/task.md @@ -46,7 +46,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 604800 + timeout_sec: 1814400 user: agent network_mode: no-network verifier: diff --git a/tasks/simulation_heuristics_breakout_ram_v1/task.md b/tasks/simulation_heuristics_breakout_ram_v1/task.md index fbb6c13..6ed5020 100644 --- a/tasks/simulation_heuristics_breakout_ram_v1/task.md +++ b/tasks/simulation_heuristics_breakout_ram_v1/task.md @@ -36,7 +36,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 1800 + timeout_sec: 5400 user: agent network_mode: public verifier: diff --git a/tasks/simulation_heuristics_breakout_rgb_v1/task.md b/tasks/simulation_heuristics_breakout_rgb_v1/task.md index 3ad7fad..d4db822 100644 --- a/tasks/simulation_heuristics_breakout_rgb_v1/task.md +++ b/tasks/simulation_heuristics_breakout_rgb_v1/task.md @@ -37,7 +37,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 1800 + timeout_sec: 5400 user: agent network_mode: public verifier: diff --git a/tasks/simulation_heuristics_halfcheetah_v1/task.md b/tasks/simulation_heuristics_halfcheetah_v1/task.md index ad935f1..b0c30c3 100644 --- a/tasks/simulation_heuristics_halfcheetah_v1/task.md +++ b/tasks/simulation_heuristics_halfcheetah_v1/task.md @@ -36,7 +36,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 3600 + timeout_sec: 10800 user: agent network_mode: public verifier: diff --git a/tasks/simulation_heuristics_montezuma_v1/task.md b/tasks/simulation_heuristics_montezuma_v1/task.md index 7fac50a..b32c0d6 100644 --- a/tasks/simulation_heuristics_montezuma_v1/task.md +++ b/tasks/simulation_heuristics_montezuma_v1/task.md @@ -37,7 +37,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 1800 + timeout_sec: 5400 user: agent network_mode: public verifier: diff --git a/tasks/simulation_heuristics_pong_ram_v1/task.md b/tasks/simulation_heuristics_pong_ram_v1/task.md index ff5aad7..7f1be32 100644 --- a/tasks/simulation_heuristics_pong_ram_v1/task.md +++ b/tasks/simulation_heuristics_pong_ram_v1/task.md @@ -36,7 +36,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 1800 + timeout_sec: 5400 user: agent network_mode: public verifier: diff --git a/tasks/simulation_heuristics_vizdoom_d1_v1/task.md b/tasks/simulation_heuristics_vizdoom_d1_v1/task.md index 3f53b4c..f84f211 100644 --- a/tasks/simulation_heuristics_vizdoom_d1_v1/task.md +++ b/tasks/simulation_heuristics_vizdoom_d1_v1/task.md @@ -37,7 +37,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 1800 + timeout_sec: 5400 user: agent network_mode: public verifier: diff --git a/tasks/simulation_heuristics_vizdoom_d3_v1/task.md b/tasks/simulation_heuristics_vizdoom_d3_v1/task.md index d2f35fa..3e862fd 100644 --- a/tasks/simulation_heuristics_vizdoom_d3_v1/task.md +++ b/tasks/simulation_heuristics_vizdoom_d3_v1/task.md @@ -37,7 +37,7 @@ metadata: anchors: verifier/anchors.json supports_private_config: true agent: - timeout_sec: 1800 + timeout_sec: 5400 user: agent network_mode: public verifier: diff --git a/tests/test_article_suite_tooling.py b/tests/test_article_suite_tooling.py index b26b828..85d5f19 100644 --- a/tests/test_article_suite_tooling.py +++ b/tests/test_article_suite_tooling.py @@ -17,6 +17,23 @@ def test_article_suite_declares_exactly_nine_unique_tasks() -> None: assert "simulation_heuristics_ant_v1" in runner.TASKS +def test_article_suite_protocol_uses_five_trials_and_triple_timeouts() -> None: + protocol = runner._protocol() + + assert protocol["trials"] == 5 + assert protocol["agent_timeout_multiplier"] == 3 + runner._validate_protocol(runner.TASKS, protocol["trials"]) + + +def test_trial_storage_scope_supports_full_and_partial_resumes() -> None: + assert runner._task_scope(runner.TASKS) == "full-suite" + assert ( + runner._task_scope(("simulation_heuristics_vizdoom_d1_v1",)) + == "simulation_heuristics_vizdoom_d1_v1" + ) + assert runner._task_scope(runner.TASKS[:2]).startswith("tasks-") + + def test_command_uses_opencode_and_fail_closed_artifacts(tmp_path: Path) -> None: model = { "id": "gpt-5.6-sol", @@ -183,6 +200,14 @@ def test_provider_environment_is_least_privilege() -> None: assert "AZURE_API_ENDPOINT" not in claude +def test_authoritative_docker_runs_pin_calibrated_amd64_platform() -> None: + env: dict[str, str] = {} + + runner._configure_docker_platform(env, platform="linux/amd64") + + assert env["DOCKER_DEFAULT_PLATFORM"] == "linux/amd64" + + def test_aggregate_loader_requires_every_task(tmp_path: Path) -> None: model_root = tmp_path / "gpt-5.6-sol" job = model_root / "jobs" / "run" @@ -420,6 +445,91 @@ def test_latest_task_results_merges_partial_runs(tmp_path: Path) -> None: assert selected[model_id][leaderboard.TASKS[1]][3] == "sha256:digest-2" +def test_latest_complete_trial_batch_requires_every_trial( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + tasks = ("task-a", "task-b") + monkeypatch.setattr(leaderboard, "TASKS", tasks) + protocol = {"version": "test", "trials": 2} + batch = tmp_path / "batch-1" + batch.mkdir() + (batch / "batch_manifest.json").write_text( + json.dumps({"protocol": protocol}) + ) + + for trial in (1, 2): + trial_root = batch / "model-a" / f"trial-{trial:02d}" + job = trial_root / "jobs" / "run" + job.mkdir(parents=True) + metadata = { + "model": {"id": "model-a"}, + "trial": trial, + "tasks": list(tasks), + "status": "completed", + "return_code": 0, + "dry_run": False, + "finished_at": float(trial), + } + (trial_root / "run_metadata.json").write_text( + json.dumps(metadata) + ) + (trial_root / "task_manifest.json").write_text( + json.dumps( + { + "tasks": [ + {"task_id": task, "digest": f"sha256:{task}"} + for task in tasks + ] + } + ) + ) + rows = [] + for index, task in enumerate(tasks): + rollout = trial_root / f"rollout-{index}" + verifier = rollout / "verifier" + verifier.mkdir(parents=True) + (verifier / "genesis-score.json").write_text( + json.dumps({"normalized_score": trial * 10 + index}) + ) + rows.append( + { + "info": { + "task_name": task, + "rollout_dir": str(rollout), + }, + "reward": 0.5, + "error": None, + } + ) + (job / "results.jsonl").write_text( + "\n".join(json.dumps(row) for row in rows) + "\n" + ) + + selected_batch, results = leaderboard._latest_complete_trial_batch( + tmp_path, + expected_models={"model-a"}, + protocol=protocol, + ) + + assert selected_batch == batch + assert set(results["model-a"]["task-a"]) == {1, 2} + assert ( + leaderboard._normalized_task_score( + results["model-a"]["task-b"][2][1] + ) + == 21 + ) + + (batch / "model-a" / "trial-02" / "run_metadata.json").unlink() + with pytest.raises(RuntimeError, match="2 missing"): + leaderboard._latest_complete_trial_batch( + tmp_path, + expected_models={"model-a"}, + protocol=protocol, + ) + + def test_iqm_matches_rliable_25_percent_trimmed_mean() -> None: assert leaderboard._interquartile_mean(list(range(9))) == 4.0 assert leaderboard._interquartile_mean( @@ -442,20 +552,27 @@ def test_offline_report_builds_nine_task_boards_then_final() -> None: task_scores[leaderboard.TASKS[1]] = 5.0 aggregates = leaderboard._aggregate_task_scores(task_scores) rows.append( - { - "model_id": model_id, - "model": model, - "model_route": f"test/{model_id}", - "provider": "test", - "provider_label": "Test provider", - "harness": "opencode", + { + "model_id": model_id, + "model": model, + "model_route": f"test/{model_id}", + "provider": "test", + "provider_label": "Test provider", + "harness": "opencode", "provider_reasoning_effort": "max", + "trial_count": 5, **aggregates, "average_normalized_score": aggregates[ "arithmetic_mean_normalized_score" ], "task_scores": task_scores, + "task_score_stddevs": { + task: 1.0 for task in leaderboard.TASKS + }, "raw_task_scores": task_scores, + "raw_task_score_stddevs": { + task: 2.0 for task in leaderboard.TASKS + }, "task_anchors": { task: { "starter_score": 0.0, diff --git a/website/README.md b/website/README.md index dbf3ec8..a3b3ca5 100644 --- a/website/README.md +++ b/website/README.md @@ -34,9 +34,10 @@ Offline README/report images are generated separately by: uv run python scripts/plot_article_suite_leaderboards.py ``` -The HTML task panels use native raw environment scores. The final HTML plot -uses a fixed positive display index (`IQM + 100`) while retaining raw IQM as -the official ranking metric. +The HTML task panels show five-trial mean ± sample standard deviation for +native raw environment scores. The final HTML plot shows mean ± sample standard +deviation across five trial-level IQMs and uses a fixed positive display index +(`IQM + 100`) while retaining raw IQM as the official ranking metric. ## Deploy — genesisbench.benchflow.ai (Vercel, same as clawsbench) diff --git a/website/index.html b/website/index.html index 6c22c69..fde84ba 100644 --- a/website/index.html +++ b/website/index.html @@ -495,14 +495,14 @@

Inference settings

Leaderboards

-

Native raw score for each environment. Dotted markers show the starter; dashed markers show the article reference.

+

Five-trial mean ± sample standard deviation for each environment's native raw score. Dotted markers show the starter; dashed markers show the article reference.

Loading task leaderboards…

Final normalized score

-

Positive display index = IQM + 100. Ranking and score gaps are unchanged; the dotted marker is the starter-level aggregate.

+

Mean ± sample standard deviation across five trial-level IQMs. Positive display index = IQM + 100; the dotted marker is the starter-level aggregate.

Loading final leaderboard…

@@ -877,7 +877,8 @@

Research background

return''+esc(row.model)+''; }).join(""); var values=board.rows.map(function(row){ - return''+number(row.raw_score)+''; + return''+number(row.raw_score)+' ± '+ + number(row.raw_score_stddev||0)+''; }).join(""); var bars=board.rows.map(function(row){ var score=percent(row.raw_score,domain.min,domain.max); @@ -916,7 +917,8 @@

Research background

''+ ''+ ''+ - ''+row.positive_display_score.toFixed(2)+ + ''+row.positive_display_score.toFixed(2)+' ± '+ + (row.final_normalized_score_stddev||0).toFixed(2)+ 'raw IQM '+row.final_normalized_score.toFixed(2)+''+ '
'; }).join(""); From e145039e38c5e1a70eba7bd087198d87ead8a760 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Tue, 14 Jul 2026 12:40:42 -0700 Subject: [PATCH 02/15] support sharded five-trial model batches --- docs/article-suite-scoring.md | 3 +- experiments/article_suite/README.md | 13 ++- scripts/build_article_suite_leaderboard.py | 108 +++++++++++++-------- tests/test_article_suite_tooling.py | 86 +++++++++++++++- 4 files changed, 160 insertions(+), 50 deletions(-) diff --git a/docs/article-suite-scoring.md b/docs/article-suite-scoring.md index e7f8209..09eefee 100644 --- a/docs/article-suite-scoring.md +++ b/docs/article-suite-scoring.md @@ -45,7 +45,8 @@ trial_iqm = mean(scores[2:7]) The final model score is: ```text -final_normalized_score = mean(trial_iqm[1:5]) +final_normalized_score = + mean(trial_iqm_1, trial_iqm_2, trial_iqm_3, trial_iqm_4, trial_iqm_5) ``` The leaderboard also publishes the sample standard deviation of the five diff --git a/experiments/article_suite/README.md b/experiments/article_suite/README.md index 623db9b..b08746f 100644 --- a/experiments/article_suite/README.md +++ b/experiments/article_suite/README.md @@ -106,7 +106,12 @@ uv run python scripts/run_article_suite.py \ ``` Runs are resumable at model-trial granularity. Reusing `--batch-id` skips -completed trials. Use `--trial` to schedule or repair selected trials: +completed trials. Models may also be scheduled into separate batch directories +for parallel execution. The leaderboard builder chooses one complete +five-trial batch per model and never mixes trials from different batches within +the same model. + +Use `--trial` to schedule or repair selected trials: ```bash uv run python scripts/run_article_suite.py \ @@ -118,9 +123,9 @@ uv run python scripts/run_article_suite.py \ --sandbox daytona ``` -The leaderboard builder selects the latest complete batch matching -`protocol.toml`; it never mixes older single-run results into the five-trial -leaderboard. +The leaderboard builder selects the latest complete per-model batch matching +`protocol.toml`; it never mixes older single-run results or partial batches into +the five-trial leaderboard. GPT-5.6 Sol is routed directly through OpenCode's Azure Responses-API provider with reasoning effort `max`. BenchFlow still owns sandboxing, task staging, diff --git a/scripts/build_article_suite_leaderboard.py b/scripts/build_article_suite_leaderboard.py index abae929..21391fd 100644 --- a/scripts/build_article_suite_leaderboard.py +++ b/scripts/build_article_suite_leaderboard.py @@ -655,13 +655,13 @@ def _trial_batch_missing( return missing -def _latest_complete_trial_batch( +def _latest_complete_model_batches( runs_root: Path, *, expected_models: set[str], protocol: dict[str, Any], ) -> tuple[ - Path, + dict[str, Path], dict[ str, dict[ @@ -671,48 +671,64 @@ def _latest_complete_trial_batch( ], ]: expected_trials = int(protocol["trials"]) - candidates = [] - incomplete: list[tuple[Path, int]] = [] + manifests = [] for manifest_path in runs_root.glob("*/batch_manifest.json"): manifest = json.loads(manifest_path.read_text()) - if manifest.get("protocol") != protocol: - continue - batch_root = manifest_path.parent - results = _trial_batch_results( - batch_root, - expected_models=expected_models, - expected_trials=expected_trials, - ) - missing = _trial_batch_missing( - results, - expected_models=expected_models, - expected_trials=expected_trials, - ) - if missing: - incomplete.append((batch_root, len(missing))) - continue - finished_at = max( - entry[0]["finished_at"] - for model_results in results.values() - for task_results in model_results.values() - for entry in task_results.values() - ) - candidates.append((float(finished_at), batch_root, results)) - if not candidates: - detail = ( - ", ".join( - f"{path.name}: {missing_count} missing" - for path, missing_count in sorted(incomplete)[-3:] + if manifest.get("protocol") == protocol: + manifests.append(manifest_path.parent) + + selected_batches: dict[str, Path] = {} + selected_results: dict[ + str, + dict[ + str, + dict[int, tuple[dict[str, Any], dict[str, Any], Path, str]], + ], + ] = {} + for model_id in sorted(expected_models): + candidates = [] + incomplete: list[tuple[Path, int]] = [] + for batch_root in manifests: + results = _trial_batch_results( + batch_root, + expected_models={model_id}, + expected_trials=expected_trials, ) - or "no matching protocol batches" - ) - raise RuntimeError( - f"No complete {expected_trials}-trial article-suite batch is " - "available: " - + detail + missing = _trial_batch_missing( + results, + expected_models={model_id}, + expected_trials=expected_trials, + ) + if missing: + incomplete.append((batch_root, len(missing))) + continue + finished_at = max( + entry[0]["finished_at"] + for task_results in results[model_id].values() + for entry in task_results.values() + ) + candidates.append( + (float(finished_at), batch_root, results[model_id]) + ) + if not candidates: + detail = ( + ", ".join( + f"{path.name}: {missing_count} missing" + for path, missing_count in sorted(incomplete)[-3:] + ) + or "no matching protocol batches" + ) + raise RuntimeError( + f"No complete {expected_trials}-trial article-suite batch is " + f"available for {model_id}: {detail}" + ) + _, batch_root, model_results = max( + candidates, + key=lambda item: item[0], ) - _, batch_root, results = max(candidates, key=lambda item: item[0]) - return batch_root, results + selected_batches[model_id] = batch_root + selected_results[model_id] = model_results + return selected_batches, selected_results def main() -> None: @@ -721,7 +737,7 @@ def main() -> None: expected_models = _expected_models() protocol = _protocol() trial_count = int(protocol["trials"]) - batch_root, task_results = _latest_complete_trial_batch( + model_batch_roots, task_results = _latest_complete_model_batches( runs_root, expected_models=set(expected_models), protocol=protocol, @@ -953,7 +969,15 @@ def main() -> None: payload = { "benchmark": "learning_beyond_gradients_article_suite", - "batch_id": batch_root.name, + "batch_id": ( + next(iter(model_batch_roots.values())).name + if len(set(model_batch_roots.values())) == 1 + else "per-model-batches" + ), + "batch_ids": { + model_id: batch_root.name + for model_id, batch_root in model_batch_roots.items() + }, "protocol": protocol, "task_count": len(TASKS), "leaderboard_count": len(leaderboards), diff --git a/tests/test_article_suite_tooling.py b/tests/test_article_suite_tooling.py index 85d5f19..a188903 100644 --- a/tests/test_article_suite_tooling.py +++ b/tests/test_article_suite_tooling.py @@ -506,13 +506,13 @@ def test_latest_complete_trial_batch_requires_every_trial( "\n".join(json.dumps(row) for row in rows) + "\n" ) - selected_batch, results = leaderboard._latest_complete_trial_batch( + selected_batches, results = leaderboard._latest_complete_model_batches( tmp_path, expected_models={"model-a"}, protocol=protocol, ) - assert selected_batch == batch + assert selected_batches == {"model-a": batch} assert set(results["model-a"]["task-a"]) == {1, 2} assert ( leaderboard._normalized_task_score( @@ -523,13 +523,93 @@ def test_latest_complete_trial_batch_requires_every_trial( (batch / "model-a" / "trial-02" / "run_metadata.json").unlink() with pytest.raises(RuntimeError, match="2 missing"): - leaderboard._latest_complete_trial_batch( + leaderboard._latest_complete_model_batches( tmp_path, expected_models={"model-a"}, protocol=protocol, ) +def test_latest_complete_model_batches_support_parallel_model_shards( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + tasks = ("task-a",) + monkeypatch.setattr(leaderboard, "TASKS", tasks) + protocol = {"version": "test", "trials": 2} + + for model_index, model_id in enumerate(("model-a", "model-b"), start=1): + batch = tmp_path / f"batch-{model_id}" + batch.mkdir() + (batch / "batch_manifest.json").write_text( + json.dumps({"protocol": protocol}) + ) + for trial in (1, 2): + trial_root = batch / model_id / f"trial-{trial:02d}" + job = trial_root / "jobs" / "run" + rollout = trial_root / f"rollout-{trial}" + verifier = rollout / "verifier" + job.mkdir(parents=True) + verifier.mkdir(parents=True) + (trial_root / "run_metadata.json").write_text( + json.dumps( + { + "model": {"id": model_id}, + "trial": trial, + "tasks": list(tasks), + "status": "completed", + "return_code": 0, + "dry_run": False, + "finished_at": float(model_index * 10 + trial), + } + ) + ) + (trial_root / "task_manifest.json").write_text( + json.dumps( + { + "tasks": [ + { + "task_id": "task-a", + "digest": f"sha256:{model_id}", + } + ] + } + ) + ) + (verifier / "genesis-score.json").write_text( + json.dumps({"normalized_score": model_index * 10 + trial}) + ) + (job / "results.jsonl").write_text( + json.dumps( + { + "info": { + "task_name": "task-a", + "rollout_dir": str(rollout), + }, + "reward": 0.5, + "error": None, + } + ) + + "\n" + ) + + selected_batches, results = ( + leaderboard._latest_complete_model_batches( + tmp_path, + expected_models={"model-a", "model-b"}, + protocol=protocol, + ) + ) + + assert selected_batches == { + "model-a": tmp_path / "batch-model-a", + "model-b": tmp_path / "batch-model-b", + } + assert set(results) == {"model-a", "model-b"} + assert set(results["model-a"]["task-a"]) == {1, 2} + assert set(results["model-b"]["task-a"]) == {1, 2} + + def test_iqm_matches_rliable_25_percent_trimmed_mean() -> None: assert leaderboard._interquartile_mean(list(range(9))) == 4.0 assert leaderboard._interquartile_mean( From d8e58e47fe2260b8ff856ea68bd56731a5b3e3d8 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Tue, 14 Jul 2026 12:43:42 -0700 Subject: [PATCH 03/15] enforce the five-trial protocol --- scripts/run_article_suite.py | 8 ++++++-- tests/test_article_suite_tooling.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/run_article_suite.py b/scripts/run_article_suite.py index b69c1ad..874ae71 100644 --- a/scripts/run_article_suite.py +++ b/scripts/run_article_suite.py @@ -174,8 +174,12 @@ def _task_agent_timeout(task: str) -> int: def _validate_protocol(tasks: tuple[str, ...], trials: int) -> dict[str, Any]: protocol = _protocol() - if trials < 1: - raise ValueError("--trials must be >= 1") + expected_trials = int(protocol["trials"]) + if trials != expected_trials: + raise ValueError( + f"--trials must match protocol.toml: expected " + f"{expected_trials}, got {trials}" + ) multiplier = protocol["agent_timeout_multiplier"] baselines = protocol["baseline_agent_timeout_sec"] for task in tasks: diff --git a/tests/test_article_suite_tooling.py b/tests/test_article_suite_tooling.py index a188903..c3fa31f 100644 --- a/tests/test_article_suite_tooling.py +++ b/tests/test_article_suite_tooling.py @@ -23,6 +23,8 @@ def test_article_suite_protocol_uses_five_trials_and_triple_timeouts() -> None: assert protocol["trials"] == 5 assert protocol["agent_timeout_multiplier"] == 3 runner._validate_protocol(runner.TASKS, protocol["trials"]) + with pytest.raises(ValueError, match="must match protocol.toml"): + runner._validate_protocol(runner.TASKS, 4) def test_trial_storage_scope_supports_full_and_partial_resumes() -> None: From 052c2294635af7cafbf5fbe7cfdc3ee17eab08bf Mon Sep 17 00:00:00 2001 From: Bingran You Date: Tue, 14 Jul 2026 12:48:25 -0700 Subject: [PATCH 04/15] validate generated five-trial artifacts --- scripts/build_article_suite_leaderboard.py | 2 + tests/test_leaderboard_artifacts.py | 53 ++++++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/scripts/build_article_suite_leaderboard.py b/scripts/build_article_suite_leaderboard.py index 21391fd..d4a63a7 100644 --- a/scripts/build_article_suite_leaderboard.py +++ b/scripts/build_article_suite_leaderboard.py @@ -434,6 +434,8 @@ def _leaderboard_relative_path(path: str) -> str: def _published_artifact_path(path: Path, *, output_parent: Path) -> str: + path = path.resolve() + output_parent = output_parent.resolve() try: return str(path.relative_to(REPO_ROOT)) except ValueError: diff --git a/tests/test_leaderboard_artifacts.py b/tests/test_leaderboard_artifacts.py index b964960..9f575f8 100644 --- a/tests/test_leaderboard_artifacts.py +++ b/tests/test_leaderboard_artifacts.py @@ -4,6 +4,7 @@ import math from pathlib import Path import re +import statistics import struct @@ -64,6 +65,15 @@ def test_article_suite_leaderboard_is_complete_and_self_contained() -> None: assert payload["aggregation"]["trim_fraction_per_tail"] == 0.25 assert payload["aggregation"]["trimmed_score_count_per_tail"] == 2 assert payload["aggregation"]["retained_score_count"] == 5 + if "protocol" in payload: + assert payload["protocol"]["trials"] == 5 + assert payload["protocol"]["agent_timeout_multiplier"] == 3 + assert set(payload["batch_ids"]) == { + "gpt-5.6-sol", + "gpt-5.5", + "claude-opus-4.8", + "gpt-5.4-mini", + } assert ( payload["inference_settings"]["cross_provider_comparability"] == "labels_are_not_a_shared_numeric_compute_scale" @@ -93,8 +103,19 @@ def test_article_suite_leaderboard_is_complete_and_self_contained() -> None: assert set(row["raw_task_scores"]) == set(payload["tasks"]) assert set(row["task_anchors"]) == set(payload["tasks"]) assert set(row["submission_details"]) == set(payload["tasks"]) - sorted_scores = sorted(row["task_scores"].values()) - expected_iqm = sum(sorted_scores[2:7]) / 5 + trial_final_scores = row.get("trial_final_normalized_scores") + if trial_final_scores is None: + sorted_scores = sorted(row["task_scores"].values()) + expected_iqm = sum(sorted_scores[2:7]) / 5 + else: + assert row["trial_count"] == 5 + assert len(trial_final_scores) == 5 + trial_values = list(trial_final_scores.values()) + expected_iqm = statistics.fmean(trial_values) + assert math.isclose( + row["final_normalized_score_stddev"], + statistics.stdev(trial_values), + ) assert math.isclose( row["final_normalized_score"], expected_iqm, @@ -120,6 +141,22 @@ def test_article_suite_leaderboard_is_complete_and_self_contained() -> None: assert metadata["harness"] == "opencode" assert metadata["normalized_score"] == row["task_scores"][task] assert score["normalized_score"] == row["task_scores"][task] + if row.get("trial_count") is not None: + assert metadata["trial_count"] == 5 + assert score["trial_count"] == 5 + assert len(score["trials"]) == 5 + assert metadata["normalized_score_stddev"] == row[ + "task_score_stddevs" + ][task] + assert score["normalized_score_stddev"] == row[ + "task_score_stddevs" + ][task] + assert metadata["raw_score_stddev"] == row[ + "raw_task_score_stddevs" + ][task] + assert score["score_stddev"] == row[ + "raw_task_score_stddevs" + ][task] def assert_no_absolute_artifact_paths(value: object) -> None: if isinstance(value, dict): @@ -152,6 +189,14 @@ def assert_no_absolute_artifact_paths(value: object) -> None: assert row["reference_score"] == model_row["task_anchors"][ board["id"] ]["reference_score"] + if model_row.get("trial_count") is not None: + assert row["trial_count"] == 5 + assert row["normalized_score_stddev"] == model_row[ + "task_score_stddevs" + ][board["id"]] + assert row["raw_score_stddev"] == model_row[ + "raw_task_score_stddevs" + ][board["id"]] final_board = payload["leaderboards"][-1] assert [ @@ -165,12 +210,12 @@ def assert_no_absolute_artifact_paths(value: object) -> None: ) for row in final_board["rows"] ) - assert [row["model"] for row in payload["rows"]] == [ + assert {row["model"] for row in payload["rows"]} == { "GPT-5.5", "GPT-5.6 Sol", "Claude Opus 4.8", "GPT-5.4 Mini", - ] + } markdown = ARTICLE_SUITE_MARKDOWN.read_text() assert "article_suite_task_leaderboards.png" in markdown From 7119bc1dd71c0faa82266ec88dc814932ca6d7b7 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Tue, 14 Jul 2026 12:55:51 -0700 Subject: [PATCH 05/15] align final IQM with RLiable --- README.md | 14 ++--- docs/article-suite-scoring.md | 28 ++++----- docs/learning-beyond-gradients-suite.md | 10 +-- experiments/article_suite/README.md | 18 +++--- experiments/article_suite/protocol.toml | 6 +- scripts/build_article_suite_leaderboard.py | 68 +++++++++++++++----- scripts/run_article_suite.py | 17 ++++- tests/test_article_suite_tooling.py | 73 ++++++++++++++++++++++ tests/test_leaderboard_artifacts.py | 31 ++++++++- website/README.md | 7 ++- website/index.html | 4 +- 11 files changed, 214 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 9a5497e..4a3f0e4 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ mapping is documented in `docs/learning-beyond-gradients-suite.md`. ## Article-Suite Leaderboard -The first OpenCode sweep across all nine article-derived tasks: +The five-trial OpenCode sweep across all nine article-derived tasks: ![GenesisBench final normalized leaderboard](leaderboard/article_suite_final_leaderboard.png) @@ -128,12 +128,12 @@ Scores are unbounded normalized values: `0` matches the public starter and `100` matches the trusted article-level reference. Negative scores are genuine regressions; scores above `100` exceed the reference. -Each model runs five independent trials. Within each trial, the final ranking -uses the interquartile mean (IQM) of the nine task scores: sort them, remove the -lowest two and highest two, then average the middle five. The published score -is the mean of the five trial IQMs and includes sample standard deviation. The -chart uses a plot-only positive index equal to `IQM + 100`; raw IQM remains in -the JSON. +Each model runs five independent trials. The final ranking uses RLiable-style +interquartile mean (IQM) over all 45 normalized trial-task scores: remove the +lowest 11 and highest 11, then average the middle 23. The displayed `±` value +is the sample standard deviation of the five per-trial nine-task IQMs. The chart +uses a plot-only positive index equal to `IQM + 100`; raw IQM remains in the +JSON. Inference settings are provider-specific: GPT-5.6 Sol and Claude Opus 4.8 use `max`; GPT-5.5 and GPT-5.4 Mini use `xhigh`. These labels come from different diff --git a/docs/article-suite-scoring.md b/docs/article-suite-scoring.md index 09eefee..c063a7c 100644 --- a/docs/article-suite-scoring.md +++ b/docs/article-suite-scoring.md @@ -32,30 +32,30 @@ deviation. ## Final normalized score -Within each trial, the primary cross-task metric is the interquartile mean -(IQM), implemented as the 25% trimmed mean used by -[RLiable](https://github.com/google-research/rliable): +The primary cross-task metric is the interquartile mean (IQM), implemented as +the 25% trimmed mean used by +[RLiable](https://github.com/google-research/rliable). RLiable computes IQM +across the complete `num_runs × num_tasks` score matrix: ```text -scores = sort(the nine normalized task scores for one trial) -trim_count = floor(0.25 * 9) = 2 -trial_iqm = mean(scores[2:7]) +scores = sort(flatten(the 5 × 9 normalized score matrix)) +trim_count = floor(0.25 * 45) = 11 +final_normalized_score = mean(scores[11:34]) ``` -The final model score is: +This retains the middle `23` of the `45` trial-task scores. It matches +RLiable's `aggregate_iqm`, which applies a 25% trimmed mean with `axis=None`. -```text -final_normalized_score = - mean(trial_iqm_1, trial_iqm_2, trial_iqm_3, trial_iqm_4, trial_iqm_5) -``` - -The leaderboard also publishes the sample standard deviation of the five -trial-level IQMs. +For a readable run-to-run variability diagnostic, the leaderboard separately +computes one nine-task IQM per trial and publishes the sample standard +deviation of those five trial-level IQMs. That displayed `±` value is not a +bootstrap confidence interval around the pooled IQM. The JSON also publishes: - `arithmetic_mean_normalized_score`; - `median_normalized_score`; +- `mean_trial_iqm_normalized_score`; - the original `average_normalized_score` as a backward-compatible alias for the arithmetic mean. diff --git a/docs/learning-beyond-gradients-suite.md b/docs/learning-beyond-gradients-suite.md index fc2a5de..762b15d 100644 --- a/docs/learning-beyond-gradients-suite.md +++ b/docs/learning-beyond-gradients-suite.md @@ -80,11 +80,11 @@ task score = 100 * (candidate - starter) / (reference - starter) ``` Each model runs five independent trials. Task leaderboards report mean ± sample -standard deviation across those trials. Within each trial, the primary -cross-task score is the 25% trimmed interquartile mean (IQM): remove the two -lowest and two highest normalized task scores and average the middle five. The -final model score is the mean ± sample standard deviation of the five -trial-level IQMs. +standard deviation across those trials. The primary cross-task score is the +RLiable-style 25% trimmed interquartile mean (IQM) over the complete `5 × 9` +score matrix: flatten all 45 normalized scores, remove the lowest 11 and +highest 11, and average the middle 23. The displayed `±` value is the sample +standard deviation of the five per-trial nine-task IQMs. The repository README intentionally shows only the final leaderboard image. The nine task panels use each environment's native raw score. The final chart diff --git a/experiments/article_suite/README.md b/experiments/article_suite/README.md index b08746f..f8d4660 100644 --- a/experiments/article_suite/README.md +++ b/experiments/article_suite/README.md @@ -23,9 +23,10 @@ The current leaderboard protocol is defined in `protocol.toml`: - five independent full-suite trials per model; - three times the original agent wall-clock timeout for every task; - task scores reported as mean ± sample standard deviation across trials; -- one nine-task IQM computed per trial; -- final score reported as mean ± sample standard deviation across the five - trial-level IQMs. +- final score computed as RLiable-style IQM over all `5 × 9 = 45` + normalized trial-task scores; +- variability shown as the sample standard deviation of the five per-trial + nine-task IQMs. The resulting matrix is: @@ -152,8 +153,9 @@ The builder writes: ranking displayed as `IQM + 100` so every current plotted value is positive. Each task maps its starter policy to `0` and its trusted article-level reference -to `100`. For each trial, remove the two lowest and two highest of the nine -normalized scores and average the middle five. The final leaderboard averages -the five trial-level IQMs. Per-task and final sample standard deviations remain -visible diagnostics. The additive display offset does not affect ranking or -score gaps; raw IQM remains available in the JSON. +to `100`. The final leaderboard flattens the five-by-nine normalized score +matrix, removes the lowest 11 and highest 11 values, and averages the middle +23. Per-task sample standard deviations and the sample standard deviation of +the five trial-level IQMs remain visible diagnostics. The additive display +offset does not affect ranking or score gaps; raw IQM remains available in the +JSON. diff --git a/experiments/article_suite/protocol.toml b/experiments/article_suite/protocol.toml index 134202c..cbb5a6f 100644 --- a/experiments/article_suite/protocol.toml +++ b/experiments/article_suite/protocol.toml @@ -1,4 +1,4 @@ -version = "2.0" +version = "2.1" trials = 5 agent_timeout_multiplier = 3 @@ -17,5 +17,5 @@ simulation_heuristics_montezuma_v1 = 1800 task_score = "arithmetic_mean_across_trials" task_variability = "sample_standard_deviation" trial_score = "interquartile_mean_across_nine_tasks" -final_score = "arithmetic_mean_across_trial_scores" -final_variability = "sample_standard_deviation" +final_score = "interquartile_mean_across_all_trial_task_scores" +final_variability = "sample_standard_deviation_of_trial_level_iqms" diff --git a/scripts/build_article_suite_leaderboard.py b/scripts/build_article_suite_leaderboard.py index d4a63a7..cc44768 100644 --- a/scripts/build_article_suite_leaderboard.py +++ b/scripts/build_article_suite_leaderboard.py @@ -256,6 +256,17 @@ def _protocol() -> dict[str, Any]: return tomllib.loads(PROTOCOL_PATH.read_text()) +def _execution_protocol(protocol: dict[str, Any]) -> dict[str, Any]: + return { + key: protocol.get(key) + for key in ( + "trials", + "agent_timeout_multiplier", + "baseline_agent_timeout_sec", + ) + } + + def _rank_rows( rows: list[dict[str, Any]], *, @@ -302,6 +313,11 @@ def _aggregate_task_scores( "median_normalized_score": float(statistics.median(scores)), } if trial_task_scores: + pooled_scores = [ + float(scores_for_trial[task]) + for _, scores_for_trial in sorted(trial_task_scores.items()) + for task in TASKS + ] trial_final_scores = { trial: _interquartile_mean( [float(task_scores_for_trial[task]) for task in TASKS] @@ -313,18 +329,30 @@ def _aggregate_task_scores( values = list(trial_final_scores.values()) aggregate.update( { - "final_normalized_score": statistics.fmean(values), + "final_normalized_score": _interquartile_mean(pooled_scores), + "arithmetic_mean_normalized_score": statistics.fmean( + pooled_scores + ), + "median_normalized_score": float( + statistics.median(pooled_scores) + ), "final_normalized_score_stddev": statistics.stdev(values), + "mean_trial_iqm_normalized_score": statistics.fmean(values), "trial_final_normalized_scores": trial_final_scores, + "trial_task_normalized_scores": trial_task_scores, } ) else: aggregate.update( { "final_normalized_score_stddev": 0.0, + "mean_trial_iqm_normalized_score": aggregate[ + "final_normalized_score" + ], "trial_final_normalized_scores": { 1: aggregate["final_normalized_score"] }, + "trial_task_normalized_scores": {1: task_scores}, } ) return aggregate @@ -393,6 +421,9 @@ def _build_leaderboards( "arithmetic_mean_normalized_score" ], "median_normalized_score": row["median_normalized_score"], + "mean_trial_iqm_normalized_score": row[ + "mean_trial_iqm_normalized_score" + ], "final_normalized_score_stddev": row[ "final_normalized_score_stddev" ], @@ -465,11 +496,12 @@ def _render_article_suite_markdown( "", "## Final normalized score", "", - "Each trial computes an interquartile mean (IQM): sort the nine task " - "scores, remove the lowest two and highest two, then average the middle " - "five. The final score is mean ± sample standard deviation across five " - "trial IQMs. The image uses a plot-only positive display index equal " - "to `IQM + 100`; raw metrics remain in the JSON.", + "The final score is RLiable-style IQM over the complete 5 × 9 " + "trial-task score matrix: flatten all 45 normalized scores, trim the " + "lowest 11 and highest 11, and average the middle 23. The displayed " + "± value is the sample standard deviation of the five per-trial IQMs. " + "The image uses a plot-only positive display index equal to `IQM + " + "100`; raw metrics remain in the JSON.", "", "![Final GenesisBench article-suite leaderboard]" "(article_suite_final_leaderboard.png)", @@ -676,7 +708,9 @@ def _latest_complete_model_batches( manifests = [] for manifest_path in runs_root.glob("*/batch_manifest.json"): manifest = json.loads(manifest_path.read_text()) - if manifest.get("protocol") == protocol: + if _execution_protocol( + manifest.get("protocol", {}) + ) == _execution_protocol(protocol): manifests.append(manifest_path.parent) selected_batches: dict[str, Path] = {} @@ -968,6 +1002,8 @@ def main() -> None: score_key="final_normalized_score", ) leaderboards = _build_leaderboards(ranked) + pooled_score_count = trial_count * len(TASKS) + pooled_trim_count = int(pooled_score_count * IQM_TRIM_FRACTION) payload = { "benchmark": "learning_beyond_gradients_article_suite", @@ -988,21 +1024,25 @@ def main() -> None: "primary_metric": "interquartile_mean", "primary_field": "final_normalized_score", "trim_fraction_per_tail": IQM_TRIM_FRACTION, - "trimmed_score_count_per_tail": int( - len(TASKS) * IQM_TRIM_FRACTION - ), - "retained_score_count": len(TASKS) - - 2 * int(len(TASKS) * IQM_TRIM_FRACTION), + "pooled_score_count": pooled_score_count, + "trimmed_score_count_per_tail": pooled_trim_count, + "retained_score_count": pooled_score_count + - 2 * pooled_trim_count, "score_bounds": "unbounded", "secondary_fields": [ "arithmetic_mean_normalized_score", "median_normalized_score", + "mean_trial_iqm_normalized_score", ], "task_estimator": "mean_across_five_trials", "task_variability": "sample_standard_deviation", "trial_estimator": "iqm_across_nine_tasks", - "final_estimator": "mean_across_five_trial_iqms", - "final_variability": "sample_standard_deviation", + "final_estimator": ( + "iqm_across_all_45_trial_task_normalized_scores" + ), + "final_variability": ( + "sample_standard_deviation_of_five_trial_iqms" + ), "display_transform": { "type": "additive_offset", "offset": FINAL_DISPLAY_OFFSET, diff --git a/scripts/run_article_suite.py b/scripts/run_article_suite.py index 874ae71..d9acd44 100644 --- a/scripts/run_article_suite.py +++ b/scripts/run_article_suite.py @@ -110,6 +110,17 @@ def _protocol() -> dict[str, Any]: return tomllib.loads(PROTOCOL_PATH.read_text()) +def _execution_protocol(protocol: dict[str, Any]) -> dict[str, Any]: + return { + key: protocol.get(key) + for key in ( + "trials", + "agent_timeout_multiplier", + "baseline_agent_timeout_sec", + ) + } + + def _read_env(path: Path) -> dict[str, str]: result: dict[str, str] = {} if not path.is_file(): @@ -476,9 +487,11 @@ def main() -> None: } if manifest_path.is_file(): existing = json.loads(manifest_path.read_text()) - if existing.get("protocol") != protocol: + if _execution_protocol( + existing.get("protocol", {}) + ) != _execution_protocol(protocol): raise RuntimeError( - f"{batch_root} uses a different experiment protocol" + f"{batch_root} uses incompatible trial or timeout settings" ) batch_manifest["created_at"] = existing.get( "created_at", diff --git a/tests/test_article_suite_tooling.py b/tests/test_article_suite_tooling.py index c3fa31f..15c5d2f 100644 --- a/tests/test_article_suite_tooling.py +++ b/tests/test_article_suite_tooling.py @@ -2,6 +2,7 @@ import json from pathlib import Path +import statistics import pytest @@ -20,6 +21,7 @@ def test_article_suite_declares_exactly_nine_unique_tasks() -> None: def test_article_suite_protocol_uses_five_trials_and_triple_timeouts() -> None: protocol = runner._protocol() + assert protocol["version"] == "2.1" assert protocol["trials"] == 5 assert protocol["agent_timeout_multiplier"] == 3 runner._validate_protocol(runner.TASKS, protocol["trials"]) @@ -27,6 +29,28 @@ def test_article_suite_protocol_uses_five_trials_and_triple_timeouts() -> None: runner._validate_protocol(runner.TASKS, 4) +def test_execution_protocol_ignores_posthoc_aggregation_changes() -> None: + current = runner._protocol() + prior_scoring = { + **current, + "version": "2.0", + "aggregation": { + "final_score": "arithmetic_mean_across_trial_scores" + }, + } + incompatible_timeout = { + **current, + "agent_timeout_multiplier": 2, + } + + assert runner._execution_protocol(prior_scoring) == ( + runner._execution_protocol(current) + ) + assert runner._execution_protocol(incompatible_timeout) != ( + runner._execution_protocol(current) + ) + + def test_trial_storage_scope_supports_full_and_partial_resumes() -> None: assert runner._task_scope(runner.TASKS) == "full-suite" assert ( @@ -617,6 +641,55 @@ def test_iqm_matches_rliable_25_percent_trimmed_mean() -> None: assert leaderboard._interquartile_mean( [-100.0, -50.0, 1.0, 2.0, 3.0, 4.0, 5.0, 100.0, 500.0] ) == 3.0 + assert leaderboard._interquartile_mean(list(range(45))) == 22.0 + + +def test_five_trial_final_score_pools_runs_and_tasks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tasks = tuple(f"task-{index}" for index in range(9)) + monkeypatch.setattr(leaderboard, "TASKS", tasks) + trial_scores = { + trial: { + task: (1000.0 if trial == 4 else 0.0) + for task in tasks + } + for trial in range(5) + } + task_means = { + task: statistics.fmean( + trial_scores[trial][task] for trial in trial_scores + ) + for task in tasks + } + + aggregate = leaderboard._aggregate_task_scores( + task_means, + trial_task_scores=trial_scores, + ) + pooled = [ + trial_scores[trial][task] + for trial in sorted(trial_scores) + for task in tasks + ] + trial_iqms = [ + leaderboard._interquartile_mean( + [trial_scores[trial][task] for task in tasks] + ) + for trial in sorted(trial_scores) + ] + + assert aggregate["final_normalized_score"] == ( + leaderboard._interquartile_mean(pooled) + ) + assert aggregate["mean_trial_iqm_normalized_score"] == ( + statistics.fmean(trial_iqms) + ) + assert aggregate["final_normalized_score"] == 0.0 + assert aggregate["mean_trial_iqm_normalized_score"] == 200.0 + assert aggregate["final_normalized_score_stddev"] == statistics.stdev( + trial_iqms + ) def test_offline_report_builds_nine_task_boards_then_final() -> None: diff --git a/tests/test_leaderboard_artifacts.py b/tests/test_leaderboard_artifacts.py index 9f575f8..f5fa9ee 100644 --- a/tests/test_leaderboard_artifacts.py +++ b/tests/test_leaderboard_artifacts.py @@ -63,17 +63,22 @@ def test_article_suite_leaderboard_is_complete_and_self_contained() -> None: assert payload["leaderboards"][-1]["id"] == "final" assert payload["aggregation"]["primary_metric"] == "interquartile_mean" assert payload["aggregation"]["trim_fraction_per_tail"] == 0.25 - assert payload["aggregation"]["trimmed_score_count_per_tail"] == 2 - assert payload["aggregation"]["retained_score_count"] == 5 if "protocol" in payload: + assert payload["protocol"]["version"] == "2.1" assert payload["protocol"]["trials"] == 5 assert payload["protocol"]["agent_timeout_multiplier"] == 3 + assert payload["aggregation"]["pooled_score_count"] == 45 + assert payload["aggregation"]["trimmed_score_count_per_tail"] == 11 + assert payload["aggregation"]["retained_score_count"] == 23 assert set(payload["batch_ids"]) == { "gpt-5.6-sol", "gpt-5.5", "claude-opus-4.8", "gpt-5.4-mini", } + else: + assert payload["aggregation"]["trimmed_score_count_per_tail"] == 2 + assert payload["aggregation"]["retained_score_count"] == 5 assert ( payload["inference_settings"]["cross_provider_comparability"] == "labels_are_not_a_shared_numeric_compute_scale" @@ -111,11 +116,31 @@ def test_article_suite_leaderboard_is_complete_and_self_contained() -> None: assert row["trial_count"] == 5 assert len(trial_final_scores) == 5 trial_values = list(trial_final_scores.values()) - expected_iqm = statistics.fmean(trial_values) + trial_task_scores = row["trial_task_normalized_scores"] + assert len(trial_task_scores) == 5 + pooled_scores = [ + trial_scores[task] + for _, trial_scores in sorted( + trial_task_scores.items(), + key=lambda item: int(item[0]), + ) + for task in payload["tasks"] + ] + ordered_scores = sorted(pooled_scores) + trim_count = int(len(ordered_scores) * 0.25) + expected_iqm = statistics.fmean( + ordered_scores[ + trim_count : len(ordered_scores) - trim_count + ] + ) assert math.isclose( row["final_normalized_score_stddev"], statistics.stdev(trial_values), ) + assert math.isclose( + row["mean_trial_iqm_normalized_score"], + statistics.fmean(trial_values), + ) assert math.isclose( row["final_normalized_score"], expected_iqm, diff --git a/website/README.md b/website/README.md index a3b3ca5..93b1902 100644 --- a/website/README.md +++ b/website/README.md @@ -35,9 +35,10 @@ uv run python scripts/plot_article_suite_leaderboards.py ``` The HTML task panels show five-trial mean ± sample standard deviation for -native raw environment scores. The final HTML plot shows mean ± sample standard -deviation across five trial-level IQMs and uses a fixed positive display index -(`IQM + 100`) while retaining raw IQM as the official ranking metric. +native raw environment scores. The final HTML plot shows RLiable-style pooled +IQM over all 45 trial-task scores, plus the sample standard deviation of the +five per-trial IQMs, and uses a fixed positive display index (`IQM + 100`) while +retaining raw IQM as the official ranking metric. ## Deploy — genesisbench.benchflow.ai (Vercel, same as clawsbench) diff --git a/website/index.html b/website/index.html index fde84ba..3786f83 100644 --- a/website/index.html +++ b/website/index.html @@ -481,8 +481,6 @@

Real-world operation

Results

-

Four coding models, nine article-derived tasks, one robust cross-task score.

-

The task panels show each environment's native raw score. Cross-task ranking first normalizes each task so the public starter scores 0 and the trusted article-level reference scores 100, then uses the interquartile mean: remove the lowest two and highest two task scores and average the middle five.