From 1ee51895dcec34caa8a5e0c9ffe8f775c9413ba9 Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Fri, 17 Jul 2026 08:58:22 +0000 Subject: [PATCH] fix: validate n >= 1 in MMLU/GSM8K slice loaders (#8) n=0 or n<0 previously yielded an empty slice that scored as a misleading 0% accuracy. Both loaders now raise a clear ValueError at the top of the function, before the cache check and before the gated datasets import, so the guard needs no download or network. Co-Authored-By: Kimi K3 --- src/steerbench/metrics.py | 4 ++++ tests/test_metrics.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/steerbench/metrics.py b/src/steerbench/metrics.py index 88e09ec..61802f1 100644 --- a/src/steerbench/metrics.py +++ b/src/steerbench/metrics.py @@ -573,6 +573,8 @@ def load_mmlu_slice( cache miss. Slice size and seed are part of the cache key so different configs coexist. """ + if n < 1: + raise ValueError(f"n must be >= 1, got {n}") path = _cache_path(cache_dir, f"mmlu_{n}_{seed}.json") if path.exists(): rows = json.loads(path.read_text()) @@ -611,6 +613,8 @@ def load_gsm8k_slice( Gold answers are normalised from the dataset's ``#### `` format at load time so scoring never re-parses them. """ + if n < 1: + raise ValueError(f"n must be >= 1, got {n}") path = _cache_path(cache_dir, f"gsm8k_{n}_{seed}.json") if path.exists(): rows = json.loads(path.read_text()) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 5838eb2..935b505 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -24,6 +24,8 @@ extract_mmlu_answer, formality_score, format_mmlu_prompt, + load_gsm8k_slice, + load_mmlu_slice, perplexity_from_token_nlls, repetition_rate, score_gsm8k, @@ -220,6 +222,23 @@ def fake_generate(prompts: list[str]) -> list[str]: assert evaluate_gsm8k(examples, fake_generate) == 1.0 +# --------------------------------------------------------------------------- # +# SLICE LOADERS — n validation (guard fires before any cache/datasets access) +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("bad_n", [0, -3]) +def test_load_mmlu_slice_rejects_n_below_1(bad_n: int) -> None: + with pytest.raises(ValueError): + load_mmlu_slice(n=bad_n) + + +@pytest.mark.parametrize("bad_n", [0, -3]) +def test_load_gsm8k_slice_rejects_n_below_1(bad_n: int) -> None: + with pytest.raises(ValueError): + load_gsm8k_slice(n=bad_n) + + def test_no_optional_deps_imported() -> None: # Importing steerbench.metrics must not pull in the heavy optional deps. # Checked in a fresh interpreter: a global sys.modules check is unreliable