diff --git a/src/steerbench/metrics.py b/src/steerbench/metrics.py index 88e09ec..61802f1 100644 --- a/src/steerbench/metrics.py +++ b/src/steerbench/metrics.py @@ -573,6 +573,8 @@ def load_mmlu_slice( cache miss. Slice size and seed are part of the cache key so different configs coexist. """ + if n < 1: + raise ValueError(f"n must be >= 1, got {n}") path = _cache_path(cache_dir, f"mmlu_{n}_{seed}.json") if path.exists(): rows = json.loads(path.read_text()) @@ -611,6 +613,8 @@ def load_gsm8k_slice( Gold answers are normalised from the dataset's ``#### `` format at load time so scoring never re-parses them. """ + if n < 1: + raise ValueError(f"n must be >= 1, got {n}") path = _cache_path(cache_dir, f"gsm8k_{n}_{seed}.json") if path.exists(): rows = json.loads(path.read_text()) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 5838eb2..935b505 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -24,6 +24,8 @@ extract_mmlu_answer, formality_score, format_mmlu_prompt, + load_gsm8k_slice, + load_mmlu_slice, perplexity_from_token_nlls, repetition_rate, score_gsm8k, @@ -220,6 +222,23 @@ def fake_generate(prompts: list[str]) -> list[str]: assert evaluate_gsm8k(examples, fake_generate) == 1.0 +# --------------------------------------------------------------------------- # +# SLICE LOADERS — n validation (guard fires before any cache/datasets access) +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("bad_n", [0, -3]) +def test_load_mmlu_slice_rejects_n_below_1(bad_n: int) -> None: + with pytest.raises(ValueError): + load_mmlu_slice(n=bad_n) + + +@pytest.mark.parametrize("bad_n", [0, -3]) +def test_load_gsm8k_slice_rejects_n_below_1(bad_n: int) -> None: + with pytest.raises(ValueError): + load_gsm8k_slice(n=bad_n) + + def test_no_optional_deps_imported() -> None: # Importing steerbench.metrics must not pull in the heavy optional deps. # Checked in a fresh interpreter: a global sys.modules check is unreliable