diff --git a/docs/JOURNAL.md b/docs/JOURNAL.md index 13a8715..de174e6 100644 --- a/docs/JOURNAL.md +++ b/docs/JOURNAL.md @@ -18,6 +18,39 @@ protocol. **Newest entries at the top.** Tag each entry with one or more of: --- +## 2026-07-23 — MC grading was hard-capped at A–D, silently zeroing MMLU-Pro E–J #mistake #fix #repro +**Context:** issues #116 / #122 (filed independently, same root bug). RLPR routes its MMLU-Pro +subset (`MMLUPro-1000_Avg2`) through the `mmlu`/`gpqa` choice grader in `orchestration/reward.py`. +**Expected:** a correct answer grades 1.0 regardless of which option letter is gold. +**Actual:** every item whose gold answer is **E–J scored 0.0 even when the model answered +correctly** — a silently-wrong number, not a crash: +```python +for gold in ["A", "D", "E", "F", "J"]: + ref = {"ground_truth": gold, "source": "MMLUPro-1000_Avg2"} + print(gold, R.score_text("rlpr", f"The answer is ({gold}).", ref)) +# A 1.0 / D 1.0 / E 0.0 / F 0.0 / J 0.0 +``` +**Root cause:** the grader was written for 4-option MMLU/GPQA and hard-capped at A–D in **four** +independent places, so **both** sides of the comparison failed for a ten-option item — the answer +could neither be extracted nor the reference normalized: the five `_CHOICE_PATTERNS` captured +`([A-D])`, `extract_choice_letter`'s last-line fallback matched `([A-D])`, and +`_normalize_reference_letter` accepted only `{A,B,C,D}` / integer index `<= 3`. MMLU-Pro has up to +**ten** options (A–J) and its gold answers spread across all ten letters, so a large fraction of +that subset was unconditionally graded 0 — deflating measured accuracy and distorting the +per-benchmark routing signal the coordinator is trained and evaluated on. +**Fix / decision:** introduce a single `_CHOICE_ALPHABET = "ABCDEFGHIJ"` (plus a derived character +class) and drive **all four** sites from it, so the alphabet cannot drift out of sync again. This is +safe for the 4-option benchmarks: MMLU/GPQA gold answers are always A–D, so extracting a stray +higher letter still cannot match a 4-option gold — no existing score can flip; widening only *adds* +the ability to grade E–J. Added `tests/test_reward_choice_alphabet.py` (offline, pure stdlib): every +letter A–J end-to-end on RLPR MMLU-Pro, letter **and** integer-index normalization across A–J, plus +guards that MMLU/GPQA are unchanged, that a stray high letter cannot match a 4-option gold, and that +the prose article `"A"` is still not read as a choice. +**Follow-up:** the ten-option range is orthogonal to *which* letter wins when a model echoes the +option list — that ordering bug is tracked separately in #124 (PR #217). If #217 lands first, its +new trailing-standalone-letter fallback also needs to use `_CHOICE_LETTER_CLASS` rather than a fresh +`[A-D]` literal, or the cap is silently reintroduced on that path. + ## 2026-07-12 — code grader: add resource limits on top of the HOME/secrets fix #security #decision **Context:** issue #71 — the code grader (`run_pass_at_1`) runs untrusted miner/LLM candidate code. The core secret-leak fix (isolated throwaway HOME/cwd, scrubbed env, `python -I`) already landed on main. **Finding:** main's sandbox closes the HOME/secrets exfiltration vector but has **no resource limits** — an untrusted candidate can still exhaust host memory or fork-bomb the eval box within its wall-clock timeout (verified: a 4 GiB `bytearray` allocation runs to completion and "passes" on main). diff --git a/src/trinity/orchestration/reward.py b/src/trinity/orchestration/reward.py index b1da624..24073d6 100644 --- a/src/trinity/orchestration/reward.py +++ b/src/trinity/orchestration/reward.py @@ -21,9 +21,10 @@ Parse a JSON function-call payload and compare it against the ground-truth call schema stored in ``task.answer``. * ``mmlu`` / ``gpqa`` - Extract a single multiple-choice letter ``A-D`` (robust to phrasings such - as ``"the answer is (B)"``, ``"B)"``, ``"B."``) and compare to - ``task.answer``. + Extract a single multiple-choice letter (robust to phrasings such as + ``"the answer is (B)"``, ``"B)"``, ``"B."``) and compare to ``task.answer``. + The alphabet spans ``A-J`` so RLPR's ten-option MMLU-Pro items are gradeable; + 4-option benchmarks only ever use ``A-D``. * ``livecodebench`` / ``bigcodebench`` Execute candidate code against the task's tests in a subprocess sandbox with a timeout (``run_pass_at_1``). Never ``exec`` untrusted code in @@ -942,27 +943,42 @@ def _ref_to_str(reference: object) -> str: # Multiple choice: MMLU / GPQA # --------------------------------------------------------------------------- # Match in priority order. Earlier patterns are more explicit / trustworthy. +# Choice alphabet for the multiple-choice grading path. MMLU/GPQA are 4-option +# (A-D), but RLPR routes MMLU-Pro (`MMLUPro-1000_Avg2`) here and MMLU-Pro has up +# to TEN options (A-J) — capping at A-D silently scored every correct E-J answer +# 0.0. Widening is safe for the 4-option benchmarks: their gold answer is always +# A-D, so extracting a stray higher letter still cannot match a 4-option gold. +_CHOICE_ALPHABET: str = "ABCDEFGHIJ" +_CHOICE_LETTER_CLASS: str = f"[{_CHOICE_ALPHABET[0]}-{_CHOICE_ALPHABET[-1]}]" + _CHOICE_PATTERNS: tuple[re.Pattern[str], ...] = ( # Require the captured letter to be followed by a delimiter or end-of-word, # so "the answer Beats..." does NOT match "B" (P2 review fix). - re.compile(r"answer\s*(?:is|:)?\s*\(?\s*([A-D])\s*(?:[\).:]|\b)(?![A-Za-z])", re.I), - re.compile(r"\\boxed\s*\{\s*\(?\s*([A-D])\s*\)?\s*\}", re.I), - re.compile(r"\bfinal\s+answer\s*[:=]?\s*\(?\s*([A-D])(?![A-Za-z])", re.I), - re.compile(r"\boption\s*\(?\s*([A-D])(?![A-Za-z])", re.I), - re.compile(r"^\s*\(?\s*([A-D])\s*[\).:]", re.M), + re.compile( + rf"answer\s*(?:is|:)?\s*\(?\s*({_CHOICE_LETTER_CLASS})\s*(?:[\).:]|\b)(?![A-Za-z])", + re.I, + ), + re.compile(rf"\\boxed\s*\{{\s*\(?\s*({_CHOICE_LETTER_CLASS})\s*\)?\s*\}}", re.I), + re.compile(rf"\bfinal\s+answer\s*[:=]?\s*\(?\s*({_CHOICE_LETTER_CLASS})(?![A-Za-z])", re.I), + re.compile(rf"\boption\s*\(?\s*({_CHOICE_LETTER_CLASS})(?![A-Za-z])", re.I), + re.compile(rf"^\s*\(?\s*({_CHOICE_LETTER_CLASS})\s*[\).:]", re.M), ) def extract_choice_letter(text: str) -> str | None: - """Extract a single multiple-choice letter ``A``-``D`` from ``text``. + """Extract a single multiple-choice letter ``A``-``J`` from ``text``. Robust to common phrasings: ``"the answer is (B)"``, ``"Answer: C"``, ``"B)"``, ``"B."``, ``"\\boxed{D}"``, ``"Option A"``. Tries explicit answer-bearing patterns first; if none match, falls back to the **last** - standalone capital ``A``-``D`` token in the text (final answers usually come + standalone capital ``A``-``J`` token in the text (final answers usually come last). Letters embedded in words (e.g. the ``A`` in ``"And"``) are excluded by requiring word boundaries / delimiters. + The alphabet spans ``A``-``J`` (not just ``A``-``D``) because RLPR's MMLU-Pro + subset has up to ten options; 4-option benchmarks are unaffected since their + gold answer is always ``A``-``D``. + Args: text: Arbitrary model output. @@ -979,7 +995,7 @@ def extract_choice_letter(text: str) -> str | None: # it is essentially just the letter (e.g. "B", "(C)", "D."). This avoids the # English article "A" in prose like "A nice approach" being read as a choice. for line in reversed([ln.strip() for ln in text.splitlines() if ln.strip()]): - m = re.fullmatch(r"\(?\s*([A-D])\s*\)?[.:]?", line, re.I) + m = re.fullmatch(rf"\(?\s*({_CHOICE_LETTER_CLASS})\s*\)?[.:]?", line, re.I) if m: return m.group(1).upper() break # only inspect the final non-empty line @@ -998,11 +1014,14 @@ def _check_choice(candidate: str, reference: object) -> bool: def _normalize_reference_letter(reference: object) -> str | None: - """Coerce a reference answer to a single ``A``-``D`` letter. + """Coerce a reference answer to a single ``A``-``J`` letter. Accepts a letter string (``"B"``, ``"(B)"``) or a 0-based / 1-based integer index (``1`` -> ``"B"`` under 0-based; datasets vary, so a bare letter is preferred). Returns ``None`` if it cannot be resolved. + + The alphabet spans ``A``-``J`` so RLPR's MMLU-Pro gold answers (up to ten + options) are representable; 4-option benchmarks only ever use ``A``-``D``. """ if reference is None: return None @@ -1011,12 +1030,12 @@ def _normalize_reference_letter(reference: object) -> str | None: if letter is not None: return letter s = reference.strip().upper() - return s if s in {"A", "B", "C", "D"} else None + return s if s in set(_CHOICE_ALPHABET) else None if isinstance(reference, bool): return None if isinstance(reference, int): - if 0 <= reference <= 3: - return "ABCD"[reference] + if 0 <= reference < len(_CHOICE_ALPHABET): + return _CHOICE_ALPHABET[reference] return None return None diff --git a/tests/test_reward_choice_alphabet.py b/tests/test_reward_choice_alphabet.py new file mode 100644 index 0000000..b918178 --- /dev/null +++ b/tests/test_reward_choice_alphabet.py @@ -0,0 +1,82 @@ +"""Offline tests for the A-J multiple-choice alphabet (#116 / #122). + +RLPR routes MMLU-Pro (`MMLUPro-1000_Avg2`) to the choice grader, and MMLU-Pro has +up to ten options, but the grader was hard-capped at A-D: every correct E-J answer +scored 0.0 — a silently-wrong number, not a crash. These tests pin the widened +A-J alphabet and, just as importantly, that the genuinely 4-option benchmarks +(MMLU / GPQA) are unaffected. Pure stdlib: no network / GPU / torch. +""" +from __future__ import annotations + +import pytest + +from trinity.orchestration import reward as R + + +def _rlpr_ref(gold: str) -> dict: + return {"ground_truth": gold, "source": "MMLUPro-1000_Avg2"} + + +# --------------------------------------------------------------------------- # +# The bug: E-J were unrepresentable end-to-end +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("gold", list("ABCDEFGHIJ")) +def test_rlpr_mmlu_pro_grades_every_letter(gold): + assert R.score_text("rlpr", f"The answer is ({gold}).", _rlpr_ref(gold)) == 1.0 + + +@pytest.mark.parametrize("gold", list("EFGHIJ")) +def test_rlpr_mmlu_pro_still_rejects_wrong_letter(gold): + wrong = "A" if gold != "A" else "B" + assert R.score_text("rlpr", f"The answer is ({wrong}).", _rlpr_ref(gold)) == 0.0 + + +# --------------------------------------------------------------------------- # +# Helpers +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("letter", list("ABCDEFGHIJ")) +def test_extract_choice_letter_spans_a_to_j(letter): + assert R.extract_choice_letter(f"The answer is {letter}") == letter + assert R.extract_choice_letter(f"Final answer: ({letter})") == letter + assert R.extract_choice_letter(f"\\boxed{{{letter}}}") == letter + + +@pytest.mark.parametrize("letter", list("ABCDEFGHIJ")) +def test_normalize_reference_letter_spans_a_to_j(letter): + assert R._normalize_reference_letter(letter) == letter + + +def test_normalize_reference_letter_integer_index(): + assert R._normalize_reference_letter(0) == "A" + assert R._normalize_reference_letter(4) == "E" + assert R._normalize_reference_letter(9) == "J" + # Out of range stays unresolvable. + assert R._normalize_reference_letter(10) is None + assert R._normalize_reference_letter(-1) is None + + +def test_normalize_reference_letter_rejects_non_choices(): + assert R._normalize_reference_letter("Z") is None + assert R._normalize_reference_letter(None) is None + assert R._normalize_reference_letter(True) is None + + +# --------------------------------------------------------------------------- # +# Guards: the 4-option benchmarks must be unaffected +# --------------------------------------------------------------------------- # +def test_four_option_benchmarks_unchanged(): + assert R.score_text("mmlu", "The answer is (B).", "B") == 1.0 + assert R.score_text("mmlu", "The answer is (C).", "B") == 0.0 + assert R.score_text("gpqa", "Answer: D", "D") == 1.0 + + +def test_stray_high_letter_cannot_match_four_option_gold(): + # Widening only adds the ability to grade E-J; it can never flip a 4-option + # answer, because the gold there is always A-D. + assert R.score_text("mmlu", "The answer is (I).", "B") == 0.0 + assert R.score_text("gpqa", "The answer is (J).", "A") == 0.0 + + +def test_prose_article_a_still_not_read_as_choice(): + # The existing false-positive guard must survive the widened alphabet. + assert R.extract_choice_letter("A nice approach to think about it") is None