diff --git a/pyproject.toml b/pyproject.toml index 02d4f00..e911c0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"] +dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"] [build-system] requires = ["hatchling"] diff --git a/src/trinity/adapters/drop.py b/src/trinity/adapters/drop.py index 1314803..07fcbd4 100644 --- a/src/trinity/adapters/drop.py +++ b/src/trinity/adapters/drop.py @@ -130,9 +130,12 @@ def _maybe_unbox(segment: str) -> str: return boxed if boxed is not None else "" -#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` — a -#: leading sign is part of a number's value, not wrapping noise. -_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-") +#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` and +#: the decimal point ``.`` — a leading sign or decimal point is part of a number's +#: value, not wrapping noise. A genuinely-trailing ``.`` (sentence period) is handled +#: by the dedicated rstrip retry in :func:`_normalize_token`, which can tell it apart +#: from a value-bearing leading point; a blanket edge-strip cannot. +_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-.") def _normalize_token(raw: str) -> str: @@ -147,20 +150,27 @@ def _normalize_token(raw: str) -> str: dropped the ``-`` and left commas to break ``float()``) did not deliver. A token that is ALREADY a number is recognised before any punctuation is - stripped: the edge-strip set includes ``.``, so a leading-decimal token like - ``".5"`` would otherwise lose its point and normalize to ``"5.0"`` — equal to - a gold ``"5"`` (false positive) and unequal to the value-identical gold - ``"0.5"`` (false negative). The official DROP ``_remove_punc`` tests - ``_is_number`` first and leaves numbers untouched for exactly this reason.""" + stripped: a leading-decimal token like ``".5"`` must not lose its point and + normalize to ``"5.0"`` — equal to a gold ``"5"`` (false positive) and unequal + to the value-identical gold ``"0.5"`` (false negative). The official DROP + ``_remove_punc`` tests ``_is_number`` first and leaves numbers untouched for + exactly this reason (issue #423). The float-first path alone only covers the + *bare* token: with ``.`` in the edge-strip set, wrapped forms like ``"$.5"`` + and ``".5."`` still lost the leading point on the second-chance path. So the + edge strip excludes ``.`` entirely, and a genuinely-trailing period (``".5."``, + ``"16.."``) is retried with an explicit ``rstrip(".")`` — right-side dots are + sentence punctuation, left-side dots are value.""" try: return str(float(raw.replace(",", ""))) except ValueError: pass core = raw.strip(_STRIP_EDGE) - try: - return str(float(core.replace(",", ""))) - except ValueError: - return _PUNCT.sub("", raw) + for cand in (core, core.rstrip(".")): + try: + return str(float(cand.replace(",", ""))) + except ValueError: + continue + return _PUNCT.sub("", raw) def _split_internal_hyphens(token: str) -> list[str]: diff --git a/src/trinity/orchestration/reward.py b/src/trinity/orchestration/reward.py index 7192ef8..6395e08 100644 --- a/src/trinity/orchestration/reward.py +++ b/src/trinity/orchestration/reward.py @@ -505,6 +505,14 @@ def extract_last_number(text: str) -> str | None: candidates: list[tuple[int, str]] = [] for _start, end, term in _iter_latex_frac_sqrt_spans(text): candidates.append((end, term)) + # Binomial coefficients: whole ``\binom{n}{k}`` / ``{n \choose k}`` so the + # lower index digit is not taken alone (issue #489). + for m in re.finditer( + r"\\binom(?![a-zA-Z])\s*\{[^{}]*\}\s*\{[^{}]*\}" + r"|\{[^{}]*\\choose\s*[^{}]*\}", + text, + ): + candidates.append((m.end(), m.group(0))) # Match a simple fraction a/b FIRST (so "1/2" is kept whole, not read as "2"), # then decimals/integers like -1,234.56 or 42 or .5 ; require a digit somewhere. brace = r"\{(?:[^{}]|\{[^{}]*\})*\}" @@ -521,7 +529,9 @@ def extract_last_number(text: str) -> str | None: candidates.append((m.end(), m.group(0).replace(",", "").replace(" ", ""))) if not candidates: return None - candidates.sort(key=lambda item: item[0]) + # Latest end wins; on a tie prefer the longer span so ``\binom{5}{2}`` beats + # the trailing index digit that ends at the same index (issue #489). + candidates.sort(key=lambda item: (item[0], len(item[1]))) return candidates[-1][1] @@ -820,6 +830,16 @@ def normalize_math_answer(ans: str | None) -> str: # The [dt]? family — \frac, \dfrac and \tfrac — all render the same value. s = _unwrap_latex_fractions(s) s = re.sub(r"\\[dt]?frac\s*(\d)\s*(\d)", r"\1/\2", s) + # Binomial coefficients → sympy ``binomial(n,k)`` so ``\binom{5}{2}`` equals + # gold ``10`` (issue #489). Also TeX ``{n \choose k}`` / bare ``n \choose k`` + # (outer-brace peel may remove the wrapping ``{...}`` before we get here). + s = re.sub( + r"\\binom(?![a-zA-Z])\s*\{([^{}]*)\}\s*\{([^{}]*)\}", + r"binomial(\1,\2)", + s, + ) + s = re.sub(r"\{([^{}]*)\\choose\s*([^{}]*)\}", r"binomial(\1,\2)", s) + s = re.sub(r"(-?\d+)\s*\\choose\s*(-?\d+)", r"binomial(\1,\2)", s) # \sqrt{x} -> sqrt(x), and the unbraced single-token \sqrt2 -> sqrt(2). Like the # \frac handling just above, the braced and bare forms render the SAME value, so # they must normalize identically (else \sqrt{2} is a false negative against a diff --git a/tests/test_reward_binom.py b/tests/test_reward_binom.py new file mode 100644 index 0000000..4dd581f --- /dev/null +++ b/tests/test_reward_binom.py @@ -0,0 +1,23 @@ +"""Binomial ``\\binom`` / ``\\choose`` must not grade as the lower index (issue #489).""" + +from trinity.orchestration.reward import extract_last_number, normalize_math_answer, score_text + + +def test_extract_binom_as_whole_term(): + assert extract_last_number(r"\binom{5}{2}") == r"\binom{5}{2}" + assert extract_last_number(r"{5 \choose 2}") == r"{5 \choose 2}" + + +def test_normalize_binom_to_sympy(): + assert "binomial(5,2)" in normalize_math_answer(r"\binom{5}{2}").replace(" ", "") + assert "\\binom" not in normalize_math_answer(r"\binom{5}{2}") + + +def test_unboxed_binom_grades_value_not_index(): + assert score_text("math500", r"answer is $\binom{5}{2}$", "10") == 1.0 + assert score_text("math500", r"answer is $\binom{5}{2}$", "2") == 0.0 + + +def test_boxed_binom_and_choose(): + assert score_text("math500", r"\boxed{\binom{5}{2}}", "10") == 1.0 + assert score_text("math500", r"\boxed{{5 \choose 2}}", "10") == 1.0