From 8e9626e8862486be01d36a4dfdc8cf923e992114 Mon Sep 17 00:00:00 2001 From: marktech0813 Date: Fri, 24 Jul 2026 14:36:21 +0000 Subject: [PATCH] fix(scoring): keep relational prefixes on extracted math answers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extract_last_number ignored \\neq/\\le/… before a literal, so "\\neq 5" graded equal to gold "5". Attach the relation to the extract (#479). Co-authored-by: Cursor --- pyproject.toml | 2 +- src/trinity/adapters/drop.py | 34 +++++++++++++++++--------- src/trinity/orchestration/reward.py | 26 +++++++++++++++----- tests/test_reward_relational_prefix.py | 16 ++++++++++++ 4 files changed, 59 insertions(+), 19 deletions(-) create mode 100644 tests/test_reward_relational_prefix.py diff --git a/pyproject.toml b/pyproject.toml index 02d4f008..e911c0da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"] +dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"] [build-system] requires = ["hatchling"] diff --git a/src/trinity/adapters/drop.py b/src/trinity/adapters/drop.py index 1314803f..07fcbd4f 100644 --- a/src/trinity/adapters/drop.py +++ b/src/trinity/adapters/drop.py @@ -130,9 +130,12 @@ def _maybe_unbox(segment: str) -> str: return boxed if boxed is not None else "" -#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` — a -#: leading sign is part of a number's value, not wrapping noise. -_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-") +#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` and +#: the decimal point ``.`` — a leading sign or decimal point is part of a number's +#: value, not wrapping noise. A genuinely-trailing ``.`` (sentence period) is handled +#: by the dedicated rstrip retry in :func:`_normalize_token`, which can tell it apart +#: from a value-bearing leading point; a blanket edge-strip cannot. +_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-.") def _normalize_token(raw: str) -> str: @@ -147,20 +150,27 @@ def _normalize_token(raw: str) -> str: dropped the ``-`` and left commas to break ``float()``) did not deliver. A token that is ALREADY a number is recognised before any punctuation is - stripped: the edge-strip set includes ``.``, so a leading-decimal token like - ``".5"`` would otherwise lose its point and normalize to ``"5.0"`` — equal to - a gold ``"5"`` (false positive) and unequal to the value-identical gold - ``"0.5"`` (false negative). The official DROP ``_remove_punc`` tests - ``_is_number`` first and leaves numbers untouched for exactly this reason.""" + stripped: a leading-decimal token like ``".5"`` must not lose its point and + normalize to ``"5.0"`` — equal to a gold ``"5"`` (false positive) and unequal + to the value-identical gold ``"0.5"`` (false negative). The official DROP + ``_remove_punc`` tests ``_is_number`` first and leaves numbers untouched for + exactly this reason (issue #423). The float-first path alone only covers the + *bare* token: with ``.`` in the edge-strip set, wrapped forms like ``"$.5"`` + and ``".5."`` still lost the leading point on the second-chance path. So the + edge strip excludes ``.`` entirely, and a genuinely-trailing period (``".5."``, + ``"16.."``) is retried with an explicit ``rstrip(".")`` — right-side dots are + sentence punctuation, left-side dots are value.""" try: return str(float(raw.replace(",", ""))) except ValueError: pass core = raw.strip(_STRIP_EDGE) - try: - return str(float(core.replace(",", ""))) - except ValueError: - return _PUNCT.sub("", raw) + for cand in (core, core.rstrip(".")): + try: + return str(float(cand.replace(",", ""))) + except ValueError: + continue + return _PUNCT.sub("", raw) def _split_internal_hyphens(token: str) -> list[str]: diff --git a/src/trinity/orchestration/reward.py b/src/trinity/orchestration/reward.py index 7192ef8e..a711f73f 100644 --- a/src/trinity/orchestration/reward.py +++ b/src/trinity/orchestration/reward.py @@ -502,9 +502,12 @@ def extract_last_number(text: str) -> str | None: # comma so the thousands-separator branch below reads it as one number instead # of splitting it into "1" and "000". text = text.replace("{,}", ",") - candidates: list[tuple[int, str]] = [] - for _start, end, term in _iter_latex_frac_sqrt_spans(text): - candidates.append((end, term)) + # (start, end, term) so a relational prefix immediately before the winning + # span can be attached — otherwise ``\neq 5`` extracts as ``"5"`` and falsely + # matches a bare gold ``5`` (issue #479). + candidates: list[tuple[int, int, str]] = [] + for start, end, term in _iter_latex_frac_sqrt_spans(text): + candidates.append((start, end, term)) # Match a simple fraction a/b FIRST (so "1/2" is kept whole, not read as "2"), # then decimals/integers like -1,234.56 or 42 or .5 ; require a digit somewhere. brace = r"\{(?:[^{}]|\{[^{}]*\})*\}" @@ -518,11 +521,22 @@ def extract_last_number(text: str) -> str | None: r"|-?\.\d+" ) for m in pattern.finditer(text): - candidates.append((m.end(), m.group(0).replace(",", "").replace(" ", ""))) + candidates.append( + (m.start(), m.end(), m.group(0).replace(",", "").replace(" ", "")) + ) if not candidates: return None - candidates.sort(key=lambda item: item[0]) - return candidates[-1][1] + candidates.sort(key=lambda item: item[1]) + start, end, term = candidates[-1] + # If a relation operator sits immediately before this span, keep it so the + # extracted answer is ``\neq5`` / ``≤5`` rather than a bare magnitude. + rel = re.search( + r"(?:\\(?:neq|ne|leq|le|geq|ge|lt|gt)|[≠≤≥<>])\s*$", + text[:start], + ) + if rel is not None: + term = text[rel.start() : end].replace(",", "").replace(" ", "") + return term def _is_thousands_grouped_number(s: str) -> bool: diff --git a/tests/test_reward_relational_prefix.py b/tests/test_reward_relational_prefix.py new file mode 100644 index 00000000..52228ad6 --- /dev/null +++ b/tests/test_reward_relational_prefix.py @@ -0,0 +1,16 @@ +"""Relational prefixes must not grade as bare magnitudes (#479).""" +from __future__ import annotations +from trinity.orchestration import reward as R + +def test_inequality_does_not_match_bare_gold(): + for cand in (r"\neq 5", r"\ne 5", r"\le 5", r"\leq 5", r"\ge 5", r"\geq 5", r"\lt 5", r"\gt 5", "≠5", "≤5", "≥5"): + assert R.score_text("math500", cand, "5") == 0.0, cand + +def test_same_relation_still_matches(): + assert R.score_text("math500", r"\neq 5", r"\neq 5") == 1.0 + assert R.score_text("math500", r"\le 5", r"\le 5") == 1.0 + +def test_bare_number_unaffected(): + assert R.score_text("math500", "5", "5") == 1.0 + assert R.score_text("math500", "The answer is 5.", "5") == 1.0 + assert R.extract_last_number(r"\neq 5") == r"\neq5"