From 544d7c839248020be623a0063ecae93bfddc640c Mon Sep 17 00:00:00 2001 From: marktech0813 Date: Fri, 24 Jul 2026 16:02:30 +0000 Subject: [PATCH] fix(scoring): unwrap \color/\textcolor/\underline presentation wrappers These change appearance only; leave them intact and boxed colored answers never match plain gold (#497). Co-authored-by: Cursor --- pyproject.toml | 2 +- src/trinity/adapters/drop.py | 34 ++++++++++++------- src/trinity/orchestration/reward.py | 50 ++++++++++++++++++++++++++++ tests/test_reward_color_underline.py | 19 +++++++++++ 4 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 tests/test_reward_color_underline.py diff --git a/pyproject.toml b/pyproject.toml index 02d4f008..e911c0da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"] +dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"] [build-system] requires = ["hatchling"] diff --git a/src/trinity/adapters/drop.py b/src/trinity/adapters/drop.py index 1314803f..07fcbd4f 100644 --- a/src/trinity/adapters/drop.py +++ b/src/trinity/adapters/drop.py @@ -130,9 +130,12 @@ def _maybe_unbox(segment: str) -> str: return boxed if boxed is not None else "" -#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` — a -#: leading sign is part of a number's value, not wrapping noise. -_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-") +#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` and +#: the decimal point ``.`` — a leading sign or decimal point is part of a number's +#: value, not wrapping noise. A genuinely-trailing ``.`` (sentence period) is handled +#: by the dedicated rstrip retry in :func:`_normalize_token`, which can tell it apart +#: from a value-bearing leading point; a blanket edge-strip cannot. +_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-.") def _normalize_token(raw: str) -> str: @@ -147,20 +150,27 @@ def _normalize_token(raw: str) -> str: dropped the ``-`` and left commas to break ``float()``) did not deliver. A token that is ALREADY a number is recognised before any punctuation is - stripped: the edge-strip set includes ``.``, so a leading-decimal token like - ``".5"`` would otherwise lose its point and normalize to ``"5.0"`` — equal to - a gold ``"5"`` (false positive) and unequal to the value-identical gold - ``"0.5"`` (false negative). The official DROP ``_remove_punc`` tests - ``_is_number`` first and leaves numbers untouched for exactly this reason.""" + stripped: a leading-decimal token like ``".5"`` must not lose its point and + normalize to ``"5.0"`` — equal to a gold ``"5"`` (false positive) and unequal + to the value-identical gold ``"0.5"`` (false negative). The official DROP + ``_remove_punc`` tests ``_is_number`` first and leaves numbers untouched for + exactly this reason (issue #423). The float-first path alone only covers the + *bare* token: with ``.`` in the edge-strip set, wrapped forms like ``"$.5"`` + and ``".5."`` still lost the leading point on the second-chance path. So the + edge strip excludes ``.`` entirely, and a genuinely-trailing period (``".5."``, + ``"16.."``) is retried with an explicit ``rstrip(".")`` — right-side dots are + sentence punctuation, left-side dots are value.""" try: return str(float(raw.replace(",", ""))) except ValueError: pass core = raw.strip(_STRIP_EDGE) - try: - return str(float(core.replace(",", ""))) - except ValueError: - return _PUNCT.sub("", raw) + for cand in (core, core.rstrip(".")): + try: + return str(float(cand.replace(",", ""))) + except ValueError: + continue + return _PUNCT.sub("", raw) def _split_internal_hyphens(token: str) -> list[str]: diff --git a/src/trinity/orchestration/reward.py b/src/trinity/orchestration/reward.py index 7192ef8e..7ad9623f 100644 --- a/src/trinity/orchestration/reward.py +++ b/src/trinity/orchestration/reward.py @@ -728,6 +728,55 @@ def _peel_outer_parens(s: str) -> str: break s = s[1:-1] return s + +def _unwrap_color_underline(s: str) -> str: + """Peel presentation-only ``\\color``/``\\textcolor``/``\\underline`` wrappers. + + These change only appearance, not value — ``\\color{red}{5}`` must equal ``5`` + (issue #497), same contract as font-command unwrap. + """ + # \textcolor{col}{body} and \color{col}{body} + for cmd in ("textcolor", "color"): + marker = f"\\{cmd}" + idx = 0 + while True: + pos = s.find(marker, idx) + if pos == -1: + break + after = pos + len(marker) + if after < len(s) and s[after].isalpha(): + idx = after + continue + col = _scan_brace_group(s, after) + if col is None: + idx = after + continue + body = _scan_brace_group(s, col[1]) + if body is None: + idx = after + continue + s = s[:pos] + body[0] + s[body[1]:] + idx = pos + # \underline{body} + marker = "\\underline" + idx = 0 + while True: + pos = s.find(marker, idx) + if pos == -1: + break + after = pos + len(marker) + if after < len(s) and s[after].isalpha(): + idx = after + continue + body = _scan_brace_group(s, after) + if body is None: + idx = after + continue + s = s[:pos] + body[0] + s[body[1]:] + idx = pos + return s + + def normalize_math_answer(ans: str | None) -> str: r"""Normalize a math answer string for robust comparison. @@ -795,6 +844,7 @@ def normalize_math_answer(ans: str | None) -> str: # \text{5}/\mathrm{5} already do (otherwise a bold-formatted answer is a false # negative against a plain reference). s = _unwrap_font_commands(s) + s = _unwrap_color_underline(s) s = s.replace(r"\%", "").replace("%", "") # Degree symbol in either brace form: ``^\circ`` and ``^{\circ}``. The braced # form is common LaTeX and was previously left intact, so ``90^{\circ}`` never diff --git a/tests/test_reward_color_underline.py b/tests/test_reward_color_underline.py new file mode 100644 index 00000000..d99e68e4 --- /dev/null +++ b/tests/test_reward_color_underline.py @@ -0,0 +1,19 @@ +"""Presentation color/underline wrappers must not affect math equality (issue #497).""" + +from trinity.orchestration.reward import normalize_math_answer, score_text + + +def test_color_unwrap(): + assert normalize_math_answer(r"\color{red}{5}") == "5" + assert score_text("math500", r"\boxed{\color{red}{5}}", "5") == 1.0 + assert score_text("math500", r"\boxed{\color{red}{6}}", "5") == 0.0 + + +def test_textcolor_unwrap(): + assert normalize_math_answer(r"\textcolor{blue}{42}") == "42" + assert score_text("math500", r"\boxed{\textcolor{blue}{42}}", "42") == 1.0 + + +def test_underline_unwrap(): + assert normalize_math_answer(r"\underline{5}") == "5" + assert score_text("math500", r"\boxed{\underline{5}}", "5") == 1.0