Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ dependencies = [
]

[project.optional-dependencies]
dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"]
dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"]

[build-system]
requires = ["hatchling"]
Expand Down
34 changes: 22 additions & 12 deletions src/trinity/adapters/drop.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,12 @@ def _maybe_unbox(segment: str) -> str:
return boxed if boxed is not None else ""


#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` — a
#: leading sign is part of a number's value, not wrapping noise.
_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-")
#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` and
#: the decimal point ``.`` — a leading sign or decimal point is part of a number's
#: value, not wrapping noise. A genuinely-trailing ``.`` (sentence period) is handled
#: by the dedicated rstrip retry in :func:`_normalize_token`, which can tell it apart
#: from a value-bearing leading point; a blanket edge-strip cannot.
_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-.")


def _normalize_token(raw: str) -> str:
Expand All @@ -147,20 +150,27 @@ def _normalize_token(raw: str) -> str:
dropped the ``-`` and left commas to break ``float()``) did not deliver.

A token that is ALREADY a number is recognised before any punctuation is
stripped: the edge-strip set includes ``.``, so a leading-decimal token like
``".5"`` would otherwise lose its point and normalize to ``"5.0"`` — equal to
a gold ``"5"`` (false positive) and unequal to the value-identical gold
``"0.5"`` (false negative). The official DROP ``_remove_punc`` tests
``_is_number`` first and leaves numbers untouched for exactly this reason."""
stripped: a leading-decimal token like ``".5"`` must not lose its point and
normalize to ``"5.0"`` — equal to a gold ``"5"`` (false positive) and unequal
to the value-identical gold ``"0.5"`` (false negative). The official DROP
``_remove_punc`` tests ``_is_number`` first and leaves numbers untouched for
exactly this reason (issue #423). The float-first path alone only covers the
*bare* token: with ``.`` in the edge-strip set, wrapped forms like ``"$.5"``
and ``".5."`` still lost the leading point on the second-chance path. So the
edge strip excludes ``.`` entirely, and a genuinely-trailing period (``".5."``,
``"16.."``) is retried with an explicit ``rstrip(".")`` — right-side dots are
sentence punctuation, left-side dots are value."""
try:
return str(float(raw.replace(",", "")))
except ValueError:
pass
core = raw.strip(_STRIP_EDGE)
try:
return str(float(core.replace(",", "")))
except ValueError:
return _PUNCT.sub("", raw)
for cand in (core, core.rstrip(".")):
try:
return str(float(cand.replace(",", "")))
except ValueError:
continue
return _PUNCT.sub("", raw)


def _split_internal_hyphens(token: str) -> list[str]:
Expand Down
7 changes: 6 additions & 1 deletion src/trinity/orchestration/reward.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,14 +514,19 @@ def extract_last_number(text: str) -> str | None:
r"|-?\d+\s*/\s*-?\d+"
# Scientific notation BEFORE bare decimals so "1e3" is not read as "3".
r"|-?(?:\d+(?:\.\d+)?|\.\d+)[eE][+-]?\d+"
# LaTeX scientific ``a\times 10^{b}`` BEFORE bare digits so the exponent
# is not taken alone (issue #485). Sibling of the ``1e3`` form above.
rf"|-?(?:\d+(?:\.\d+)?|\.\d+)\s*\\times\s*10\s*\^(?:{brace}|[+-]?\d+)"
r"|-?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?"
r"|-?\.\d+"
)
for m in pattern.finditer(text):
candidates.append((m.end(), m.group(0).replace(",", "").replace(" ", "")))
if not candidates:
return None
candidates.sort(key=lambda item: item[0])
# Latest end wins; on a tie prefer the longer span so ``3\times 10^{2}`` beats
# the trailing exponent digit that ends at the same index (issue #485).
candidates.sort(key=lambda item: (item[0], len(item[1])))
return candidates[-1][1]


Expand Down
24 changes: 24 additions & 0 deletions tests/test_reward_times_ten_scientific.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
"""Unboxed ``a\\times 10^{b}`` must not grade as the exponent digit (issue #485)."""

from trinity.orchestration.reward import extract_last_number, score_text


def test_extract_times_ten_as_whole_term():
# Spaces inside the match are stripped (same as other extract candidates).
assert extract_last_number(r"3\times 10^{2}") == r"3\times10^{2}"
assert extract_last_number(r"answer is $3\times 10^{2}$") == r"3\times10^{2}"
assert extract_last_number(r"2.5\times 10^3") == r"2.5\times10^3"


def test_unboxed_times_ten_grades_value_not_exponent():
assert score_text("math500", r"the answer is $3\times 10^{2}$", "300") == 1.0
assert score_text("math500", r"the answer is $3\times 10^{2}$", "2") == 0.0


def test_boxed_times_ten_still_works():
assert score_text("math500", r"\boxed{3\times 10^{2}}", "300") == 1.0


def test_ascii_scientific_still_works():
assert extract_last_number("the answer is 1e3") == "1e3"
assert score_text("math500", "the answer is 1e3", "1000") == 1.0
Loading