From 8493f4ab7708e234d30434f47369a04ef3bf6098 Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Tue, 7 Jul 2026 01:39:56 -0700
Subject: [PATCH 1/2] fix(skill-eval): match punctuated exact answers

---
 src/benchflow/templates/judge.py.tmpl | 12 +++++-
 tests/test_skill_eval_sweep.py        | 59 +++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/src/benchflow/templates/judge.py.tmpl b/src/benchflow/templates/judge.py.tmpl
index 8db82a722..781403bfe 100644
--- a/src/benchflow/templates/judge.py.tmpl
+++ b/src/benchflow/templates/judge.py.tmpl
@@ -163,12 +163,19 @@ def judge_with_llm(trajectory_text: str) -> float:
         return 0.0
 
 
+def exact_ground_truth_pattern(ground_truth: str) -> re.Pattern:
+    """Return a regex that matches a standalone exact ground-truth string."""
+    return re.compile(r"(?<!\w)" + re.escape(ground_truth) + r"(?!\w)")
+
+
 def judge_exact_match() -> float:
     """Fallback: exact string match against ground_truth."""
     ground_truth = CASE.get("ground_truth", "").strip()
     if not ground_truth:
         return 0.0
 
+    pattern = exact_ground_truth_pattern(ground_truth)
+
     # Only check agent output files, not arbitrary filesystem content
     search_paths = [AGENT_LOG_DIR, APP_DIR]
     for search_root in search_paths:
@@ -179,8 +186,9 @@ def judge_exact_match() -> float:
                 continue
             try:
                 content = f.read_text()
-                # Check for exact match on word boundaries (not substring)
-                if re.search(r'\b' + re.escape(ground_truth) + r'\b', content):
+                # Check for an exact standalone answer without requiring
+                # code-like punctuation such as ")" to form a word boundary.
+                if pattern.search(content):
                     return 1.0
             except (UnicodeDecodeError, PermissionError):
                 pass
diff --git a/tests/test_skill_eval_sweep.py b/tests/test_skill_eval_sweep.py
index 829ef65c2..d5f66d0af 100644
--- a/tests/test_skill_eval_sweep.py
+++ b/tests/test_skill_eval_sweep.py
@@ -9,6 +9,9 @@
 
 import json
 import math
+import os
+import subprocess
+import sys
 import tomllib
 from pathlib import Path
 
@@ -174,6 +177,62 @@ def test_generated_test_sh_copies_judge_result(self, tmp_path):
         assert "/logs/verifier/judge_result.json" in test_sh
 
 
+# Issue #897 — exact-match judge delimiter handling
+
+
+class TestExactMatchJudgeDelimiters:
+    """Guards #897: exact-match judge handles code-like answer delimiters."""
+
+    @pytest.mark.parametrize(
+        ("content", "expected_reward"),
+        [
+            ("The method is getPageCount().\n", 1.0),
+            ("The method is getPageCount()Wrapper.\n", 0.0),
+        ],
+    )
+    def test_non_word_suffix_ground_truth_uses_word_char_delimiters(
+        self, tmp_path, content, expected_reward
+    ):
+        """Guards GitHub issue #897 against exact answers ending in non-word chars."""
+        skill_dir = tmp_path / "skill"
+        skill_dir.mkdir()
+        ds = EvalDataset(
+            skill_name="method-skill",
+            skill_dir=skill_dir,
+            cases=[
+                EvalCase(
+                    id="case-001",
+                    question="Which method returns the page count?",
+                    ground_truth="getPageCount()",
+                )
+            ],
+        )
+        task = generate_tasks(ds, tmp_path / "tasks", with_skill=False)[0]
+        verifier_dir = TaskPaths(task).tests_dir
+
+        agent_log_dir = tmp_path / "logs" / "agent"
+        agent_log_dir.mkdir(parents=True)
+        (agent_log_dir / "answer.txt").write_text(content)
+
+        workspace = tmp_path / "workspace"
+        workspace.mkdir()
+        subprocess.run(
+            [sys.executable, str(verifier_dir / "judge.py")],
+            check=True,
+            env={
+                **os.environ,
+                "BENCHFLOW_VERIFIER_DIR": str(verifier_dir),
+                "BENCHFLOW_AGENT_LOG_DIR": str(agent_log_dir),
+                "BENCHFLOW_WORKSPACE": str(workspace),
+            },
+            text=True,
+            capture_output=True,
+        )
+
+        reward = json.loads((verifier_dir / "reward.json").read_text())
+        assert reward == {"reward": expected_reward}
+
+
 # Issue #424 — evals.json schema validation
 
 

From 95de7752c938b21d57ef9c06214c5f33b12d35be Mon Sep 17 00:00:00 2001
From: Bingran You <bingran.you@berkeley.edu>
Date: Tue, 7 Jul 2026 01:41:03 -0700
Subject: [PATCH 2/2] test(skill-eval): name PR 900 regression

---
 tests/test_skill_eval_sweep.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_skill_eval_sweep.py b/tests/test_skill_eval_sweep.py
index d5f66d0af..b9b1290c2 100644
--- a/tests/test_skill_eval_sweep.py
+++ b/tests/test_skill_eval_sweep.py
@@ -193,7 +193,7 @@ class TestExactMatchJudgeDelimiters:
     def test_non_word_suffix_ground_truth_uses_word_char_delimiters(
         self, tmp_path, content, expected_reward
     ):
-        """Guards GitHub issue #897 against exact answers ending in non-word chars."""
+        """Guards PR #900 / issue #897 against punctuated exact answers."""
         skill_dir = tmp_path / "skill"
         skill_dir.mkdir()
         ds = EvalDataset(