Skip to content

Commit 38f810f

Browse files
tottenjordanclaude
andcommitted
fix: guard empty rubric text in substring fallback + debug logging
Address reviewer feedback on #6072: - Guard `if not rubric and normalized_rubric_text:` prevents empty judge Property: lines from matching every rubric via substring - Guard `if ct and` prevents empty rubric keys from matching - Add logger.debug when substring fallback rescues a match to track judge drift in eval logs - Add test_empty_property_text_does_not_match test case Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 9bb59fc commit 38f810f

2 files changed

Lines changed: 15 additions & 2 deletions

File tree

src/google/adk/evaluation/rubric_based_evaluator.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -417,14 +417,19 @@ def convert_auto_rater_response_to_score(
417417
normalized_rubric_text = _normalize_text(rubric_response.property_text)
418418
rubric = normalized_rubric_to_rubric_map.get(normalized_rubric_text, None)
419419

420-
if not rubric:
420+
if not rubric and normalized_rubric_text:
421421
candidates = [
422422
r
423423
for ct, r in normalized_rubric_to_rubric_map.items()
424-
if ct in normalized_rubric_text or normalized_rubric_text in ct
424+
if ct and (ct in normalized_rubric_text or normalized_rubric_text in ct)
425425
]
426426
if len(candidates) == 1:
427427
rubric = candidates[0]
428+
logger.debug(
429+
"Rubric substring fallback: '%s' → '%s'",
430+
rubric_response.property_text[:60],
431+
rubric.rubric_id,
432+
)
428433

429434
if rubric:
430435
rubric_scores.append(

tests/unittests/evaluation/test_rubric_based_evaluator.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -771,3 +771,11 @@ def test_ambiguous_substring_match_rejected(self):
771771
judge_property_text="Uses tools",
772772
)
773773
assert len(result.rubric_scores) == 0
774+
775+
def test_empty_property_text_does_not_match(self):
776+
"""Empty judge Property: line must not match via substring fallback."""
777+
result = self._build_evaluator_and_score(
778+
rubric_texts=["Uses tools correctly"],
779+
judge_property_text="",
780+
)
781+
assert len(result.rubric_scores) == 0

0 commit comments

Comments
 (0)