diff --git a/openagent_eval/synthesis/adversarial.py b/openagent_eval/synthesis/adversarial.py index c139518..9f90468 100644 --- a/openagent_eval/synthesis/adversarial.py +++ b/openagent_eval/synthesis/adversarial.py @@ -294,6 +294,22 @@ def _parse_response( import re as _re test_cases: list[TestCase] = [] + seen_questions: set[str] = set() + + def _add_test_case(question: str, answer: str) -> None: + """Add test case if question not already seen.""" + if question and question not in seen_questions: + seen_questions.add(question) + test_cases.append( + TestCase( + question=question, + ground_truth=answer, + context=context, + test_type=test_type, + source_document=source_document, + chunk_index=chunk_index, + ) + ) # Clean code blocks and normalize text text = raw_response.strip() @@ -312,20 +328,10 @@ def _parse_response( try: data = json.loads(text) if isinstance(data, dict): - question = data.get("question", "").strip() - answer = data.get("answer", "").strip() - if question: - test_cases.append( - TestCase( - question=question, - ground_truth=answer, - context=context, - test_type=test_type, - source_document=source_document, - chunk_index=chunk_index, - ) - ) - return test_cases + question = data.get("question", "") + answer = data.get("answer", "") + _add_test_case(question, answer) + # Don't return here - continue to other strategies to find more questions except (json.JSONDecodeError, ValueError): pass @@ -347,19 +353,9 @@ def _parse_response( if isinstance(data, list): for item in data: if isinstance(item, dict): - question = item.get("question", "").strip() - answer = item.get("answer", "").strip() - if question: - test_cases.append( - TestCase( - question=question, - ground_truth=answer, - context=context, - test_type=test_type, - source_document=source_document, - chunk_index=chunk_index, - ) - ) + question = item.get("question", "") + answer = item.get("answer", "") + _add_test_case(question, answer) if test_cases: return test_cases except (json.JSONDecodeError, ValueError): @@ -378,19 +374,9 @@ def _parse_response( obj_str = _re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]", "", obj_str) item = json.loads(obj_str) - question = item.get("question", "").strip() - answer = item.get("answer", "").strip() - if question: - test_cases.append( - TestCase( - question=question, - ground_truth=answer, - context=context, - test_type=test_type, - source_document=source_document, - chunk_index=chunk_index, - ) - ) + question = item.get("question", "") + answer = item.get("answer", "") + _add_test_case(question, answer) except (json.JSONDecodeError, ValueError): continue @@ -407,19 +393,7 @@ def _parse_response( matches = qa_pattern.findall(text) for question, answer in matches: - question = question.strip() - answer = answer.strip() - if question: - test_cases.append( - TestCase( - question=question, - ground_truth=answer, - context=context, - test_type=test_type, - source_document=source_document, - chunk_index=chunk_index, - ) - ) + _add_test_case(question, answer) if test_cases: return test_cases @@ -431,20 +405,8 @@ def _parse_response( questions = question_pattern.findall(text) answers = answer_pattern.findall(text) - for q, a in zip(questions, answers): - q = q.strip() - a = a.strip() - if q: - test_cases.append( - TestCase( - question=q, - ground_truth=a, - context=context, - test_type=test_type, - source_document=source_document, - chunk_index=chunk_index, - ) - ) + for q, a in zip(questions, answers, strict=False): + _add_test_case(q, a) if test_cases: return test_cases diff --git a/openagent_eval/synthesis/question_gen.py b/openagent_eval/synthesis/question_gen.py index 9b65d9f..044e1f1 100644 --- a/openagent_eval/synthesis/question_gen.py +++ b/openagent_eval/synthesis/question_gen.py @@ -145,6 +145,22 @@ def _parse_response( import re as _re test_cases: list[TestCase] = [] + seen_questions: set[str] = set() + + def _add_test_case(question: str, answer: str) -> None: + """Add test case if question not already seen.""" + if question and question not in seen_questions: + seen_questions.add(question) + test_cases.append( + TestCase( + question=question, + ground_truth=answer, + context=context, + test_type=TestCaseType.STANDARD, + source_document=source_document, + chunk_index=chunk_index, + ) + ) # Clean code blocks and normalize text text = raw_response.strip() @@ -166,17 +182,8 @@ def _parse_response( question = data.get("question", "").strip() answer = data.get("answer", "").strip() if question and answer: - test_cases.append( - TestCase( - question=question, - ground_truth=answer, - context=context, - test_type=TestCaseType.STANDARD, - source_document=source_document, - chunk_index=chunk_index, - ) - ) - return test_cases + _add_test_case(question, answer) + # Don't return here - continue to other strategies to find more questions except (json.JSONDecodeError, ValueError): pass @@ -201,16 +208,7 @@ def _parse_response( question = item.get("question", "").strip() answer = item.get("answer", "").strip() if question and answer: - test_cases.append( - TestCase( - question=question, - ground_truth=answer, - context=context, - test_type=TestCaseType.STANDARD, - source_document=source_document, - chunk_index=chunk_index, - ) - ) + _add_test_case(question, answer) if test_cases: return test_cases except (json.JSONDecodeError, ValueError): @@ -232,16 +230,7 @@ def _parse_response( question = item.get("question", "").strip() answer = item.get("answer", "").strip() if question and answer: - test_cases.append( - TestCase( - question=question, - ground_truth=answer, - context=context, - test_type=TestCaseType.STANDARD, - source_document=source_document, - chunk_index=chunk_index, - ) - ) + _add_test_case(question, answer) except (json.JSONDecodeError, ValueError): continue @@ -261,16 +250,7 @@ def _parse_response( question = question.strip() answer = answer.strip() if question and answer: - test_cases.append( - TestCase( - question=question, - ground_truth=answer, - context=context, - test_type=TestCaseType.STANDARD, - source_document=source_document, - chunk_index=chunk_index, - ) - ) + _add_test_case(question, answer) if test_cases: return test_cases @@ -282,20 +262,11 @@ def _parse_response( questions = question_pattern.findall(text) answers = answer_pattern.findall(text) - for q, a in zip(questions, answers): + for q, a in zip(questions, answers, strict=False): q = q.strip() a = a.strip() if q and a: - test_cases.append( - TestCase( - question=q, - ground_truth=a, - context=context, - test_type=TestCaseType.STANDARD, - source_document=source_document, - chunk_index=chunk_index, - ) - ) + _add_test_case(q, a) if test_cases: return test_cases