Skip to content

fix(synthesis): Strategy 0 in _parse_response returns prematurely on single JSON objects (#94) #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
himanshu231204 merged 3 commits into from
Jul 17, 2026
+52 −119
Merged

fix(synthesis): Strategy 0 in _parse_response returns prematurely on single JSON objects (#94) #175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1a9ae13
fix(synthesis): prevent premature return in _parse_response Strategy 0
himanshu231204 Jul 17, 2026
fcff4ab
fix(synthesis): add strict=False to zip() calls for ruff B905 compliance
himanshu231204 Jul 17, 2026
2239b1e
Merge branch 'main' into fix/synthesis-parse-response-premature-return
himanshu231204 Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 29 additions & 67 deletions openagent_eval/synthesis/adversarial.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,22 @@ def _parse_response(
import re as _re

test_cases: list[TestCase] = []
seen_questions: set[str] = set()

def _add_test_case(question: str, answer: str) -> None:
"""Add test case if question not already seen."""
if question and question not in seen_questions:
seen_questions.add(question)
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=test_type,
source_document=source_document,
chunk_index=chunk_index,
)
)

# Clean code blocks and normalize text
text = raw_response.strip()
Expand All @@ -312,20 +328,10 @@ def _parse_response(
try:
data = json.loads(text)
if isinstance(data, dict):
question = data.get("question", "").strip()
answer = data.get("answer", "").strip()
if question:
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=test_type,
source_document=source_document,
chunk_index=chunk_index,
)
)
return test_cases
question = data.get("question", "")
answer = data.get("answer", "")
_add_test_case(question, answer)
# Don't return here - continue to other strategies to find more questions
except (json.JSONDecodeError, ValueError):
pass

Expand All @@ -347,19 +353,9 @@ def _parse_response(
if isinstance(data, list):
for item in data:
if isinstance(item, dict):
question = item.get("question", "").strip()
answer = item.get("answer", "").strip()
if question:
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=test_type,
source_document=source_document,
chunk_index=chunk_index,
)
)
question = item.get("question", "")
answer = item.get("answer", "")
_add_test_case(question, answer)
if test_cases:
return test_cases
except (json.JSONDecodeError, ValueError):
Expand All @@ -378,19 +374,9 @@ def _parse_response(
obj_str = _re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]", "", obj_str)

item = json.loads(obj_str)
question = item.get("question", "").strip()
answer = item.get("answer", "").strip()
if question:
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=test_type,
source_document=source_document,
chunk_index=chunk_index,
)
)
question = item.get("question", "")
answer = item.get("answer", "")
_add_test_case(question, answer)
except (json.JSONDecodeError, ValueError):
continue

Expand All @@ -407,19 +393,7 @@ def _parse_response(
matches = qa_pattern.findall(text)

for question, answer in matches:
question = question.strip()
answer = answer.strip()
if question:
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=test_type,
source_document=source_document,
chunk_index=chunk_index,
)
)
_add_test_case(question, answer)

if test_cases:
return test_cases
Expand All @@ -431,20 +405,8 @@ def _parse_response(
questions = question_pattern.findall(text)
answers = answer_pattern.findall(text)

for q, a in zip(questions, answers):
q = q.strip()
a = a.strip()
if q:
test_cases.append(
TestCase(
question=q,
ground_truth=a,
context=context,
test_type=test_type,
source_document=source_document,
chunk_index=chunk_index,
)
)
for q, a in zip(questions, answers, strict=False):
_add_test_case(q, a)

if test_cases:
return test_cases
Expand Down
75 changes: 23 additions & 52 deletions openagent_eval/synthesis/question_gen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,22 @@ def _parse_response(
import re as _re

test_cases: list[TestCase] = []
seen_questions: set[str] = set()

def _add_test_case(question: str, answer: str) -> None:
"""Add test case if question not already seen."""
if question and question not in seen_questions:
seen_questions.add(question)
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=TestCaseType.STANDARD,
source_document=source_document,
chunk_index=chunk_index,
)
)

# Clean code blocks and normalize text
text = raw_response.strip()
Expand All @@ -166,17 +182,8 @@ def _parse_response(
question = data.get("question", "").strip()
answer = data.get("answer", "").strip()
if question and answer:
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=TestCaseType.STANDARD,
source_document=source_document,
chunk_index=chunk_index,
)
)
return test_cases
_add_test_case(question, answer)
# Don't return here - continue to other strategies to find more questions
except (json.JSONDecodeError, ValueError):
pass

Expand All @@ -201,16 +208,7 @@ def _parse_response(
question = item.get("question", "").strip()
answer = item.get("answer", "").strip()
if question and answer:
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=TestCaseType.STANDARD,
source_document=source_document,
chunk_index=chunk_index,
)
)
_add_test_case(question, answer)
if test_cases:
return test_cases
except (json.JSONDecodeError, ValueError):
Expand All @@ -232,16 +230,7 @@ def _parse_response(
question = item.get("question", "").strip()
answer = item.get("answer", "").strip()
if question and answer:
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=TestCaseType.STANDARD,
source_document=source_document,
chunk_index=chunk_index,
)
)
_add_test_case(question, answer)
except (json.JSONDecodeError, ValueError):
continue

Expand All @@ -261,16 +250,7 @@ def _parse_response(
question = question.strip()
answer = answer.strip()
if question and answer:
test_cases.append(
TestCase(
question=question,
ground_truth=answer,
context=context,
test_type=TestCaseType.STANDARD,
source_document=source_document,
chunk_index=chunk_index,
)
)
_add_test_case(question, answer)

if test_cases:
return test_cases
Expand All @@ -282,20 +262,11 @@ def _parse_response(
questions = question_pattern.findall(text)
answers = answer_pattern.findall(text)

for q, a in zip(questions, answers):
for q, a in zip(questions, answers, strict=False):
q = q.strip()
a = a.strip()
if q and a:
test_cases.append(
TestCase(
question=q,
ground_truth=a,
context=context,
test_type=TestCaseType.STANDARD,
source_document=source_document,
chunk_index=chunk_index,
)
)
_add_test_case(q, a)

if test_cases:
return test_cases
Expand Down
Loading