format

Author: mayinghan

eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -73,11 +73,11 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
 
             _litellm = importlib.import_module("litellm")
             acompletion = getattr(_litellm, "acompletion")
-            logger.debug(f'********** request_params: {request_params} **********')
+            logger.debug(f"********** request_params: {request_params} **********")
             response = await acompletion(**request_params)
 
             assistant_content = response.choices[0].message.content or ""
-            logger.debug(f'********** assistant_content: {assistant_content} **********')
+            logger.debug(f"********** assistant_content: {assistant_content} **********")
             tool_calls = response.choices[0].message.tool_calls if response.choices[0].message.tool_calls else None
 
             converted_tool_calls = None

eval_protocol/pytest/evaluation_test.py

@@ -634,7 +634,9 @@ async def _collect_result(config, lst, max_retry):
                                 for ori_row in fresh_dataset:
                                     copied_row = ori_row.model_copy(deep=True)
                                     # overwrite the rollout_id to the index of the completion_params
-                                    copied_row.execution_metadata.rollout_id = str(ori_row.execution_metadata.rollout_id) + "_" + str(idx)
+                                    copied_row.execution_metadata.rollout_id = (
+                                        str(ori_row.execution_metadata.rollout_id) + "_" + str(idx)
+                                    )
                                     copied_row.input_metadata.completion_params = cp
                                     lst.append(copied_row)
                                 tasks.append(asyncio.create_task(_collect_result(config, lst, max_retry)))

tests/pytest/test_pytest_groupwise.py

@@ -3,6 +3,7 @@
 from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 from eval_protocol.pytest import SingleTurnRolloutProcessor, evaluation_test
 
+
 @evaluation_test(
     input_messages=[
         [
@@ -24,4 +25,4 @@ def test_pytest_groupwise(rows: List[EvaluationRow]) -> List[EvaluationRow]:
     rows[1].evaluation_result = EvaluateResult(score=0.0, reason="test")
     print(rows[0].model_dump_json())
     print(rows[1].model_dump_json())
-    return rows
+    return rows

tests/pytest/test_svgbench.py

@@ -288,7 +288,6 @@ def evaluate_with_llm_judge_groupwise(image_paths: List[str], requirements: List
 Requirements:
 {requirements_text}"""
 
-
     messages = [
         {
             "role": "user",
@@ -302,7 +301,9 @@ def evaluate_with_llm_judge_groupwise(image_paths: List[str], requirements: List
     for image_path in image_paths:
         with open(image_path, "rb") as f:
             image_data = base64.b64encode(f.read()).decode("utf-8")
-            messages[0]["content"].append({"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_data}"}})
+            messages[0]["content"].append(
+                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_data}"}}
+            )
 
     # Use GPT-4.1 for vision capabilities to match project's OpenAI model preference
     response = litellm.completion(
@@ -331,7 +332,6 @@ def evaluate_with_llm_judge_groupwise(image_paths: List[str], requirements: List
         raise ValueError("Missing required field in response")
 
 
-
 @evaluation_test(
     input_dataset=["tests/pytest/data/svgbench_dataset.jsonl"],
     dataset_adapter=svgbench_to_evaluation_row,
@@ -540,15 +540,14 @@ def test_svg_generation_evaluation_groupwise(rows: List[EvaluationRow]) -> List[
             row.evaluation_result = EvaluateResult(score=0.0, reason=f"Evaluation error: {str(e)}")
 
     judge_result = evaluate_with_llm_judge_groupwise(image_paths, requirements)
-    print(f'********** judge_result: {judge_result} **********')
+    print(f"********** judge_result: {judge_result} **********")
     if judge_result.get("best_image_index") == 0:
         rows[0].evaluation_result = EvaluateResult(score=1.0, reason=judge_result.get("reasoning", ""))
         rows[1].evaluation_result = EvaluateResult(score=0.0, reason=judge_result.get("reasoning", ""))
     else:
         rows[0].evaluation_result = EvaluateResult(score=0.0, reason=judge_result.get("reasoning", ""))
         rows[1].evaluation_result = EvaluateResult(score=1.0, reason=judge_result.get("reasoning", ""))
-    
-    
+
     # Clean up temporary PNG file (only if not saving debug files)
     if not save_debug_files:
         for png_path in image_paths: