include details of error in message

Author: Dylan Huang

eval_protocol/pytest/utils.py

@@ -318,7 +318,7 @@ async def execute_row_with_backoff(task: asyncio.Task, row: EvaluationRow) -> Ev
                         return row
                 else:
                     # Non-retryable exception - fail immediately
-                    row.rollout_status = Status.rollout_error(str(e))
+                    row.rollout_status = Status.rollout_error(repr(e))
                     return row
 
         async def execute_row_with_backoff_and_log(task: asyncio.Task, row: EvaluationRow) -> EvaluationRow:

tests/pytest/test_pytest_propagate_error.py

@@ -61,3 +61,8 @@ def eval_fn(row: EvaluationRow) -> EvaluationRow:
     # assert that the status of eval_metadata.status is "error"
     assert len(rollouts) == 5
     assert all(row.eval_metadata.status.is_error() for row in rollouts.values())
+
+    # make sure the error message includes details of the error
+    assert all("HTTPStatusError" in row.rollout_status.message for row in rollouts.values())
+    assert all("405 Method Not Allowed" in row.rollout_status.message for row in rollouts.values())
+    assert all("https://docs.fireworks.ai/mcp-non-existent" in row.rollout_status.message for row in rollouts.values())