Revert "Revert "add default try catch to evaluator function"" (#302)

* Revert "Revert "add try catch (#297)" (#301)"

This reverts commit 56e00c2.

* set the eval metadata status as well

* add

* lint

* add

* add

* avoid override

* fix ut

* add

Author: mayinghan

eval_protocol/pytest/evaluation_test.py

@@ -19,6 +19,7 @@
     EvaluationRow,
     EvaluationThreshold,
     EvaluationThresholdDict,
+    EvaluateResult,
     Status,
 )
 from eval_protocol.pytest.dual_mode_wrapper import create_dual_mode_wrapper
@@ -370,7 +371,7 @@ def _log_eval_error(status: Status, rows: list[EvaluationRow] | None, passed: bo
                             row.input_metadata.session_data = {}
                         row.input_metadata.session_data["mode"] = mode
                         # Initialize eval_metadata for each row
-                        row.eval_metadata = eval_metadata
+                        row.eval_metadata = eval_metadata.model_copy(deep=True)
                         row.execution_metadata.experiment_id = experiment_id
                         row.execution_metadata.invocation_id = invocation_id
 
@@ -429,11 +430,23 @@ async def _execute_pointwise_eval_with_semaphore(
                                     experiment_id=experiment_id,
                                     run_id=run_id,
                                 ):
-                                    result = await execute_pytest(
-                                        test_func,
-                                        processed_row=row,
-                                        evaluation_test_kwargs=evaluation_test_kwargs,
-                                    )
+                                    try:
+                                        result = await execute_pytest(
+                                            test_func,
+                                            processed_row=row,
+                                            evaluation_test_kwargs=evaluation_test_kwargs,
+                                        )
+                                    except Exception as e:
+                                        result = row
+                                        result.evaluation_result = EvaluateResult(
+                                            score=0.0,
+                                            is_score_valid=False,
+                                            reason=f"Error during evaluation: {type(e).__name__}: {e}",
+                                        )
+                                        if result.eval_metadata is not None:
+                                            result.eval_metadata.status = Status.error(
+                                                f"Error during evaluation: {type(e).__name__}: {e}",
+                                            )
                                 if not isinstance(result, EvaluationRow):
                                     raise ValueError(
                                         f"Test function {test_func.__name__} did not return an EvaluationRow instance. You must return an EvaluationRow instance from your test function decorated with @evaluation_test."
@@ -455,11 +468,24 @@ async def _execute_groupwise_eval_with_semaphore(
                                     run_id=run_id,
                                     rollout_ids=group_rollout_ids or None,
                                 ):
-                                    results = await execute_pytest(
-                                        test_func,
-                                        processed_dataset=rows,
-                                        evaluation_test_kwargs=evaluation_test_kwargs,
-                                    )
+                                    try:
+                                        results = await execute_pytest(
+                                            test_func,
+                                            processed_dataset=rows,
+                                            evaluation_test_kwargs=evaluation_test_kwargs,
+                                        )
+                                    except Exception as e:
+                                        results = rows
+                                        for row in results:
+                                            row.evaluation_result = EvaluateResult(
+                                                score=0.0,
+                                                is_score_valid=False,
+                                                reason=f"Error during evaluation: {type(e).__name__}: {e}",
+                                            )
+                                            if row.eval_metadata is not None:
+                                                row.eval_metadata.status = Status.error(
+                                                    f"Error during evaluation: {type(e).__name__}: {e}",
+                                                )
                                 if not isinstance(results, list):
                                     raise ValueError(
                                         f"Test function {test_func.__name__} did not return a list of EvaluationRow instances. You must return a list of EvaluationRow instances from your test function decorated with @evaluation_test."
@@ -576,7 +602,10 @@ async def _collect_result(config, lst):
                                     r.eval_metadata.status = Status.error(
                                         r.rollout_status.message, r.rollout_status.details
                                     )
-                                else:
+                                elif not (
+                                    r.eval_metadata.status and r.eval_metadata.status.code != Status.Code.RUNNING
+                                ):
+                                    # if the eval_metadata status code has not been set to something else, consider it as finished
                                     r.eval_metadata.status = Status.eval_finished()
                             # Optional debug print for assistant/tool sequence
                             if os.getenv("EP_DEBUG_SERIALIZATION", "0").strip() == "1":

eval_protocol/pytest/evaluation_test_postprocess.py

@@ -91,7 +91,8 @@ def postprocess(
                     result.evaluation_result.standard_error = standard_error
                 if result.evaluation_result.is_score_valid is False:
                     if result.eval_metadata is not None:
-                        result.eval_metadata.status = Status.score_invalid()
+                        if not result.eval_metadata.status or not result.eval_metadata.status.is_error():
+                            result.eval_metadata.status = Status.score_invalid()
             result.execution_metadata.experiment_duration_seconds = experiment_duration_seconds
             active_logger.log(result)
 

eval_protocol/pytest/exception_config.py

@@ -33,9 +33,9 @@
     litellm.exceptions.InternalServerError,
     litellm.exceptions.Timeout,
     litellm.exceptions.NotFoundError,
-    litellm.exceptions.BadRequestError,  # remove this once we have a long term solution
+    litellm.exceptions.BadRequestError,
     litellm.exceptions.ServiceUnavailableError,
-    litellm.exceptions.APIError
+    litellm.exceptions.APIError,
 }