updated model

Author: xzrderek

eval_protocol/models.py

@@ -231,7 +231,9 @@ class EvalMetadata(BaseModel):
     )
     num_runs: int = Field(..., description="Number of times the evaluation was repeated")
     aggregation_method: str = Field(..., description="Method used to aggregate scores across runs")
-    threshold: Optional[EvaluationThreshold] = Field(None, description="Threshold configuration for test success")
+    passed_threshold: Optional[EvaluationThreshold] = Field(
+        None, description="Threshold configuration for test success"
+    )
     passed: Optional[bool] = Field(None, description="Whether the evaluation passed based on the threshold")
 
 

eval_protocol/pytest/evaluation_test.py

@@ -53,7 +53,7 @@ def evaluation_test(  # noqa: C901
     rollout_processor: RolloutProcessor = default_no_op_rollout_processor,
     evaluation_test_kwargs: Optional[List[EvaluationInputParam]] = None,
     aggregation_method: AggregationMethod = "mean",
-    threshold: Optional[EvaluationThreshold] = None,
+    passed_threshold: Optional[Union[EvaluationThreshold, float]] = None,
     num_runs: int = 1,
     max_dataset_rows: Optional[int] = None,
     mcp_config_path: Optional[str] = None,
@@ -113,7 +113,7 @@ def evaluation_test(  # noqa: C901
         rollout_processor: Function used to perform the rollout.
         evaluation_test_kwargs: Kwargs for the evaluation function.
         aggregation_method: How to aggregate scores across rows.
-        threshold: Threshold configuration for test success.
+        passed_threshold: Threshold configuration for test success.
             Success rate must be above success, and if set, standard deviation must be below standard_deviation.
         num_runs: Number of times to repeat the rollout and evaluations.
         max_dataset_rows: Limit dataset to the first N rows.
@@ -129,11 +129,11 @@ def evaluation_test(  # noqa: C901
     def decorator(
         test_func: TestFunction,
     ):
-        if threshold is not None:
-            if isinstance(threshold, dict):
-                evaluation_threshold = EvaluationThreshold(**threshold)
-            elif isinstance(threshold, float):
-                evaluation_threshold = EvaluationThreshold(success=threshold)
+        if passed_threshold is not None:
+            if isinstance(passed_threshold, float):
+                threshold = EvaluationThreshold(success=passed_threshold)
+            else:
+                threshold = EvaluationThreshold(**passed_threshold)
 
         sig = inspect.signature(test_func)
 
@@ -361,7 +361,7 @@ def _log_eval_error(
                         status="running",
                         num_runs=num_runs,
                         aggregation_method=aggregation_method,
-                        threshold=evaluation_threshold,
+                        passed_threshold=threshold,
                         passed=None,
                     )
 
@@ -459,6 +459,7 @@ def _log_eval_error(
                         sum([r.evaluation_result.score for r in result if r.evaluation_result]) / len(result)
                         for result in all_results
                     ]
+                    print(f"SCORES: {scores}")
                     agg_score = aggregate(scores, aggregation_method)
                     score_std = statistics.stdev(scores) if len(scores) > 1 else 0.0
 
@@ -495,13 +496,13 @@ def _log_eval_error(
                     # Determine if the evaluation passed based on threshold
                     passed = None
 
-                    if evaluation_threshold is not None:
+                    if threshold is not None:
                         success_passed, std_passed = True, True
 
-                        success_passed = agg_score >= evaluation_threshold.success
+                        success_passed = agg_score >= threshold.success
 
-                        if evaluation_threshold.standard_deviation is not None:
-                            std_passed = score_std <= evaluation_threshold.standard_deviation
+                        if threshold.standard_deviation is not None:
+                            std_passed = score_std <= threshold.standard_deviation
 
                         passed = success_passed and std_passed
 
@@ -636,14 +637,14 @@ def _extract_effort_tag(params: dict) -> str | None:
                         pass
 
                     # Check threshold after logging
-                    if evaluation_threshold is not None and not passed:
+                    if threshold is not None and not passed:
                         assert (
-                            agg_score >= evaluation_threshold.success
-                        ), f"Aggregated score {agg_score:.3f} below threshold {evaluation_threshold.success}"
-                        if evaluation_threshold.standard_deviation is not None:
+                            agg_score >= threshold.success
+                        ), f"Aggregated score {agg_score:.3f} below threshold {threshold.success}"
+                        if threshold.standard_deviation is not None:
                             assert (
-                                score_std <= evaluation_threshold.standard_deviation
-                            ), f"Standard deviation {score_std:.3f} above threshold {evaluation_threshold.standard_deviation}"
+                                score_std <= threshold.standard_deviation
+                            ), f"Standard deviation {score_std:.3f} above threshold {threshold.standard_deviation}"
 
                 except AssertionError:
                     _log_eval_error("finished", data if "data" in locals() else None, passed=False)

tests/pytest/test_apps_coding.py

@@ -28,7 +28,7 @@ def apps_dataset_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluatio
     dataset_adapter=apps_dataset_to_evaluation_row,
     model=["fireworks_ai/accounts/fireworks/models/kimi-k2-instruct"],
     rollout_input_params=[{"temperature": 0.0, "max_tokens": 4096}],
-    threshold=0.33,
+    passed_threshold=0.33,
     rollout_processor=default_single_turn_rollout_processor,
     num_runs=1,
     mode="pointwise",

tests/pytest/test_basic_coding.py

@@ -30,7 +30,7 @@ def coding_dataset_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluat
     dataset_adapter=coding_dataset_to_evaluation_row,
     model=["fireworks_ai/accounts/fireworks/models/kimi-k2-instruct"],
     rollout_input_params=[{"temperature": 0.0, "max_tokens": 4096}],
-    threshold=0.8,
+    passed_threshold=0.8,
     rollout_processor=default_single_turn_rollout_processor,
     num_runs=1,
     mode="pointwise",

tests/pytest/test_frozen_lake.py

@@ -41,7 +41,7 @@ def frozen_lake_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluation
     model=["fireworks_ai/accounts/fireworks/models/kimi-k2-instruct"],
     rollout_input_params=[{"temperature": 0.0, "max_tokens": 4096}],
     rollout_processor=default_mcp_gym_rollout_processor,
-    threshold=0.66,
+    passed_threshold=0.66,
     num_runs=1,
     max_concurrent_rollouts=3,
     mode="pointwise",

tests/pytest/test_hallucination.py

@@ -35,7 +35,7 @@ def hallucination_dataset_adapter(data: List[Dict[str, Any]]) -> List[Evaluation
     model=["fireworks_ai/accounts/fireworks/models/kimi-k2-instruct"],
     rollout_input_params=[{"temperature": 0.0, "max_tokens": 512}],
     rollout_processor=default_single_turn_rollout_processor,
-    threshold=0.33,
+    passed_threshold=0.33,
     num_runs=1,
     mode="pointwise",
 )

tests/pytest/test_lunar_lander.py

@@ -41,7 +41,7 @@ def lunar_lander_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluatio
     model=["gpt-4.1"],
     rollout_input_params=[{"temperature": 0.0, "max_tokens": 4096}],
     rollout_processor=default_mcp_gym_rollout_processor,
-    threshold=0.0,
+    passed_threshold=0.0,
     num_runs=1,
     mode="pointwise",
     max_concurrent_rollouts=3,

tests/pytest/test_markdown_highlighting.py

@@ -30,7 +30,7 @@ def markdown_dataset_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evalu
     dataset_adapter=markdown_dataset_to_evaluation_row,
     model=["fireworks_ai/accounts/fireworks/models/kimi-k2-instruct"],
     rollout_input_params=[{"temperature": 0.0, "max_tokens": 4096}],
-    threshold=0.5,
+    passed_threshold=0.5,
     rollout_processor=default_single_turn_rollout_processor,
     num_runs=1,
     mode="pointwise",

tests/pytest/test_pytest_math_example.py

@@ -11,7 +11,7 @@
     model=["fireworks_ai/accounts/fireworks/models/kimi-k2-instruct"],
     rollout_input_params=[{"temperature": 0.0}],
     max_dataset_rows=5,
-    threshold=0.0,
+    passed_threshold=0.0,
     rollout_processor=default_single_turn_rollout_processor,
     mode="pointwise",
     evaluation_test_kwargs=[

tests/pytest/test_pytest_math_format_length.py

@@ -14,7 +14,7 @@
     model=["fireworks_ai/accounts/fireworks/models/kimi-k2-instruct"],
     rollout_input_params=[{"temperature": 0.0}],
     max_dataset_rows=5,
-    threshold=0.0,
+    passed_threshold=0.0,
     rollout_processor=default_single_turn_rollout_processor,
     mode="pointwise",
     evaluation_test_kwargs=[