convert rollout_input_params to completion_params

Author: Dylan Huang

eval_protocol/benchmarks/suites/aime25.py

@@ -60,13 +60,18 @@ def aime2025_dataset_adapter(rows: List[Dict[str, Any]]) -> List[EvaluationRow]:
 
 @export_benchmark("aime25")
 @evaluation_test(
-    model=["fireworks_ai/accounts/fireworks/models/gpt-oss-120b"],
     input_dataset=[
         "https://huggingface.co/datasets/opencompass/AIME2025/raw/main/aime2025-I.jsonl",
         "https://huggingface.co/datasets/opencompass/AIME2025/raw/main/aime2025-II.jsonl",
     ],
     dataset_adapter=aime2025_dataset_adapter,
-    rollout_input_params=[{"max_tokens": 131000, "extra_body": {"reasoning_effort": "low"}}],
+    completion_params=[
+        {
+            "max_tokens": 131000,
+            "extra_body": {"reasoning_effort": "low"},
+            "model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b",
+        }
+    ],
     rollout_processor=default_single_turn_rollout_processor,
     aggregation_method="mean",
     num_runs=8,

eval_protocol/benchmarks/suites/gpqa.py

@@ -60,9 +60,10 @@ def _extract_abcd_letter(text: str) -> str | None:
 
 @export_benchmark("gpqa")
 @evaluation_test(
-    model=["fireworks_ai/accounts/fireworks/models/gpt-oss-120b"],
     input_messages=_GPQA_INPUT_MESSAGES,
-    rollout_input_params=[{"extra_body": {"reasoning_effort": "low"}}],
+    completion_params=[
+        {"extra_body": {"reasoning_effort": "low"}, "model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b"}
+    ],
     rollout_processor=default_single_turn_rollout_processor,
     aggregation_method="mean",
     num_runs=8,

eval_protocol/benchmarks/suites/tau_bench_retail.py

@@ -66,8 +66,13 @@ def tau_bench_retail_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evalu
 @evaluation_test(
     input_dataset=["tests/pytest/data/retail_dataset.jsonl"],
     dataset_adapter=tau_bench_retail_to_evaluation_row,
-    model=["fireworks_ai/accounts/fireworks/models/gpt-oss-120b"],
-    rollout_input_params=[{"temperature": 0.8, "extra_body": {"reasoning_effort": "medium"}}],
+    completion_params=[
+        {
+            "temperature": 0.8,
+            "extra_body": {"reasoning_effort": "medium"},
+            "model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b",
+        }
+    ],
     rollout_processor=default_mcp_gym_rollout_processor,
     num_runs=8,
     mode="pointwise",

eval_protocol/models.py

@@ -186,6 +186,9 @@ class CompletionParams(BaseModel):
     max_tokens: Optional[int] = Field(None, description="Maximum tokens to generate")
     max_tool_calls: Optional[int] = Field(None, description="Maximum tool calls per turn")
 
+    # there might be model or provider specific parameters that you want to pass that should be preserved
+    model_config = ConfigDict(extra="allow")
+
 
 class InputMetadata(BaseModel):
     """Comprehensive metadata for input to evaluation and logging systems."""

eval_protocol/pytest/default_agent_rollout_processor.py

@@ -117,7 +117,9 @@ async def default_agent_rollout_processor(
 ) -> List[EvaluationRow]:
     dataset: Dataset = []
     for row in rows:
-        agent = Agent(model=config.model, row=row, config_path=config.mcp_config_path, logger=config.logger)
+        agent = Agent(
+            model=config.completion_params.model, row=row, config_path=config.mcp_config_path, logger=config.logger
+        )
         await agent.setup()
         await agent.call_agent()
         dataset.append(agent.evaluation_row)

eval_protocol/pytest/default_mcp_gym_rollout_processor.py

@@ -219,10 +219,10 @@ async def default_mcp_gym_rollout_processor(
         server.start()
 
         policy = ep.LiteLLMPolicy(
-            model_id=config.model,
-            temperature=config.input_params.get("temperature", 0.0),
-            max_tokens=config.input_params.get("max_tokens", 4096),
-            reasoning_effort=config.input_params.get("reasoning_effort", None),
+            model_id=config.completion_params.model,
+            temperature=config.completion_params.get("temperature", 0.0),
+            max_tokens=config.completion_params.get("max_tokens", 4096),
+            reasoning_effort=config.completion_params.get("reasoning_effort", None),
         )
 
         # Create MCP environments directly from evaluation_rows

eval_protocol/pytest/default_single_turn_rollout_process.py

@@ -33,17 +33,24 @@ async def process_row(row: EvaluationRow) -> EvaluationRow:
 
         messages_payload = [{"role": m.role, "content": m.content} for m in row.messages]
 
-        request_params = {"model": config.model, "messages": messages_payload, **config.input_params}
+        request_params = {
+            "model": config.completion_params.model,
+            "messages": messages_payload,
+            **config.completion_params,
+        }
         # Ensure caching is disabled only for this request (review feedback)
         request_params["cache"] = {"no-cache": True}
         # Single-level reasoning effort: expect `reasoning_effort` only
         effort_val = None
-        if isinstance(config.input_params, dict):
-            if "reasoning_effort" in config.input_params:
-                effort_val = str(config.input_params["reasoning_effort"])  # flat shape
-            elif isinstance(config.input_params.get("extra_body"), dict) and "reasoning_effort" in config.input_params["extra_body"]:
+        if isinstance(config.completion_params, dict):
+            if "reasoning_effort" in config.completion_params:
+                effort_val = str(config.completion_params["reasoning_effort"])  # flat shape
+            elif (
+                isinstance(config.completion_params.get("extra_body"), dict)
+                and "reasoning_effort" in config.completion_params["extra_body"]
+            ):
                 # Accept if user passed it directly inside extra_body
-                effort_val = str(config.input_params["extra_body"]["reasoning_effort"])  # already in extra_body
+                effort_val = str(config.completion_params["extra_body"]["reasoning_effort"])  # already in extra_body
 
         if effort_val:
             # Always under extra_body so LiteLLM forwards to provider-specific param set