addressedcomments

Author: Shrey Modi

.github/workflows/rollout.yml

@@ -5,9 +5,9 @@ run-name: rollout:${{ fromJSON(inputs.metadata).rollout_id }}
 on:
   workflow_dispatch:
     inputs:
-      model:
-        description: 'Model to use'
-        required: true
+      completion_params:
+        description: 'JSON completion params (optional, includes model_kwargs)'
+        required: false
         type: string
       metadata:
         description: 'JSON serialized metadata object'
@@ -17,10 +17,7 @@ on:
         description: 'Base URL for the model API'
         required: true
         type: string
-      completion_params:
-        description: 'JSON completion params (optional, includes model_kwargs)'
-        required: false
-        type: string
+
 
 jobs:
   rollout:
@@ -45,7 +42,6 @@ jobs:
           FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
         run: |
           python tests/github_actions/rollout_worker.py \
-            --model "${{ inputs.model }}" \
+            --completion-params '${{ inputs.completion_params }}' \
             --metadata '${{ inputs.metadata }}' \
-            --model-base-url "${{ inputs.model_base_url }}" \
-            ${{ inputs.completion_params && format('--completion-params ''{0}''', inputs.completion_params) || '' }}
+            --model-base-url "${{ inputs.model_base_url }}"

eval_protocol/pytest/github_action_rollout_processor.py

@@ -94,10 +94,9 @@ def _dispatch_workflow():
                 payload = {
                     "ref": self.ref,
                     "inputs": {
-                        "model": model,
+                        "completion_params": json.dumps(init_request.completion_params),
                         "metadata": init_request.metadata.model_dump_json(),
                         "model_base_url": init_request.model_base_url,
-                        "completion_params": json.dumps(init_request.completion_params),
                     },
                 }
                 r = requests.post(url, json=payload, headers=self._headers(), timeout=30)

tests/github_actions/rollout_worker.py

@@ -17,20 +17,23 @@ def main():
     parser = argparse.ArgumentParser(description="GitHub Actions rollout worker")
 
     # Required arguments from workflow inputs
-    parser.add_argument("--model", required=True, help="Model to use")
-    parser.add_argument("--completion-params", required=False, help="JSON completion params (optional)")
+    parser.add_argument("--completion-params", required=True, help="JSON completion params (includes model)")
     parser.add_argument("--metadata", required=True, help="JSON serialized metadata object")
     parser.add_argument("--model-base-url", required=True, help="Base URL for the model API")
 
     args = parser.parse_args()
 
-    # Parse the metadata
-    completion_params = {}
-    if args.completion_params:
-        try:
-            completion_params = json.loads(args.completion_params)
-        except Exception as e:
-            print(f"⚠️  Failed to parse completion_params: {e}")
+    # Parse completion_params
+    try:
+        completion_params = json.loads(args.completion_params)
+    except Exception as e:
+        print(f"❌ Failed to parse completion_params: {e}")
+        exit(1)
+
+    model = completion_params.get("model")
+    if not model:
+        print("Error: model is required in completion_params")
+        exit(1)
 
     try:
         metadata = json.loads(args.metadata)
@@ -42,7 +45,7 @@ def main():
     row_id = metadata["row_id"]
 
     print(f"🚀 Starting rollout {rollout_id}")
-    print(f"   Model: {args.model}")
+    print(f"   Model: {model}")
     print(f"   Row ID: {row_id}")
 
     dataset = [  # In this example, worker has access to the dataset and we use index to associate rows.
@@ -57,16 +60,8 @@ def main():
     print(f"   Messages: {len(messages)} messages")
 
     try:
-        completion_kwargs = {"model": args.model, "messages": messages}
-        # Parse and apply completion_params if provided
-        if args.completion_params:
-            try:
-                cp = json.loads(args.completion_params)
-                if cp.get("model_kwargs"):
-                    completion_kwargs.update(cp["model_kwargs"])
-                    print(f"   Applied model_kwargs: {cp.get('model_kwargs')}")
-            except Exception as e:
-                print(f"⚠️  Failed to parse completion_params: {e}")
+        # Build completion kwargs from completion_params
+        completion_kwargs = {"messages": messages, **completion_params}
 
         client = OpenAI(base_url=args.model_base_url, api_key=os.environ.get("FIREWORKS_API_KEY"))
 

tests/github_actions/test_github_actions_rollout.py

@@ -55,8 +55,7 @@ def rows() -> List[EvaluationRow]:
 
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Only run this test locally (skipped in CI)")
 @pytest.mark.parametrize(
-    "completion_params",
-    [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b", "model_kwargs": {"temperature": 0.5}}],
+    "completion_params", [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b", "temperature": 0.5}]
 )
 @evaluation_test(
     data_loaders=DynamicDataLoader(

tests/remote_server/remote_server.py

@@ -40,16 +40,8 @@ def _worker():
             if not model:
                 raise ValueError("model is required in completion_params")
 
-            completion_kwargs = {
-                "model": model,
-                "messages": req.messages,
-            }
-
-            # Apply model_kwargs if present
-            if req.completion_params.get("model_kwargs"):
-                model_kwargs = req.completion_params["model_kwargs"]
-                if isinstance(model_kwargs, dict):
-                    completion_kwargs.update(model_kwargs)
+            # Spread all completion_params (model, temperature, max_tokens, etc.)
+            completion_kwargs = {"messages": req.messages, **req.completion_params}
 
             if req.tools:
                 completion_kwargs["tools"] = req.tools

tests/remote_server/remote_server_multi_turn.py

@@ -38,11 +38,6 @@ def _worker():
 
             client = OpenAI(base_url=req.model_base_url, api_key=os.environ.get("FIREWORKS_API_KEY"))
 
-            # Apply model_kwargs if present
-            if req.completion_params.get("model_kwargs"):
-                model_kwargs = req.completion_params["model_kwargs"]
-                if isinstance(model_kwargs, dict):
-                    completion_kwargs.update(model_kwargs)
             # Build up conversation over 6 turns (3 user messages + 3 assistant responses)
             # Convert Message objects to dicts for OpenAI API
             conversation_history = [{"role": m.role, "content": m.content} for m in req.messages]
@@ -55,9 +50,8 @@ def _worker():
             # First completion (turns 1-2: initial user message + assistant response)
             logger.info(f"Turn 1-2: Sending initial completion request to model {model}")
             completion = client.chat.completions.create(
-                model=model,
-                messages=conversation_history,  # type: ignore,
-                **completion_kwargs,
+                messages=conversation_history,  # type: ignore
+                **req.completion_params,
             )
             assistant_message = completion.choices[0].message
             assistant_content = assistant_message.content or ""
@@ -68,8 +62,8 @@ def _worker():
             conversation_history.append({"role": "user", "content": follow_up_questions[0]})
             logger.info(f"Turn 3: User asks: {follow_up_questions[0]}")
             completion = client.chat.completions.create(
-                model=model,
                 messages=conversation_history,  # type: ignore
+                **req.completion_params,
             )
             assistant_message = completion.choices[0].message
             assistant_content = assistant_message.content or ""
@@ -80,8 +74,8 @@ def _worker():
             conversation_history.append({"role": "user", "content": follow_up_questions[1]})
             logger.info(f"Turn 5: User asks: {follow_up_questions[1]}")
             completion = client.chat.completions.create(
-                model=model,
                 messages=conversation_history,  # type: ignore
+                **req.completion_params,
             )
             assistant_message = completion.choices[0].message
             assistant_content = assistant_message.content or ""

tests/remote_server/test_remote_fireworks.py

@@ -60,7 +60,7 @@ def rows() -> List[EvaluationRow]:
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Only run this test locally (skipped in CI)")
 @pytest.mark.parametrize(
     "completion_params",
-    [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b", "model_kwargs": {"temperature": 0.5}}],
+    [{"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b", "temperature": 0.5}],
 )
 @evaluation_test(
     data_loaders=DynamicDataLoader(
@@ -85,8 +85,6 @@ async def test_remote_rollout_and_fetch_fireworks(row: EvaluationRow) -> Evaluat
     assert row.execution_metadata.rollout_id in ROLLOUT_IDS, (
         f"Row rollout_id {row.execution_metadata.rollout_id} should be in tracked rollout_ids: {ROLLOUT_IDS}"
     )
-    assert row.input_metadata.completion_params["model_kwargs"] == {"temperature": 0.5}, (
-        "Row should have correct model_kwargs"
-    )
+    assert row.input_metadata.completion_params["temperature"] == 0.5, "Row should have temperature at top level"
 
     return row

typescript/index.ts

@@ -89,19 +89,17 @@ export function initRequestToCompletionParams(
     throw new Error("messages is required");
   }
 
-  const baseParams: ChatCompletionCreateParamsNonStreaming = {
+  // Spread completion_params directly (model, temperature, max_tokens, etc.)
+  const { model: _, ...otherParams } = initRequest.completion_params || {};
+
+  const completionParams: ChatCompletionCreateParamsNonStreaming = {
     model: model,
     messages: initRequest.messages,
     ...(toolsToOpenAI && { tools: toolsToOpenAI }),
+    ...otherParams  // Spreads temperature, max_tokens, etc.
   };
 
-  // Apply model_kwargs if present
-  const model_kwargs = initRequest.completion_params?.['model_kwargs'];
-  if (model_kwargs && typeof model_kwargs === 'object') {
-    Object.assign(baseParams, model_kwargs);
-  }
-
-  return baseParams;
+  return completionParams;
 }
 
 export function createLangfuseConfigTags(initRequest: InitRequest): string[] {