Add eval-protocol user-agent to fireworks api requests

eval_protocol/auth.py

@@ -4,8 +4,6 @@
 from pathlib import Path
 from typing import Dict, Optional  # Added Dict
 
-import requests
-
 logger = logging.getLogger(__name__)
 
 # Default locations (used for tests and as fallback). Actual resolution is dynamic via _get_auth_ini_file().
@@ -242,9 +240,11 @@ def verify_api_key_and_get_account_id(
         if not resolved_key:
             return None
         resolved_base = api_base or get_fireworks_api_base()
-        url = f"{resolved_base.rstrip('/')}/verifyApiKey"
-        headers = {"Authorization": f"Bearer {resolved_key}"}
-        resp = requests.get(url, headers=headers, timeout=10)
+
+        from .fireworks_api_client import FireworksAPIClient
+        client = FireworksAPIClient(api_key=resolved_key, api_base=resolved_base)
+        resp = client.get("verifyApiKey", timeout=10)
+
         if resp.status_code != 200:
             logger.debug("verifyApiKey returned status %s", resp.status_code)
             return None

eval_protocol/common_utils.py

@@ -5,6 +5,22 @@
 import requests
 
 
+def get_user_agent() -> str:
+    """
+    Returns the user-agent string for eval-protocol CLI requests.
+
+    Format: eval-protocol-cli/{version}
+
+    Returns:
+        User-agent string identifying the eval-protocol CLI and version.
+    """
+    try:
+        from . import __version__
+        return f"eval-protocol-cli/{__version__}"
+    except Exception:
+        return "eval-protocol-cli/unknown"
+
+
 def load_jsonl(file_path: str) -> List[Dict[str, Any]]:
     """
     Reads a JSONL file where each line is a valid JSON object and returns a list of these objects.

eval_protocol/evaluation.py

@@ -20,6 +20,7 @@
     get_fireworks_api_key,
     verify_api_key_and_get_account_id,
 )
+from eval_protocol.fireworks_api_client import FireworksAPIClient
 from eval_protocol.typed_interface import EvaluationMode
 
 from eval_protocol.get_pep440_version import get_pep440_version
@@ -401,19 +402,15 @@ def preview(self, sample_file, max_samples=5):
         if "dev.api.fireworks.ai" in api_base and account_id == "fireworks":
             account_id = "pyroworks-dev"
 
-        url = f"{api_base}/v1/accounts/{account_id}/evaluators:previewEvaluator"
-        headers = {
-            "Authorization": f"Bearer {auth_token}",
-            "Content-Type": "application/json",
-        }
-        logger.info(f"Previewing evaluator using API endpoint: {url} with account: {account_id}")
-        logger.debug(f"Preview API Request URL: {url}")
-        logger.debug(f"Preview API Request Headers: {json.dumps(headers, indent=2)}")
+        client = FireworksAPIClient(api_key=auth_token, api_base=api_base)
+        path = f"v1/accounts/{account_id}/evaluators:previewEvaluator"
+
+        logger.info(f"Previewing evaluator using API endpoint: {api_base}/{path} with account: {account_id}")
         logger.debug(f"Preview API Request Payload: {json.dumps(payload, indent=2)}")
 
         global used_preview_api
         try:
-            response = requests.post(url, json=payload, headers=headers)
+            response = client.post(path, json=payload)
             response.raise_for_status()
             result = response.json()
             used_preview_api = True
@@ -744,11 +741,8 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
         if "dev.api.fireworks.ai" in self.api_base and account_id == "fireworks":
             account_id = "pyroworks-dev"
 
-        base_url = f"{self.api_base}/v1/{parent}/evaluatorsV2"
-        headers = {
-            "Authorization": f"Bearer {auth_token}",
-            "Content-Type": "application/json",
-        }
+        client = FireworksAPIClient(api_key=auth_token, api_base=self.api_base)
+        path = f"v1/{parent}/evaluatorsV2"
 
         self._ensure_requirements_present(os.getcwd())
 
@@ -810,7 +804,7 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
                 upload_payload = {"name": evaluator_name, "filename_to_size": {tar_filename: tar_size}}
 
                 logger.info(f"Requesting upload endpoint for {tar_filename}")
-                upload_response = requests.post(upload_endpoint_url, json=upload_payload, headers=headers)
+                upload_response = client.post(upload_endpoint_url, json=upload_payload)
                 upload_response.raise_for_status()
 
                 # Check for signed URLs
@@ -892,7 +886,7 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
                 # Step 3: Validate upload
                 validate_url = f"{self.api_base}/v1/{evaluator_name}:validateUpload"
                 validate_payload = {"name": evaluator_name}
-                validate_response = requests.post(validate_url, json=validate_payload, headers=headers)
+                validate_response = client.post(validate_url, json=validate_payload)
                 validate_response.raise_for_status()
 
                 validate_data = validate_response.json()

eval_protocol/fireworks_api_client.py

@@ -0,0 +1,127 @@
+"""Centralized client for making requests to Fireworks API with consistent headers."""
+
+import os
+from typing import Any, Dict, Optional
+
+import requests
+
+from .common_utils import get_user_agent
+
+
+class FireworksAPIClient:
+    """Client for making authenticated requests to Fireworks API with proper headers.
+
+    This client automatically includes:
+    - Authorization header (Bearer token)
+    - User-Agent header for tracking eval-protocol CLI usage
+    """
+
+    def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None):
+        """Initialize the Fireworks API client.
+
+        Args:
+            api_key: Fireworks API key. If None, will be read from environment.
+            api_base: API base URL. If None, defaults to https://api.fireworks.ai
+        """
+        self.api_key = api_key
+        self.api_base = api_base or os.environ.get("FIREWORKS_API_BASE", "https://api.fireworks.ai")
+        self._session = requests.Session()
+
+    def _get_headers(self, content_type: Optional[str] = "application/json", 
+                     additional_headers: Optional[Dict[str, str]] = None) -> Dict[str, str]:
+        """Build headers for API requests.
+
+        Args:
+            content_type: Content-Type header value. If None, Content-Type won't be set.
+            additional_headers: Additional headers to merge in.
+
+        Returns:
+            Dictionary of headers including authorization and user-agent.
+        """
+        headers = {
+            "User-Agent": get_user_agent(),
+        }
+
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+
+        if content_type:
+            headers["Content-Type"] = content_type
+
+        if additional_headers:
+            headers.update(additional_headers)
+
+        return headers
+
+    def get(self, path: str, params: Optional[Dict[str, Any]] = None, 
+            timeout: int = 30, **kwargs) -> requests.Response:
+        """Make a GET request to the Fireworks API.
+
+        Args:
+            path: API path (relative to api_base)
+            params: Query parameters
+            timeout: Request timeout in seconds
+            **kwargs: Additional arguments passed to requests.get
+
+        Returns:
+            Response object
+        """
+        url = f"{self.api_base.rstrip('/')}/{path.lstrip('/')}"
+        headers = self._get_headers(content_type=None)
+        if "headers" in kwargs:
+            headers.update(kwargs.pop("headers"))
+        return self._session.get(url, params=params, headers=headers, timeout=timeout, **kwargs)
+
+    def post(self, path: str, json: Optional[Dict[str, Any]] = None,
+             data: Optional[Any] = None, files: Optional[Dict[str, Any]] = None,
+             timeout: int = 60, **kwargs) -> requests.Response:
+        """Make a POST request to the Fireworks API.
+
+        Args:
+            path: API path (relative to api_base)
+            json: JSON payload
+            data: Form data payload
+            files: Files to upload
+            timeout: Request timeout in seconds
+            **kwargs: Additional arguments passed to requests.post
+
+        Returns:
+            Response object
+        """
+        url = f"{self.api_base.rstrip('/')}/{path.lstrip('/')}"
+
+        # For file uploads, don't set Content-Type (let requests handle multipart/form-data)
+        content_type = None if files else "application/json"
+        headers = self._get_headers(content_type=content_type)
+
+        if "headers" in kwargs:
+            headers.update(kwargs.pop("headers"))
+
+        return self._session.post(url, json=json, data=data, files=files, 
+                                 headers=headers, timeout=timeout, **kwargs)
+
+    def put(self, path: str, json: Optional[Dict[str, Any]] = None,
+            timeout: int = 60, **kwargs) -> requests.Response:
+        """Make a PUT request to the Fireworks API."""
+        url = f"{self.api_base.rstrip('/')}/{path.lstrip('/')}"
+        headers = self._get_headers()
+        if "headers" in kwargs:
+            headers.update(kwargs.pop("headers"))
+        return self._session.put(url, json=json, headers=headers, timeout=timeout, **kwargs)
+
+    def patch(self, path: str, json: Optional[Dict[str, Any]] = None,
+              timeout: int = 60, **kwargs) -> requests.Response:
+        """Make a PATCH request to the Fireworks API."""
+        url = f"{self.api_base.rstrip('/')}/{path.lstrip('/')}"
+        headers = self._get_headers()
+        if "headers" in kwargs:
+            headers.update(kwargs.pop("headers"))
+        return self._session.patch(url, json=json, headers=headers, timeout=timeout, **kwargs)
+
+    def delete(self, path: str, timeout: int = 30, **kwargs) -> requests.Response:
+        """Make a DELETE request to the Fireworks API."""
+        url = f"{self.api_base.rstrip('/')}/{path.lstrip('/')}"
+        headers = self._get_headers(content_type=None)
+        if "headers" in kwargs:
+            headers.update(kwargs.pop("headers"))
+        return self._session.delete(url, headers=headers, timeout=timeout, **kwargs)

eval_protocol/fireworks_rft.py

@@ -11,6 +11,7 @@
 import requests
 
 from .auth import get_fireworks_account_id, get_fireworks_api_base, get_fireworks_api_key
+from .fireworks_api_client import FireworksAPIClient
 
 
 def _map_api_host_to_app_host(api_base: str) -> str:
@@ -157,13 +158,14 @@ def create_dataset_from_jsonl(
     display_name: Optional[str],
     jsonl_path: str,
 ) -> Tuple[str, Dict[str, Any]]:
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    client = FireworksAPIClient(api_key=api_key, api_base=api_base)
+
     # Count examples quickly
     example_count = 0
     with open(jsonl_path, "r", encoding="utf-8") as f:
         for _ in f:
             example_count += 1
-    dataset_url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/datasets"
+
     payload = {
         "dataset": {
             "displayName": display_name or dataset_id,
@@ -173,16 +175,15 @@ def create_dataset_from_jsonl(
         },
         "datasetId": dataset_id,
     }
-    resp = requests.post(dataset_url, json=payload, headers=headers, timeout=60)
+    resp = client.post(f"v1/accounts/{account_id}/datasets", json=payload, timeout=60)
     if resp.status_code not in (200, 201):
         raise RuntimeError(f"Dataset creation failed: {resp.status_code} {resp.text}")
     ds = resp.json()
 
-    upload_url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/datasets/{dataset_id}:upload"
     with open(jsonl_path, "rb") as f:
         files = {"file": f}
-        up_headers = {"Authorization": f"Bearer {api_key}"}
-        up_resp = requests.post(upload_url, files=files, headers=up_headers, timeout=600)
+        up_resp = client.post(f"v1/accounts/{account_id}/datasets/{dataset_id}:upload", 
+                             files=files, timeout=600)
     if up_resp.status_code not in (200, 201):
         raise RuntimeError(f"Dataset upload failed: {up_resp.status_code} {up_resp.text}")
     return dataset_id, ds
@@ -194,9 +195,10 @@ def create_reinforcement_fine_tuning_job(
     api_base: str,
     body: Dict[str, Any],
 ) -> Dict[str, Any]:
-    url = f"{api_base.rstrip('/')}/v1/accounts/{account_id}/reinforcementFineTuningJobs"
-    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json", "Accept": "application/json"}
-    resp = requests.post(url, json=body, headers=headers, timeout=60)
+    client = FireworksAPIClient(api_key=api_key, api_base=api_base)
+    resp = client.post(f"v1/accounts/{account_id}/reinforcementFineTuningJobs", 
+                      json=body, timeout=60,
+                      headers={"Accept": "application/json"})
     if resp.status_code not in (200, 201):
         raise RuntimeError(f"RFT job creation failed: {resp.status_code} {resp.text}")
     return resp.json()

eval_protocol/generation/clients.py

@@ -13,6 +13,8 @@
 from omegaconf import DictConfig
 from pydantic import BaseModel  # Added for new models
 
+from ..common_utils import get_user_agent
+
 logger = logging.getLogger(__name__)
 
 
@@ -101,6 +103,7 @@ async def generate(
             "Authorization": f"Bearer {self.api_key}",
             "Content-Type": "application/json",
             "Accept": "application/json",
+            "User-Agent": get_user_agent(),
         }
 
         debug_payload_log = json.loads(json.dumps(payload))