chore: consolidate nightly hardening PRs into one reviewed change set

README.md

@@ -248,10 +248,9 @@ models:
   - name: "Local Llama"
     provider: http
     model: llama3.1:8b
+    endpoint: "http://localhost:11434/api/generate"
     temperature: 0.7
     max_tokens: 1024
-    additional_params:
-      endpoint: "http://localhost:11434/api/generate"
 ```
 
 **Setup:**
@@ -270,10 +269,12 @@ models:
   - name: "Display Name"           # Human-readable name
     provider: anthropic             # anthropic, openai, google, http
     model: model-identifier         # Model ID
+    endpoint: "http://..."          # Optional, for HTTP provider
     temperature: 0.7                # 0.0-1.0
     max_tokens: 1024                # Maximum output tokens
     additional_params:              # Provider-specific params
-      endpoint: "http://..."        # For HTTP provider
+      # endpoint under additional_params is still accepted for backwards compatibility
+      custom_option: "value"
 ```
 
 ### Judge

examples/configs/local_model.yaml

@@ -13,10 +13,9 @@ models:
   - name: "Local Llama 3.1 8B"
     provider: http
     model: llama3.1:8b
+    endpoint: "http://localhost:11434/api/generate"
     temperature: 0.7
     max_tokens: 1024
-    additional_params:
-      endpoint: "http://localhost:11434/api/generate"
 
 judge:
   provider: anthropic

promptlens/cli.py

@@ -21,6 +21,21 @@
 from promptlens.models.config import RunConfig
 from promptlens.runners.runner import Runner
 
+
+def _load_config_data(config_path: str) -> dict:
+    """Load and validate top-level config structure from YAML."""
+    with open(config_path, "r") as f:
+        config_data = yaml.safe_load(f)
+
+    if config_data is None:
+        raise ValueError("Configuration file is empty")
+    if not isinstance(config_data, dict):
+        raise ValueError(
+            f"Configuration must be a YAML object at top level, got {type(config_data).__name__}"
+        )
+
+    return config_data
+
 # Load environment variables
 load_dotenv()
 
@@ -99,8 +114,7 @@ def run(
     try:
         # Load config
         console.print(f"\n[cyan]Loading configuration from {config}...[/cyan]")
-        with open(config, "r") as f:
-            config_data = yaml.safe_load(f)
+        config_data = _load_config_data(config)
 
         # Override with CLI options
         if golden_set:

promptlens/models/config.py

@@ -1,8 +1,9 @@
 """Configuration data models."""
 
 from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 
 class ProviderConfig(BaseModel):
@@ -28,6 +29,20 @@ class ProviderConfig(BaseModel):
     endpoint: Optional[str] = None
     additional_params: Dict[str, Any] = Field(default_factory=dict)
 
+    @field_validator("endpoint")
+    @classmethod
+    def validate_endpoint(cls, value: Optional[str]) -> Optional[str]:
+        """Validate endpoint URL when provided."""
+        if value is None:
+            return value
+
+        parsed = urlparse(value)
+        if parsed.scheme not in {"http", "https"}:
+            raise ValueError("endpoint must use http or https scheme")
+        if not parsed.netloc:
+            raise ValueError("endpoint must include a host")
+        return value
+
 
 class ModelConfig(BaseModel):
     """Configuration for a model to test.
@@ -36,6 +51,8 @@ class ModelConfig(BaseModel):
         name: Display name for the model
         provider: Provider name
         model: Model identifier
+        endpoint: Optional endpoint URL (for HTTP/local providers)
+        timeout: Optional request timeout in seconds
         temperature: Sampling temperature
         max_tokens: Maximum tokens to generate
         additional_params: Provider-specific parameters
@@ -44,10 +61,26 @@ class ModelConfig(BaseModel):
     name: str
     provider: str
     model: str
+    endpoint: Optional[str] = None
+    timeout: Optional[int] = None
     temperature: float = 0.7
     max_tokens: int = 1024
     additional_params: Dict[str, Any] = Field(default_factory=dict)
 
+    @field_validator("temperature")
+    @classmethod
+    def validate_temperature(cls, value: float) -> float:
+        if not 0.0 <= value <= 2.0:
+            raise ValueError("temperature must be between 0.0 and 2.0")
+        return value
+
+    @field_validator("max_tokens")
+    @classmethod
+    def validate_max_tokens(cls, value: int) -> int:
+        if value <= 0:
+            raise ValueError("max_tokens must be greater than 0")
+        return value
+
 
 class JudgeConfig(BaseModel):
     """Configuration for the judge.
@@ -82,6 +115,34 @@ class ExecutionConfig(BaseModel):
     retry_delay_seconds: float = 1.0
     timeout_seconds: int = 60
 
+    @field_validator("parallel_requests")
+    @classmethod
+    def validate_parallel_requests(cls, value: int) -> int:
+        if value <= 0:
+            raise ValueError("parallel_requests must be greater than 0")
+        return value
+
+    @field_validator("retry_attempts")
+    @classmethod
+    def validate_retry_attempts(cls, value: int) -> int:
+        if value < 0:
+            raise ValueError("retry_attempts must be greater than or equal to 0")
+        return value
+
+    @field_validator("retry_delay_seconds")
+    @classmethod
+    def validate_retry_delay_seconds(cls, value: float) -> float:
+        if value < 0:
+            raise ValueError("retry_delay_seconds must be greater than or equal to 0")
+        return value
+
+    @field_validator("timeout_seconds")
+    @classmethod
+    def validate_timeout_seconds(cls, value: int) -> int:
+        if value <= 0:
+            raise ValueError("timeout_seconds must be greater than 0")
+        return value
+
 
 class OutputConfig(BaseModel):
     """Configuration for output settings.
@@ -96,6 +157,18 @@ class OutputConfig(BaseModel):
     formats: List[str] = Field(default_factory=lambda: ["html", "json"])
     run_name: Optional[str] = None
 
+    @field_validator("formats")
+    @classmethod
+    def validate_formats(cls, value: List[str]) -> List[str]:
+        allowed = {"html", "json", "csv", "md"}
+        normalized = [fmt.lower() for fmt in value]
+        invalid = sorted({fmt for fmt in normalized if fmt not in allowed})
+        if invalid:
+            raise ValueError(f"unsupported output format(s): {', '.join(invalid)}")
+        if not normalized:
+            raise ValueError("output formats must contain at least one format")
+        return normalized
+
 
 class RunConfig(BaseModel):
     """Complete run configuration.
@@ -114,9 +187,35 @@ class RunConfig(BaseModel):
     execution: ExecutionConfig = Field(default_factory=ExecutionConfig)
     output: OutputConfig = Field(default_factory=OutputConfig)
 
-    class Config:
-        """Pydantic config."""
-        json_schema_extra = {
+    @model_validator(mode="after")
+    def validate_models(self) -> "RunConfig":
+        if not self.models:
+            raise ValueError("models must contain at least one model configuration")
+        return self
+
+    @field_validator("models")
+    @classmethod
+    def validate_models_unique(cls, models: List[ModelConfig]) -> List[ModelConfig]:
+        """Ensure model display names are unique to avoid ambiguous reports."""
+        seen = set()
+        duplicates = set()
+
+        for model in models:
+            normalized = model.name.strip().lower()
+            if normalized in seen:
+                duplicates.add(model.name)
+            else:
+                seen.add(normalized)
+
+        if duplicates:
+            duplicate_list = ", ".join(sorted(duplicates))
+            raise ValueError(
+                f"Model names must be unique (case-insensitive). Duplicates: {duplicate_list}"
+            )
+
+        return models
+
+    model_config = ConfigDict(json_schema_extra={
             "example": {
                 "golden_set": "./examples/golden_sets/customer_support.yaml",
                 "models": [
@@ -142,4 +241,4 @@ class Config:
                     "formats": ["html", "json"],
                 },
             }
-        }
+        })

promptlens/models/test_case.py

@@ -2,7 +2,7 @@
 
 from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from promptlens.models.tools import ToolDefinition, ExpectedToolCall
 
@@ -50,17 +50,15 @@ class TestCase(BaseModel):
         description="Whether to actually execute tools (default: False, evaluation only)"
     )
 
-    class Config:
-        """Pydantic config."""
-        json_schema_extra = {
+    model_config = ConfigDict(json_schema_extra={
             "example": {
                 "id": "cs-001",
                 "query": "How do I reset my password?",
                 "expected_behavior": "Provide clear step-by-step instructions",
                 "category": "account_management",
                 "tags": ["password", "account"],
             }
-        }
+        })
 
 
 class GoldenSet(BaseModel):
@@ -80,9 +78,7 @@ class GoldenSet(BaseModel):
     test_cases: List[TestCase]
     metadata: Dict[str, Any] = Field(default_factory=dict)
 
-    class Config:
-        """Pydantic config."""
-        json_schema_extra = {
+    model_config = ConfigDict(json_schema_extra={
             "example": {
                 "name": "Customer Support Tests",
                 "description": "Test cases for customer support chatbot",
@@ -97,4 +93,4 @@ class Config:
                     }
                 ],
             }
-        }
+        })

promptlens/models/tools.py

@@ -8,7 +8,7 @@
 """
 
 from typing import Any, Dict, List, Optional
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 
 class ToolParameter(BaseModel):
@@ -24,8 +24,7 @@ class ToolParameter(BaseModel):
     properties: Optional[Dict[str, "ToolParameter"]] = Field(None, description="For object types, nested properties")
     items: Optional["ToolParameter"] = Field(None, description="For array types, the item schema")
 
-    class Config:
-        extra = "allow"  # Allow additional JSON Schema fields
+    model_config = ConfigDict(extra="allow")  # Allow additional JSON Schema fields
 
 
 class ToolDefinition(BaseModel):

promptlens/providers/factory.py

@@ -32,22 +32,39 @@ def get_provider(model_config: ModelConfig) -> BaseProvider:
     Raises:
         ValueError: If provider is not supported
     """
-    provider_name = model_config.provider.lower()
+    provider_name = model_config.provider.strip().lower()
+
+    if not provider_name:
+        available = ", ".join(sorted(PROVIDER_REGISTRY.keys()))
+        raise ValueError(
+            "Provider name cannot be empty. "
+            f"Available providers: {available}"
+        )
 
     if provider_name not in PROVIDER_REGISTRY:
-        available = ", ".join(PROVIDER_REGISTRY.keys())
+        available = ", ".join(sorted(PROVIDER_REGISTRY.keys()))
         raise ValueError(
             f"Provider '{provider_name}' not supported. "
             f"Available providers: {available}"
         )
 
+    # Copy params so we can normalize legacy keys without mutating input
+    additional_params = dict(model_config.additional_params)
+
+    # Backwards compatibility: allow endpoint under additional_params for HTTP provider
+    endpoint = model_config.endpoint
+    if provider_name == "http" and endpoint is None:
+        endpoint = additional_params.pop("endpoint", None)
+
     # Convert ModelConfig to ProviderConfig
     provider_config = ProviderConfig(
         name=provider_name,
         model=model_config.model,
+        endpoint=endpoint,
+        timeout=model_config.timeout or 60,
         temperature=model_config.temperature,
         max_tokens=model_config.max_tokens,
-        additional_params=model_config.additional_params,
+        additional_params=additional_params,
     )
 
     # Get provider class and instantiate