feat: add JUnit XML exporter and --fail-under CI quality gate

README.md

@@ -16,7 +16,8 @@ PromptLens runs golden test sets against multiple models, scores outputs using L
 - **LLM-as-Judge Scoring** - Automated evaluation using another LLM with configurable criteria
 - **Cost & Latency Tracking** - Monitor per-query costs and response times across models
 - **Beautiful Reports** - Interactive HTML reports with charts, comparisons, and detailed results
-- **Multiple Export Formats** - HTML, JSON, CSV, and Markdown outputs
+- **Multiple Export Formats** - HTML, JSON, CSV, Markdown, and JUnit XML outputs
+- **CI-Native Quality Gates** - JUnit XML reports plus a `--fail-under` score gate that fails the build on quality regressions
 - **Parallel Execution** - Async execution with configurable concurrency and retry logic
 - **Portable & Local** - No cloud backend, all data stays on your machine
 - **Easy to Extend** - Plugin architecture for custom providers, judges, and exporters
@@ -308,11 +309,43 @@ output:
     - json                          # Raw JSON data
     - csv                           # Flattened spreadsheet
     - md                            # Markdown summary
+    - junit                         # JUnit XML for CI test reporting
   run_name: "My Evaluation"         # Display name
 ```
 
 ---
 
+## CI/CD Integration
+
+PromptLens speaks the language your CI system already understands: JUnit XML test reports and exit codes.
+
+Add `junit` to your output formats, then gate the build on judge scores:
+
+```bash
+promptlens run config.yaml --fail-under 3.5
+```
+
+- Each golden-set test case becomes a JUnit test case (one test suite per model).
+- A test case scoring below the threshold is reported as a failure, a model API error as an error, and an unjudged case as skipped.
+- If any model's average judge score falls below `--fail-under`, the command exits with code 2, failing the pipeline. Exit code 1 is reserved for run errors, so CI can tell quality regressions apart from infrastructure failures.
+
+Example GitHub Actions step:
+
+```yaml
+- name: Run prompt evals
+  run: promptlens run config.yaml --fail-under 3.5
+
+- name: Publish eval report
+  uses: mikepenz/action-junit-report@v5
+  if: always()
+  with:
+    report_paths: "promptlens_results/*/junit.xml"
+```
+
+The same `junit.xml` works with GitLab (`artifacts:reports:junit`), Jenkins, CircleCI, and any other JUnit-compatible report viewer.
+
+---
+
 ## Examples
 
 ### Basic Single Model Evaluation
@@ -557,6 +590,7 @@ class RuleBasedJudge(BaseJudge):
 - [x] LLM-as-judge scoring
 - [x] HTML reports with charts
 - [x] JSON/CSV/Markdown export
+- [x] JUnit XML export and `--fail-under` CI quality gate
 - [x] Parallel execution with retry logic
 - [ ] Multi-judge consensus scoring
 - [ ] Synthetic test case generation

promptlens/cli.py

@@ -17,8 +17,10 @@
 from promptlens.exporters.csv_exporter import CSVExporter
 from promptlens.exporters.html_exporter import HTMLExporter
 from promptlens.exporters.json_exporter import JSONExporter
+from promptlens.exporters.junit_exporter import JUnitXMLExporter
 from promptlens.exporters.markdown_exporter import MarkdownExporter
 from promptlens.models.config import RunConfig
+from promptlens.models.result import RunResult
 from promptlens.runners.runner import Runner
 
 # Load environment variables
@@ -35,6 +37,27 @@ def _remove_path_if_exists(path: Path) -> None:
         shutil.rmtree(path)
 
 
+def _check_fail_under(result: "RunResult", fail_under: float) -> list:
+    """Return models whose average judge score falls below the gate.
+
+    A model with no judge scores at all also fails the gate, since the gate
+    cannot be evaluated without scores and a silent pass would be misleading.
+
+    Args:
+        result: The completed run result
+        fail_under: Minimum acceptable average judge score (1-5 scale)
+
+    Returns:
+        List of (model, average_score_or_None) tuples that fail the gate
+    """
+    failing = []
+    for model in result.models_tested:
+        avg = result.get_average_score(model)
+        if avg is None or avg < fail_under:
+            failing.append((model, avg))
+    return failing
+
+
 def setup_logging(level: str = "INFO") -> None:
     """Set up logging configuration.
 
@@ -81,11 +104,22 @@ def cli(log_level: str) -> None:
     is_flag=True,
     help="Validate config without running evaluation",
 )
+@click.option(
+    "--fail-under",
+    type=click.FloatRange(1.0, 5.0),
+    default=None,
+    help=(
+        "Quality gate for CI: exit with code 2 if any model's average judge "
+        "score falls below this value (1-5 scale). Also sets the per-test "
+        "failure threshold used by the junit export format."
+    ),
+)
 def run(
     config: str,
     golden_set: Optional[str],
     output_dir: Optional[str],
     dry_run: bool,
+    fail_under: Optional[float],
 ) -> None:
     """Run evaluation with the given configuration file.
 
@@ -95,6 +129,7 @@ def run(
         promptlens run config.yaml
         promptlens run config.yaml --output-dir ./results
         promptlens run config.yaml --dry-run
+        promptlens run config.yaml --fail-under 3.5
     """
     try:
         # Load config
@@ -141,6 +176,7 @@ def run(
             "csv": (CSVExporter(), "results.csv"),
             "md": (MarkdownExporter(), "results.md"),
             "html": (HTMLExporter(), "report.html"),
+            "junit": (JUnitXMLExporter(fail_under=fail_under), "junit.xml"),
         }
 
         exported_files = []
@@ -168,6 +204,21 @@ def run(
             html_path = run_output_dir / "report.html"
             console.print(f"\n[cyan]View report: file://{html_path.absolute()}[/cyan]")
 
+        # Quality gate for CI
+        if fail_under is not None:
+            failing_models = _check_fail_under(result, fail_under)
+            if failing_models:
+                console.print(
+                    f"\n[bold red]✗ Quality gate failed (--fail-under {fail_under:g}):[/bold red]"
+                )
+                for model, avg in failing_models:
+                    avg_display = f"{avg:.2f}" if avg is not None else "no scores"
+                    console.print(f"  {model}: average judge score {avg_display}")
+                sys.exit(2)
+            console.print(
+                f"\n[bold green]✓ Quality gate passed (--fail-under {fail_under:g})[/bold green]"
+            )
+
     except Exception as e:
         console.print(f"\n[bold red]Error:[/bold red] {e}")
         logging.exception("Evaluation failed")
@@ -268,7 +319,7 @@ def list_runs(output_dir: str) -> None:
 @click.option(
     "--format",
     "export_format",
-    type=click.Choice(["json", "csv", "md", "html"], case_sensitive=False),
+    type=click.Choice(["json", "csv", "md", "html", "junit"], case_sensitive=False),
     required=True,
     help="Export format",
 )
@@ -313,7 +364,13 @@ def export(run_id: str, export_format: str, output: Optional[str], output_dir: s
 
         # Determine output path
         if not output:
-            extensions = {"json": ".json", "csv": ".csv", "md": ".md", "html": ".html"}
+            extensions = {
+                "json": ".json",
+                "csv": ".csv",
+                "md": ".md",
+                "html": ".html",
+                "junit": ".xml",
+            }
             output = f"export_{run_id}{extensions[export_format]}"
 
         # Export
@@ -322,6 +379,7 @@ def export(run_id: str, export_format: str, output: Optional[str], output_dir: s
             "csv": CSVExporter(),
             "md": MarkdownExporter(),
             "html": HTMLExporter(),
+            "junit": JUnitXMLExporter(),
         }
 
         exporter = exporters[export_format]

promptlens/exporters/__init__.py

@@ -5,11 +5,13 @@
 from promptlens.exporters.json_exporter import JSONExporter
 from promptlens.exporters.csv_exporter import CSVExporter
 from promptlens.exporters.markdown_exporter import MarkdownExporter
+from promptlens.exporters.junit_exporter import JUnitXMLExporter
 
 __all__ = [
     "BaseExporter",
     "HTMLExporter",
     "JSONExporter",
     "CSVExporter",
     "MarkdownExporter",
+    "JUnitXMLExporter",
 ]