From 522fc6553ff82171699b0c0eb90db5989d3782b5 Mon Sep 17 00:00:00 2001 From: Brian Sparker Date: Thu, 13 Aug 2026 06:11:58 -0700 Subject: [PATCH] feat: add JUnit XML exporter and --fail-under CI quality gate Co-Authored-By: Claude Fable 5 --- README.md | 36 +++- promptlens/cli.py | 62 ++++++- promptlens/exporters/__init__.py | 2 + promptlens/exporters/junit_exporter.py | 226 ++++++++++++++++++++++++ tests/test_junit_exporter.py | 234 +++++++++++++++++++++++++ 5 files changed, 557 insertions(+), 3 deletions(-) create mode 100644 promptlens/exporters/junit_exporter.py create mode 100644 tests/test_junit_exporter.py diff --git a/README.md b/README.md index 1435635..076e148 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ PromptLens runs golden test sets against multiple models, scores outputs using L - **LLM-as-Judge Scoring** - Automated evaluation using another LLM with configurable criteria - **Cost & Latency Tracking** - Monitor per-query costs and response times across models - **Beautiful Reports** - Interactive HTML reports with charts, comparisons, and detailed results -- **Multiple Export Formats** - HTML, JSON, CSV, and Markdown outputs +- **Multiple Export Formats** - HTML, JSON, CSV, Markdown, and JUnit XML outputs +- **CI-Native Quality Gates** - JUnit XML reports plus a `--fail-under` score gate that fails the build on quality regressions - **Parallel Execution** - Async execution with configurable concurrency and retry logic - **Portable & Local** - No cloud backend, all data stays on your machine - **Easy to Extend** - Plugin architecture for custom providers, judges, and exporters @@ -308,11 +309,43 @@ output: - json # Raw JSON data - csv # Flattened spreadsheet - md # Markdown summary + - junit # JUnit XML for CI test reporting run_name: "My Evaluation" # Display name ``` --- +## CI/CD Integration + +PromptLens speaks the language your CI system already understands: JUnit XML test reports and exit codes. + +Add `junit` to your output formats, then gate the build on judge scores: + +```bash +promptlens run config.yaml --fail-under 3.5 +``` + +- Each golden-set test case becomes a JUnit test case (one test suite per model). +- A test case scoring below the threshold is reported as a failure, a model API error as an error, and an unjudged case as skipped. +- If any model's average judge score falls below `--fail-under`, the command exits with code 2, failing the pipeline. Exit code 1 is reserved for run errors, so CI can tell quality regressions apart from infrastructure failures. + +Example GitHub Actions step: + +```yaml +- name: Run prompt evals + run: promptlens run config.yaml --fail-under 3.5 + +- name: Publish eval report + uses: mikepenz/action-junit-report@v5 + if: always() + with: + report_paths: "promptlens_results/*/junit.xml" +``` + +The same `junit.xml` works with GitLab (`artifacts:reports:junit`), Jenkins, CircleCI, and any other JUnit-compatible report viewer. + +--- + ## Examples ### Basic Single Model Evaluation @@ -557,6 +590,7 @@ class RuleBasedJudge(BaseJudge): - [x] LLM-as-judge scoring - [x] HTML reports with charts - [x] JSON/CSV/Markdown export +- [x] JUnit XML export and `--fail-under` CI quality gate - [x] Parallel execution with retry logic - [ ] Multi-judge consensus scoring - [ ] Synthetic test case generation diff --git a/promptlens/cli.py b/promptlens/cli.py index 235963a..d7c9f90 100644 --- a/promptlens/cli.py +++ b/promptlens/cli.py @@ -17,8 +17,10 @@ from promptlens.exporters.csv_exporter import CSVExporter from promptlens.exporters.html_exporter import HTMLExporter from promptlens.exporters.json_exporter import JSONExporter +from promptlens.exporters.junit_exporter import JUnitXMLExporter from promptlens.exporters.markdown_exporter import MarkdownExporter from promptlens.models.config import RunConfig +from promptlens.models.result import RunResult from promptlens.runners.runner import Runner # Load environment variables @@ -35,6 +37,27 @@ def _remove_path_if_exists(path: Path) -> None: shutil.rmtree(path) +def _check_fail_under(result: "RunResult", fail_under: float) -> list: + """Return models whose average judge score falls below the gate. + + A model with no judge scores at all also fails the gate, since the gate + cannot be evaluated without scores and a silent pass would be misleading. + + Args: + result: The completed run result + fail_under: Minimum acceptable average judge score (1-5 scale) + + Returns: + List of (model, average_score_or_None) tuples that fail the gate + """ + failing = [] + for model in result.models_tested: + avg = result.get_average_score(model) + if avg is None or avg < fail_under: + failing.append((model, avg)) + return failing + + def setup_logging(level: str = "INFO") -> None: """Set up logging configuration. @@ -81,11 +104,22 @@ def cli(log_level: str) -> None: is_flag=True, help="Validate config without running evaluation", ) +@click.option( + "--fail-under", + type=click.FloatRange(1.0, 5.0), + default=None, + help=( + "Quality gate for CI: exit with code 2 if any model's average judge " + "score falls below this value (1-5 scale). Also sets the per-test " + "failure threshold used by the junit export format." + ), +) def run( config: str, golden_set: Optional[str], output_dir: Optional[str], dry_run: bool, + fail_under: Optional[float], ) -> None: """Run evaluation with the given configuration file. @@ -95,6 +129,7 @@ def run( promptlens run config.yaml promptlens run config.yaml --output-dir ./results promptlens run config.yaml --dry-run + promptlens run config.yaml --fail-under 3.5 """ try: # Load config @@ -141,6 +176,7 @@ def run( "csv": (CSVExporter(), "results.csv"), "md": (MarkdownExporter(), "results.md"), "html": (HTMLExporter(), "report.html"), + "junit": (JUnitXMLExporter(fail_under=fail_under), "junit.xml"), } exported_files = [] @@ -168,6 +204,21 @@ def run( html_path = run_output_dir / "report.html" console.print(f"\n[cyan]View report: file://{html_path.absolute()}[/cyan]") + # Quality gate for CI + if fail_under is not None: + failing_models = _check_fail_under(result, fail_under) + if failing_models: + console.print( + f"\n[bold red]✗ Quality gate failed (--fail-under {fail_under:g}):[/bold red]" + ) + for model, avg in failing_models: + avg_display = f"{avg:.2f}" if avg is not None else "no scores" + console.print(f" {model}: average judge score {avg_display}") + sys.exit(2) + console.print( + f"\n[bold green]✓ Quality gate passed (--fail-under {fail_under:g})[/bold green]" + ) + except Exception as e: console.print(f"\n[bold red]Error:[/bold red] {e}") logging.exception("Evaluation failed") @@ -268,7 +319,7 @@ def list_runs(output_dir: str) -> None: @click.option( "--format", "export_format", - type=click.Choice(["json", "csv", "md", "html"], case_sensitive=False), + type=click.Choice(["json", "csv", "md", "html", "junit"], case_sensitive=False), required=True, help="Export format", ) @@ -313,7 +364,13 @@ def export(run_id: str, export_format: str, output: Optional[str], output_dir: s # Determine output path if not output: - extensions = {"json": ".json", "csv": ".csv", "md": ".md", "html": ".html"} + extensions = { + "json": ".json", + "csv": ".csv", + "md": ".md", + "html": ".html", + "junit": ".xml", + } output = f"export_{run_id}{extensions[export_format]}" # Export @@ -322,6 +379,7 @@ def export(run_id: str, export_format: str, output: Optional[str], output_dir: s "csv": CSVExporter(), "md": MarkdownExporter(), "html": HTMLExporter(), + "junit": JUnitXMLExporter(), } exporter = exporters[export_format] diff --git a/promptlens/exporters/__init__.py b/promptlens/exporters/__init__.py index 057d0bb..26f72e5 100644 --- a/promptlens/exporters/__init__.py +++ b/promptlens/exporters/__init__.py @@ -5,6 +5,7 @@ from promptlens.exporters.json_exporter import JSONExporter from promptlens.exporters.csv_exporter import CSVExporter from promptlens.exporters.markdown_exporter import MarkdownExporter +from promptlens.exporters.junit_exporter import JUnitXMLExporter __all__ = [ "BaseExporter", @@ -12,4 +13,5 @@ "JSONExporter", "CSVExporter", "MarkdownExporter", + "JUnitXMLExporter", ] diff --git a/promptlens/exporters/junit_exporter.py b/promptlens/exporters/junit_exporter.py new file mode 100644 index 0000000..3468724 --- /dev/null +++ b/promptlens/exporters/junit_exporter.py @@ -0,0 +1,226 @@ +"""JUnit XML exporter for run results. + +Emits a JUnit-style XML report so evaluation runs plug directly into CI +systems (GitHub Actions test summaries, GitLab, Jenkins, CircleCI) without +any custom glue. One test suite is emitted per model, and one test case per +evaluated golden-set entry. + +Mapping rules: + - A test case whose model response errored is reported as an . + - A test case whose judge score is below the failure threshold is + reported as a . + - A test case that was never judged (judging disabled or judge failed) + is reported as , so CI does not report a false pass. + - Everything else is a pass. +""" + +import logging +import xml.etree.ElementTree as ET +from typing import List, Optional + +from promptlens.exporters.base import BaseExporter +from promptlens.models.result import EvaluationResult, RunResult + +logger = logging.getLogger(__name__) + +# Judge scores are on a 1-5 scale. Scores below this value count as failures +# unless a different threshold is provided. +DEFAULT_FAIL_UNDER = 3.0 + + +class JUnitXMLExporter(BaseExporter): + """Exporter for JUnit XML format. + + Produces a document with one per model tested, + suitable for CI test-report ingestion. + + Args: + fail_under: Judge score threshold (1-5 scale). Test cases scoring + strictly below this value are marked as failures. Defaults to + DEFAULT_FAIL_UNDER. + """ + + def __init__(self, fail_under: Optional[float] = None) -> None: + self.fail_under = DEFAULT_FAIL_UNDER if fail_under is None else float(fail_under) + + def export(self, result: RunResult, output_path: str) -> None: + """Export results to a JUnit XML file. + + Args: + result: The run result to export + output_path: Path to write the XML file + """ + path = self.ensure_output_dir(output_path) + + testsuites = ET.Element("testsuites") + testsuites.set("name", result.run_name or result.golden_set_name) + testsuites.set("timestamp", result.timestamp.isoformat()) + + total_tests = 0 + total_failures = 0 + total_errors = 0 + total_skipped = 0 + total_time = 0.0 + + for model in result.models_tested: + model_results = [ + r for r in result.results if r.model_response.model == model + ] + suite, stats = self._build_suite(result, model, model_results) + testsuites.append(suite) + total_tests += stats["tests"] + total_failures += stats["failures"] + total_errors += stats["errors"] + total_skipped += stats["skipped"] + total_time += stats["time"] + + testsuites.set("tests", str(total_tests)) + testsuites.set("failures", str(total_failures)) + testsuites.set("errors", str(total_errors)) + testsuites.set("skipped", str(total_skipped)) + testsuites.set("time", f"{total_time:.3f}") + + tree = ET.ElementTree(testsuites) + try: + ET.indent(tree, space=" ") # Python 3.9+ + except AttributeError: # pragma: no cover + pass + tree.write(str(path), encoding="utf-8", xml_declaration=True) + + logger.info(f"Exported results to {path}") + + def _build_suite( + self, + result: RunResult, + model: str, + model_results: List[EvaluationResult], + ) -> tuple: + """Build a element for one model. + + Args: + result: The full run result (for run-level metadata) + model: Model identifier for this suite + model_results: Evaluation results belonging to this model + + Returns: + Tuple of (testsuite Element, stats dict) + """ + suite = ET.Element("testsuite") + suite.set("name", model) + + failures = 0 + errors = 0 + skipped = 0 + suite_time = 0.0 + + provider = None + for eval_result in model_results: + provider = eval_result.model_response.provider + case_time = (eval_result.model_response.latency_ms or 0.0) / 1000.0 + suite_time += case_time + + testcase = ET.SubElement(suite, "testcase") + testcase.set("name", eval_result.test_case_id) + testcase.set("classname", f"{result.golden_set_name}.{model}") + testcase.set("time", f"{case_time:.3f}") + + response_error = eval_result.model_response.error + judge_score = eval_result.judge_score + + if response_error: + errors += 1 + error_el = ET.SubElement(testcase, "error") + error_el.set("message", _truncate(response_error, 300)) + error_el.set("type", "ModelResponseError") + error_el.text = response_error + elif judge_score is None: + skipped += 1 + skipped_el = ET.SubElement(testcase, "skipped") + skipped_el.set( + "message", + "No judge score available (judging disabled or judge failed)", + ) + elif judge_score.score < self.fail_under: + failures += 1 + failure_el = ET.SubElement(testcase, "failure") + failure_el.set( + "message", + f"Judge score {judge_score.score} is below " + f"threshold {self.fail_under:g}", + ) + failure_el.set("type", "JudgeScoreBelowThreshold") + failure_el.text = ( + f"Query: {_truncate(eval_result.query, 500)}\n" + f"Expected: {_truncate(eval_result.expected_behavior, 500)}\n" + f"Score: {judge_score.score}\n" + f"Explanation: {_truncate(judge_score.explanation, 1000)}" + ) + + system_out = ET.SubElement(testcase, "system-out") + out_lines = [ + f"provider: {eval_result.model_response.provider}", + f"latency_ms: {eval_result.model_response.latency_ms}", + f"cost_usd: {eval_result.model_response.cost_usd or 0.0}", + f"tokens_used: {eval_result.model_response.tokens_used or 0}", + ] + if judge_score is not None: + out_lines.append(f"judge_score: {judge_score.score}") + out_lines.append( + f"judge_explanation: {_truncate(judge_score.explanation, 500)}" + ) + system_out.text = "\n".join(out_lines) + + suite.set("tests", str(len(model_results))) + suite.set("failures", str(failures)) + suite.set("errors", str(errors)) + suite.set("skipped", str(skipped)) + suite.set("time", f"{suite_time:.3f}") + + properties = ET.Element("properties") + _add_property(properties, "model", model) + if provider: + _add_property(properties, "provider", provider) + _add_property(properties, "run_id", result.run_id) + _add_property(properties, "golden_set", result.golden_set_name) + avg_score = result.get_average_score(model) + if avg_score is not None: + _add_property(properties, "average_judge_score", f"{avg_score:.2f}") + _add_property( + properties, "total_cost_usd", f"{result.get_total_cost(model):.6f}" + ) + _add_property(properties, "fail_under", f"{self.fail_under:g}") + suite.insert(0, properties) + + stats = { + "tests": len(model_results), + "failures": failures, + "errors": errors, + "skipped": skipped, + "time": suite_time, + } + return suite, stats + + @property + def file_extension(self) -> str: + """Return the file extension. + + Returns: + The .xml extension + """ + return ".xml" + + +def _add_property(parent: ET.Element, name: str, value: str) -> None: + """Append a element to a parent.""" + prop = ET.SubElement(parent, "property") + prop.set("name", name) + prop.set("value", value) + + +def _truncate(text: str, limit: int) -> str: + """Truncate text to a character limit, marking the cut.""" + if text is None: + return "" + if len(text) <= limit: + return text + return text[: limit - 3] + "..." diff --git a/tests/test_junit_exporter.py b/tests/test_junit_exporter.py new file mode 100644 index 0000000..1f00d3b --- /dev/null +++ b/tests/test_junit_exporter.py @@ -0,0 +1,234 @@ +"""Tests for the JUnit XML exporter and the --fail-under quality gate.""" + +import xml.etree.ElementTree as ET +from datetime import datetime + +import pytest + +from promptlens.cli import _check_fail_under +from promptlens.exporters.junit_exporter import ( + DEFAULT_FAIL_UNDER, + JUnitXMLExporter, +) +from promptlens.models.result import ( + EvaluationResult, + JudgeScore, + ModelResponse, + RunResult, +) + + +def _make_response(model="model-a", provider="anthropic", error=None, **kwargs): + defaults = { + "content": "The answer is 42.", + "model": model, + "provider": provider, + "latency_ms": 1234.5, + "cost_usd": 0.0021, + "tokens_used": 150, + "error": error, + } + defaults.update(kwargs) + return ModelResponse(**defaults) + + +def _make_score(score, explanation="Looks correct."): + return JudgeScore( + score=score, + explanation=explanation, + judge_model="judge-model", + judge_provider="anthropic", + ) + + +def _make_eval(test_case_id, model="model-a", score=None, error=None): + return EvaluationResult( + test_case_id=test_case_id, + query="What is the answer?", + expected_behavior="Answers correctly", + model_response=_make_response(model=model, error=error), + judge_score=_make_score(score) if score is not None else None, + ) + + +def _make_run(results, models=None, run_name="ci-run"): + models = models or ["model-a"] + return RunResult( + run_id="run-123", + run_name=run_name, + timestamp=datetime(2026, 8, 13, 12, 0, 0), + golden_set_name="golden-set", + models_tested=models, + results=results, + ) + + +def _export(run_result, tmp_path, fail_under=None): + exporter = JUnitXMLExporter(fail_under=fail_under) + output = tmp_path / "junit.xml" + exporter.export(run_result, str(output)) + return ET.parse(str(output)).getroot() + + +class TestJUnitXMLExporter: + def test_file_extension(self): + assert JUnitXMLExporter().file_extension == ".xml" + + def test_passing_case_has_no_failure_children(self, tmp_path): + run = _make_run([_make_eval("tc-1", score=5)]) + root = _export(run, tmp_path) + + assert root.tag == "testsuites" + assert root.get("tests") == "1" + assert root.get("failures") == "0" + assert root.get("errors") == "0" + testcase = root.find("./testsuite/testcase") + assert testcase.get("name") == "tc-1" + assert testcase.find("failure") is None + assert testcase.find("error") is None + assert testcase.find("skipped") is None + + def test_low_score_marked_as_failure(self, tmp_path): + run = _make_run([_make_eval("tc-1", score=1)]) + root = _export(run, tmp_path) + + assert root.get("failures") == "1" + failure = root.find("./testsuite/testcase/failure") + assert failure is not None + assert failure.get("type") == "JudgeScoreBelowThreshold" + assert "below" in failure.get("message") + assert "Explanation:" in failure.text + + def test_custom_fail_under_threshold(self, tmp_path): + # Score of 4 passes the default gate but fails a 4.5 gate + run = _make_run([_make_eval("tc-1", score=4)]) + root = _export(run, tmp_path, fail_under=4.5) + assert root.get("failures") == "1" + + root = _export(run, tmp_path) + assert root.get("failures") == "0" + + def test_default_threshold_constant(self): + assert JUnitXMLExporter().fail_under == DEFAULT_FAIL_UNDER + + def test_model_error_marked_as_error(self, tmp_path): + run = _make_run([_make_eval("tc-1", error="API timeout")]) + root = _export(run, tmp_path) + + assert root.get("errors") == "1" + assert root.get("failures") == "0" + error = root.find("./testsuite/testcase/error") + assert error is not None + assert error.get("type") == "ModelResponseError" + assert "API timeout" in error.get("message") + + def test_unjudged_case_marked_as_skipped(self, tmp_path): + run = _make_run([_make_eval("tc-1")]) + root = _export(run, tmp_path) + + assert root.get("skipped") == "1" + assert root.get("failures") == "0" + skipped = root.find("./testsuite/testcase/skipped") + assert skipped is not None + + def test_one_suite_per_model(self, tmp_path): + results = [ + _make_eval("tc-1", model="model-a", score=5), + _make_eval("tc-1", model="model-b", score=2), + ] + run = _make_run(results, models=["model-a", "model-b"]) + root = _export(run, tmp_path) + + suites = root.findall("testsuite") + assert [s.get("name") for s in suites] == ["model-a", "model-b"] + assert suites[0].get("failures") == "0" + assert suites[1].get("failures") == "1" + assert root.get("tests") == "2" + assert root.get("failures") == "1" + + def test_suite_properties_include_run_metadata(self, tmp_path): + run = _make_run([_make_eval("tc-1", score=4)]) + root = _export(run, tmp_path) + + props = { + p.get("name"): p.get("value") + for p in root.findall("./testsuite/properties/property") + } + assert props["model"] == "model-a" + assert props["provider"] == "anthropic" + assert props["run_id"] == "run-123" + assert props["golden_set"] == "golden-set" + assert props["average_judge_score"] == "4.00" + assert "total_cost_usd" in props + + def test_testcase_time_uses_latency_seconds(self, tmp_path): + run = _make_run([_make_eval("tc-1", score=5)]) + root = _export(run, tmp_path) + + testcase = root.find("./testsuite/testcase") + assert testcase.get("time") == "1.234" + assert testcase.get("classname") == "golden-set.model-a" + + def test_special_characters_are_escaped(self, tmp_path): + result = EvaluationResult( + test_case_id="tc-<&>", + query='Query with & "quotes"', + expected_behavior="Behaves & renders bold", + model_response=_make_response(), + judge_score=_make_score(1, explanation='Bad & "wrong"'), + ) + run = _make_run([result]) + root = _export(run, tmp_path) + + # Parsing succeeded, and content round-trips intact + testcase = root.find("./testsuite/testcase") + assert testcase.get("name") == "tc-<&>" + failure = testcase.find("failure") + assert 'Bad & "wrong"' in failure.text + + def test_creates_output_directory(self, tmp_path): + run = _make_run([_make_eval("tc-1", score=5)]) + nested = tmp_path / "deep" / "nested" / "junit.xml" + JUnitXMLExporter().export(run, str(nested)) + assert nested.exists() + + def test_system_out_contains_metrics(self, tmp_path): + run = _make_run([_make_eval("tc-1", score=5)]) + root = _export(run, tmp_path) + + system_out = root.find("./testsuite/testcase/system-out") + assert "provider: anthropic" in system_out.text + assert "cost_usd: 0.0021" in system_out.text + assert "judge_score: 5" in system_out.text + + +class TestFailUnderGate: + def test_all_models_above_gate(self): + run = _make_run([_make_eval("tc-1", score=4), _make_eval("tc-2", score=5)]) + assert _check_fail_under(run, 3.0) == [] + + def test_model_below_gate_is_reported(self): + run = _make_run([_make_eval("tc-1", score=2), _make_eval("tc-2", score=2)]) + failing = _check_fail_under(run, 3.0) + assert len(failing) == 1 + model, avg = failing[0] + assert model == "model-a" + assert avg == pytest.approx(2.0) + + def test_average_exactly_at_gate_passes(self): + run = _make_run([_make_eval("tc-1", score=3)]) + assert _check_fail_under(run, 3.0) == [] + + def test_model_with_no_scores_fails_gate(self): + run = _make_run([_make_eval("tc-1")]) + failing = _check_fail_under(run, 3.0) + assert failing == [("model-a", None)] + + def test_mixed_models_only_failing_reported(self): + results = [ + _make_eval("tc-1", model="model-a", score=5), + _make_eval("tc-1", model="model-b", score=1), + ] + run = _make_run(results, models=["model-a", "model-b"]) + failing = _check_fail_under(run, 3.0) + assert [m for m, _ in failing] == ["model-b"]