feat(cel): preserve native values

.github/scripts/benchstat-summary.py

@@ -1,136 +1,162 @@
 #!/usr/bin/env python3
-"""
-benchstat-summary.py — parse benchstat output and produce a markdown summary.
-
-Usage: benchstat-summary.py [--threshold PCT] benchstat.txt
-
-Extracts significant regressions/improvements from benchstat output.
-Exit code 1 if any regression exceeds the threshold (default: 5%).
-"""
+"""Render significant benchstat CSV changes and enforce regression thresholds."""
 
 import argparse
+import csv
 import re
 import sys
 from dataclasses import dataclass
+from typing import Iterable
+
 
+CHANGE_RE = re.compile(r"^([+-]\d+(?:\.\d+)?)%$")
 
-@dataclass
+
+@dataclass(frozen=True)
 class Change:
     name: str
+    metric: str
     base: str
     head: str
     pct: float
     pval: str
 
 
-# Matches lines like:
-#   BenchmarkName-4   1.030m ±  3%   1.304m ±  5%  +26.57% (p=0.000 n=10)
-#   BenchmarkName-4   3.997  ± 1%    3.910  ± 0%   -2.16%  (p=0.000 n=10)
-LINE_RE = re.compile(
-    r"^(\S+)"  # benchmark name
-    r"\s+"
-    r"(\S+)"  # base value
-    r"\s+±\s+\d+%"  # base variance
-    r"\s+"
-    r"(\S+)"  # head value
-    r"\s+±\s+\d+%"  # head variance
-    r"\s+"
-    r"([+-]\d+\.\d+)%"  # percentage change
-    r"\s+"
-    r"\(p=(\d+\.\d+)"  # p-value
-)
+def parse_benchstat_rows(rows: Iterable[list[str]]) -> list[Change]:
+    changes: list[Change] = []
+    columns: tuple[str, int, int, int, int] | None = None
+    found_table = False
+    for row in rows:
+        if not row:
+            continue
+        if row[0] == "" and "vs base" in row:
+            metric_indexes = [
+                index
+                for index, value in enumerate(row)
+                if value and value not in {"CI", "vs base", "P"}
+            ]
+            if len(metric_indexes) < 2:
+                raise ValueError(f"invalid benchstat metric header: {row}")
+            columns = (
+                row[metric_indexes[0]],
+                metric_indexes[0],
+                metric_indexes[1],
+                row.index("vs base"),
+                row.index("P"),
+            )
+            found_table = True
+            continue
+        if columns is None or row[0] == "geomean":
+            continue
+
+        metric, base_index, head_index, change_index, pval_index = columns
+        required_length = max(base_index, head_index, change_index, pval_index) + 1
+        row.extend([""] * (required_length - len(row)))
+        if not row[base_index] or not row[head_index]:
+            continue
+        match = CHANGE_RE.match(row[change_index])
+        if match is None:
+            continue
+        changes.append(
+            Change(
+                name=row[0],
+                metric=metric,
+                base=format_metric(row[base_index], metric),
+                head=format_metric(row[head_index], metric),
+                pct=float(match.group(1)),
+                pval=row[pval_index].removeprefix("p=").split()[0],
+            )
+        )
+    if not found_table:
+        raise ValueError("input contains no benchstat CSV metric tables")
+    return changes
 
 
 def parse_benchstat(path: str) -> list[Change]:
-    changes = []
-    with open(path) as f:
-        for line in f:
-            stripped = line.strip()
-            if stripped.startswith("geomean"):
-                continue
-            m = LINE_RE.match(stripped)
-            if not m:
-                continue
-            changes.append(
-                Change(
-                    name=m.group(1),
-                    base=m.group(2),
-                    head=m.group(3),
-                    pct=float(m.group(4)),
-                    pval=m.group(5),
-                )
-            )
-    return changes
+    with open(path, newline="", encoding="utf-8") as source:
+        return parse_benchstat_rows(csv.reader(source))
+
+
+def format_metric(raw: str, metric: str) -> str:
+    value = float(raw)
+    if metric == "sec/op":
+        for scale, suffix in ((1, "s/op"), (1e3, "ms/op"), (1e6, "us/op"), (1e9, "ns/op")):
+            converted = value * scale
+            if converted >= 1:
+                return f"{converted:.3g} {suffix}"
+    if metric in {"B/op", "allocs/op"}:
+        return f"{value:.3g} {metric}"
+    return f"{value:.3g} {metric}"
+
+
+def regressions_above_threshold(changes: list[Change], threshold: float) -> list[Change]:
+    return sorted(
+        [change for change in changes if change.pct > threshold],
+        key=lambda change: -change.pct,
+    )
 
 
 def render_markdown(changes: list[Change], threshold: float) -> str:
-    regressions = sorted([c for c in changes if c.pct > 0], key=lambda c: -c.pct)
-    improvements = sorted([c for c in changes if c.pct < 0], key=lambda c: c.pct)
-
+    regressions = sorted([change for change in changes if change.pct > 0], key=lambda change: -change.pct)
+    improvements = sorted([change for change in changes if change.pct < 0], key=lambda change: change.pct)
     if not regressions and not improvements:
         return "### No significant performance changes detected\n"
 
     lines: list[str] = []
-
     if regressions:
-        above = [r for r in regressions if r.pct > threshold]
+        above = regressions_above_threshold(changes, threshold)
+        heading = f"### {len(regressions)} minor regression(s) (all within {threshold:g}% threshold)\n"
         if above:
-            lines.append(
-                f"### {len(regressions)} regression(s) detected (threshold: >{threshold:g}%)\n"
-            )
-        else:
-            lines.append(
-                f"### {len(regressions)} minor regression(s) (all within {threshold:g}% threshold)\n"
-            )
-
-        lines.append("| Benchmark | Base | Head | Change | p-value |")
-        lines.append("|-----------|------|------|--------|---------|")
-        for r in regressions:
-            change = f"+{r.pct:.2f}%"
-            if r.pct > threshold:
-                change = f"**{change}**"
-            lines.append(f"| `{r.name}` | {r.base} | {r.head} | {change} | {r.pval} |")
-        lines.append("")
+            heading = f"### {len(regressions)} regression(s) detected (threshold: >{threshold:g}%)\n"
+        lines.append(heading)
+        lines.extend(render_table(regressions, threshold, emphasize_regressions=True))
 
     if improvements:
-        lines.append(f"<details>")
+        lines.append("<details>")
         lines.append(f"<summary>{len(improvements)} improvement(s)</summary>\n")
-        lines.append("| Benchmark | Base | Head | Change | p-value |")
-        lines.append("|-----------|------|------|--------|---------|")
-        for imp in improvements:
-            lines.append(
-                f"| `{imp.name}` | {imp.base} | {imp.head} | {imp.pct:.2f}% | {imp.pval} |"
-            )
-        lines.append("")
+        lines.extend(render_table(improvements, threshold, emphasize_regressions=False))
         lines.append("</details>")
         lines.append("")
-
     return "\n".join(lines)
 
 
-def main():
+def render_table(changes: list[Change], threshold: float, emphasize_regressions: bool) -> list[str]:
+    lines = [
+        "| Benchmark | Metric | Base | Head | Change | p-value |",
+        "|-----------|--------|------|------|--------|---------|",
+    ]
+    for change in changes:
+        percentage = f"{change.pct:+.2f}%"
+        if emphasize_regressions and change.pct > threshold:
+            percentage = f"**{percentage}**"
+        lines.append(
+            f"| `{change.name}` | {change.metric} | {change.base} | {change.head} | {percentage} | {change.pval} |"
+        )
+    lines.append("")
+    return lines
+
+
+def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("input", help="Path to benchstat output file")
-    parser.add_argument(
-        "--threshold",
-        type=float,
-        default=5,
-        help="Regression percentage threshold to flag as failure (default: 5)",
-    )
+    parser.add_argument("input", help="Path to benchstat CSV output")
+    parser.add_argument("--threshold", type=float, default=5, help="Regression threshold percentage")
+    parser.add_argument("--no-fail", action="store_true", help="Render regressions without returning a failure")
     args = parser.parse_args()
 
-    changes = parse_benchstat(args.input)
-    summary = render_markdown(changes, args.threshold)
-    print(summary)
+    try:
+        changes = parse_benchstat(args.input)
+    except (OSError, ValueError) as error:
+        parser.error(str(error))
+    print(render_markdown(changes, args.threshold))
 
-    # Exit 1 if any regression exceeds the threshold
-    regressions_above_threshold = [c for c in changes if c.pct > args.threshold]
-    if regressions_above_threshold:
-        print(f"\nFailed: {len(regressions_above_threshold)} benchmark(s) regressed by more than {args.threshold:g}%:")
-        for c in sorted(regressions_above_threshold, key=lambda c: -c.pct):
-            print(f"  {c.name}: {c.base} -> {c.head} (+{c.pct:.2f}%)")
-        sys.exit(1)
+    regressions = regressions_above_threshold(changes, args.threshold)
+    if regressions and not args.no_fail:
+        print(f"\nFailed: {len(regressions)} metric(s) regressed by more than {args.threshold:g}%:")
+        for change in regressions:
+            print(f"  {change.name} {change.metric}: {change.base} -> {change.head} ({change.pct:+.2f}%)")
+        return 1
+    return 0
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

.github/scripts/benchstat-summary_test.py

@@ -0,0 +1,64 @@
+import csv
+import io
+import runpy
+from pathlib import Path
+
+import pytest
+
+
+SCRIPT = runpy.run_path(Path(__file__).with_name("benchstat-summary.py"))
+parse_benchstat_rows = SCRIPT["parse_benchstat_rows"]
+regressions_above_threshold = SCRIPT["regressions_above_threshold"]
+render_markdown = SCRIPT["render_markdown"]
+
+BENCHSTAT_CSV = """goos: linux
+goarch: amd64
+,.tmp/base.txt,,.tmp/head.txt,,,
+,sec/op,CI,sec/op,CI,vs base,P
+Thing-8,1e-06,± 1%,1.2e-06,± 1%,+20.00%,p=0.008 n=10
+Stable-8,2e-06,± 1%,2.01e-06,± 1%,~,p=0.310 n=10
+HeadOnly-8,,,3e-06,± 1%
+geomean,1e-06,,1.2e-06,,+20.00%,
+
+,.tmp/base.txt,,.tmp/head.txt,,,
+,B/op,CI,B/op,CI,vs base,P
+Thing-8,200,± 0%,250,± 0%,+25.00%,p=0.008 n=10
+
+,.tmp/base.txt,,.tmp/head.txt,,,
+,allocs/op,CI,allocs/op,CI,vs base,P
+Thing-8,4,± 0%,3,± 0%,-25.00%,p=0.008 n=10
+"""
+
+
+def test_parse_benchstat_rows_preserves_metric_identity_and_formats_values():
+    changes = parse_benchstat_rows(csv.reader(io.StringIO(BENCHSTAT_CSV)))
+
+    assert [(change.name, change.metric, change.base, change.head, change.pct) for change in changes] == [
+        ("Thing-8", "sec/op", "1 us/op", "1.2 us/op", 20.0),
+        ("Thing-8", "B/op", "200 B/op", "250 B/op", 25.0),
+        ("Thing-8", "allocs/op", "4 allocs/op", "3 allocs/op", -25.0),
+    ]
+
+
+def test_render_markdown_emphasizes_only_regressions_above_threshold():
+    changes = parse_benchstat_rows(csv.reader(io.StringIO(BENCHSTAT_CSV)))
+
+    rendered = render_markdown(changes, threshold=20)
+
+    assert "| Benchmark | Metric | Base | Head | Change | p-value |" in rendered
+    assert "| `Thing-8` | sec/op | 1 us/op | 1.2 us/op | +20.00% | 0.008 |" in rendered
+    assert "| `Thing-8` | B/op | 200 B/op | 250 B/op | **+25.00%** | 0.008 |" in rendered
+    assert "<summary>1 improvement(s)</summary>" in rendered
+
+
+def test_regression_gate_checks_each_metric():
+    changes = parse_benchstat_rows(csv.reader(io.StringIO(BENCHSTAT_CSV)))
+
+    regressions = regressions_above_threshold(changes, threshold=20)
+
+    assert [(change.metric, change.pct) for change in regressions] == [("B/op", 25.0)]
+
+
+def test_parse_benchstat_rows_rejects_non_csv_output():
+    with pytest.raises(ValueError, match="no benchstat CSV metric tables"):
+        parse_benchstat_rows(csv.reader(io.StringIO("BenchmarkThing 1 ns/op\n")))