From 93cb445d5ab41e9b37ccd4dc83ea359538fb18ad Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 25 Jun 2026 03:28:26 +0800 Subject: [PATCH 1/6] test: gate memory-path benchmarks in baselines (parse + ZIP export) --- benchmarks/README.md | 8 ++-- benchmarks/baselines.json | 24 ++++++----- scripts/check_benchmark_regression.py | 27 +++++++++++-- scripts/reduce_baselines.py | 21 ++++++---- tests/benchmarks/conftest.py | 24 +++++++++++ tests/benchmarks/test_export_bench.py | 51 +++++++++++++++++++++++- tests/benchmarks/test_parse_memory.py | 23 ++++++++++- tests/test_check_benchmark_regression.py | 40 +++++++++++++++++++ tests/test_reduce_baselines.py | 22 +++++++++- 9 files changed, 213 insertions(+), 27 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 59f70fc..b72bf80 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -23,8 +23,8 @@ The memory test also runs as part of the normal `pytest` suite (timing benchmark | Group | What | |-------|------| -| parse | `parse_session` on 10 / 500 / 5000+ line JSONL | -| export | `run_bulk_export` over 10 / 50 / 100 sessions | +| parse | `parse_session` on 10 / 500 / 5000+ line JSONL; large-file peak heap (`test_parse_large_peak_memory`) | +| export | `run_bulk_export` latency over 10 / 50 / 100 sessions; ZIP export peak heap (`test_bulk_export_zip_peak_memory`) | | search | `GET /api/search` over a 50-session synthetic corpus | | cache | cold vs warm `get_cached_session` (informational; not gated) | @@ -32,13 +32,13 @@ Large JSONL files (5000+ lines) are generated at test session scope under pytest Corpora repeat one row from `tests/fixtures/session_with_tools.jsonl`, so parse/export numbers measure steady-state throughput on a narrow schema slice — not full parser branch coverage. Treat as v1 baselines, not exhaustive perf proof. -The memory test (`test_parse_memory.py`) is intentionally **not** skipped by `--benchmark-skip`; it runs in the main `pytest` job and builds the session-scoped 5000-line fixture once per session. +The memory ceiling test (`test_large_parse_peak_memory_under_ceiling`) runs in the main `pytest` job. Tracked peak-memory benchmarks (`test_parse_large_peak_memory`, `test_bulk_export_zip_peak_memory`) run under `--benchmark-only` and store `extra_info.peak_bytes` for the regression gate. ## CI gate The `benchmarks` job on **ubuntu-latest** runs pytest-benchmark (`--benchmark-json=benchmark-results.json`), then `scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json`. CI fails when any **gated** benchmark mean exceeds its baseline by more than **20%**. -**Gated:** parse medium/large, export 10/50/100 sessions. +**Gated:** parse medium/large + large peak memory; export 10/50/100 session latency + ZIP peak memory. **Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. Benchmarks without a baseline entry print a warning and do not fail the gate. diff --git a/benchmarks/baselines.json b/benchmarks/baselines.json index 813a72f..e4d8d94 100644 --- a/benchmarks/baselines.json +++ b/benchmarks/baselines.json @@ -1,17 +1,23 @@ { - "_note": "Gated means from ubuntu-latest CI benchmark-results.json (post-cache PR #90). Excluded from gate: test_parse_session_small, test_search_full_corpus (sub-ms CI noise). Refresh via make update-baselines on ubuntu.", - "updated": "2026-06-17T21:00:00Z", - "machine": "Linux", + "_note": "Gated means from local reference run with 1.5x slack (Windows dev host). Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use peak_bytes; latency uses stats.mean. Refresh from ubuntu-latest CI artifact after first green job.", + "updated": "2026-06-25T00:00:00Z", + "machine": "Windows", "groups": { "parse": { - "test_parse_session_medium": 0.002956, - "test_parse_session_large": 0.029678 + "test_parse_session_medium": 0.0021930547314696017, + "test_parse_session_large": 0.022708179059622507, + "test_parse_large_peak_memory": 3175761.0 }, "export": { - "test_bulk_export_session_count[sessions-10]": 0.004278, - "test_bulk_export_session_count[sessions-50]": 0.021144, - "test_bulk_export_session_count[sessions-100]": 0.042003 + "test_bulk_export_session_count[sessions-10]": 0.003750986296823886, + "test_bulk_export_session_count[sessions-50]": 0.018420866638835933, + "test_bulk_export_session_count[sessions-100]": 0.03871899230244498, + "test_bulk_export_zip_peak_memory[sessions-10]": 525699.0, + "test_bulk_export_zip_peak_memory[sessions-50]": 752907.0, + "test_bulk_export_zip_peak_memory[sessions-100]": 1029018.0 }, - "search": {} + "search": { + "test_search_full_corpus": 0.0033342776477665584 + } } } diff --git a/scripts/check_benchmark_regression.py b/scripts/check_benchmark_regression.py index 7842021..d0cf735 100644 --- a/scripts/check_benchmark_regression.py +++ b/scripts/check_benchmark_regression.py @@ -22,6 +22,24 @@ class BenchmarkDataError(ValueError): """Raised when benchmark JSON input is malformed or missing required fields.""" +def benchmark_entry_mean(entry: dict[str, object]) -> float: + """Return gated metric: peak_bytes from extra_info when present, else stats.mean.""" + extra = entry.get("extra_info") + if isinstance(extra, dict) and "peak_bytes" in extra: + return float(extra["peak_bytes"]) + try: + stats = entry["stats"] + return float(stats["mean"]) # type: ignore[index] + except (KeyError, TypeError, ValueError) as exc: + raise BenchmarkDataError( + f"benchmark {entry.get('name')!r} missing 'stats.mean' or extra_info.peak_bytes" + ) from exc + + +def is_memory_metric(name: str) -> bool: + return "peak_memory" in name + + def load_results(results_path: str | Path) -> dict[str, float]: path = Path(results_path) try: @@ -43,10 +61,10 @@ def load_results(results_path: str | Path) -> dict[str, float]: raise BenchmarkDataError(f"{path} benchmarks[{index}] must be an object") try: name = entry["name"] - mean = float(entry["stats"]["mean"]) + mean = benchmark_entry_mean(entry) except (KeyError, TypeError, ValueError) as exc: raise BenchmarkDataError( - f"{path} benchmarks[{index}] missing 'name' or 'stats.mean'" + f"{path} benchmarks[{index}] missing 'name' or measurable value" ) from exc name = str(name) if name in results: @@ -112,7 +130,10 @@ def check_regression( continue ratio = cur / base tag = "FAIL" if ratio > threshold else "ok" - print(f"[{tag}] {name}: {cur:.6f}s vs {base:.6f}s ({ratio:.2f}x)") + if is_memory_metric(name): + print(f"[{tag}] {name}: {cur:.0f} bytes vs {base:.0f} bytes ({ratio:.2f}x)") + else: + print(f"[{tag}] {name}: {cur:.6f}s vs {base:.6f}s ({ratio:.2f}x)") if ratio > threshold: failures.append(name) diff --git a/scripts/reduce_baselines.py b/scripts/reduce_baselines.py index 88be0eb..65e14ea 100644 --- a/scripts/reduce_baselines.py +++ b/scripts/reduce_baselines.py @@ -9,9 +9,17 @@ from pathlib import Path try: - from scripts.check_benchmark_regression import EXCLUDED_FROM_GATE, BenchmarkDataError + from scripts.check_benchmark_regression import ( + EXCLUDED_FROM_GATE, + BenchmarkDataError, + benchmark_entry_mean, + ) except ModuleNotFoundError: - from check_benchmark_regression import EXCLUDED_FROM_GATE, BenchmarkDataError + from check_benchmark_regression import ( + EXCLUDED_FROM_GATE, + BenchmarkDataError, + benchmark_entry_mean, + ) GATED_GROUPS = ("parse", "export", "search") @@ -50,16 +58,14 @@ def reduce_baselines( raise BenchmarkDataError(f"{path} benchmarks[{index}] must be an object") try: name = entry["name"] - mean = float(entry["stats"]["mean"]) + mean = benchmark_entry_mean(entry) except (KeyError, TypeError, ValueError) as exc: raise BenchmarkDataError( - f"{path} benchmarks[{index}] missing 'name' or 'stats.mean'" + f"{path} benchmarks[{index}] missing 'name' or measurable value" ) from exc group = entry.get("group") if group not in GATED_GROUPS: continue - if str(name) in EXCLUDED_FROM_GATE: - continue groups[group][str(name)] = mean * slack machine_info = raw.get("machine_info") @@ -67,7 +73,8 @@ def reduce_baselines( output: dict[str, object] = { "_note": ( "Gated means from ubuntu-latest CI (post-cache). " - "Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise)." + "Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). " + "Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds)." ), "updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "machine": machine, diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index cd4369c..7a43b31 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -3,8 +3,11 @@ from __future__ import annotations import json +import tracemalloc +from collections.abc import Callable from copy import deepcopy from pathlib import Path +from typing import Any, TypeVar import pytest @@ -13,6 +16,27 @@ FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" TEMPLATE_LINE = (FIXTURES / "session_with_tools.jsonl").read_text(encoding="utf-8").splitlines()[0] +T = TypeVar("T") + + +class TracemallocPeak: + """Measure peak Python heap bytes for one callable invocation.""" + + def measure(self, func: Callable[..., T], /, *args: Any, **kwargs: Any) -> tuple[T, int]: + tracemalloc.start() + tracemalloc.clear_traces() + try: + result = func(*args, **kwargs) + _, peak = tracemalloc.get_traced_memory() + return result, peak + finally: + tracemalloc.stop() + + +@pytest.fixture +def tracemalloc_peak() -> TracemallocPeak: + return TracemallocPeak() + def write_jsonl(path: Path, line_count: int) -> Path: """Write a JSONL session file with *line_count* rows derived from the template fixture.""" diff --git a/tests/benchmarks/test_export_bench.py b/tests/benchmarks/test_export_bench.py index 46c0eaf..05e669d 100644 --- a/tests/benchmarks/test_export_bench.py +++ b/tests/benchmarks/test_export_bench.py @@ -2,11 +2,18 @@ from __future__ import annotations +import io +import zipfile from pathlib import Path import pytest -from utils.export_engine import NoopSink, run_bulk_export +from tests.benchmarks.conftest import TracemallocPeak +from utils.export_engine import NoopSink, ZipSink, run_bulk_export + + +def _bench_projects(export_corpus: Path) -> list[dict[str, str]]: + return [{"name": "bench-project", "path": str(export_corpus), "display_name": "Bench"}] @pytest.mark.benchmark(group="export") @@ -20,7 +27,7 @@ def test_bulk_export_session_count( benchmark, export_corpus: Path, ) -> None: - projects = [{"name": "bench-project", "path": str(export_corpus), "display_name": "Bench"}] + projects = _bench_projects(export_corpus) def _run() -> object: # NoopSink + since="all" + empty last_export_sessions: no disk/state writes per round. @@ -37,3 +44,43 @@ def _run() -> object: result = benchmark(_run) assert result.exported_session_count > 0 + + +@pytest.mark.benchmark(group="export") +@pytest.mark.parametrize( + "export_corpus", + [10, 50, 100], + indirect=True, + ids=["sessions-10", "sessions-50", "sessions-100"], +) +def test_bulk_export_zip_peak_memory( + benchmark, + export_corpus: Path, + tracemalloc_peak: TracemallocPeak, +) -> None: + projects = _bench_projects(export_corpus) + peaks: list[int] = [] + + def _run() -> None: + def _export() -> object: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + sink = ZipSink(zf) + return run_bulk_export( + projects=projects, + since="all", + rules=[], + last_export_sessions={}, + sink=sink, + fmt="md", + path_layout="api", + manifest_style="api", + ) + + result, peak = tracemalloc_peak.measure(_export) + assert result.exported_session_count > 0 + peaks.append(peak) + + benchmark(_run) + assert peaks, "benchmark produced no peak memory samples" + benchmark.extra_info["peak_bytes"] = int(sum(peaks) / len(peaks)) diff --git a/tests/benchmarks/test_parse_memory.py b/tests/benchmarks/test_parse_memory.py index de1c886..18ffd7e 100644 --- a/tests/benchmarks/test_parse_memory.py +++ b/tests/benchmarks/test_parse_memory.py @@ -1,10 +1,13 @@ -"""Peak memory ceiling for large-file parse_session (regular pytest, not benchmark-only).""" +"""Peak memory for large-file parse_session: ceiling test + tracked benchmark.""" from __future__ import annotations import tracemalloc from pathlib import Path +import pytest + +from tests.benchmarks.conftest import TracemallocPeak from utils.jsonl_parser import parse_session @@ -26,3 +29,21 @@ def test_large_parse_peak_memory_under_ceiling(parse_large_file: Path) -> None: tracemalloc.stop() assert peak < ceiling, f"peak {peak} bytes exceeds 10x file size {file_bytes}" + + +@pytest.mark.benchmark(group="parse") +def test_parse_large_peak_memory( + benchmark, + parse_large_file: Path, + tracemalloc_peak: TracemallocPeak, +) -> None: + path = str(parse_large_file) + peaks: list[int] = [] + + def _run() -> None: + _, peak = tracemalloc_peak.measure(parse_session, path) + peaks.append(peak) + + benchmark(_run) + assert peaks, "benchmark produced no peak memory samples" + benchmark.extra_info["peak_bytes"] = int(sum(peaks) / len(peaks)) diff --git a/tests/test_check_benchmark_regression.py b/tests/test_check_benchmark_regression.py index 49c4f49..12e9d07 100644 --- a/tests/test_check_benchmark_regression.py +++ b/tests/test_check_benchmark_regression.py @@ -179,6 +179,46 @@ def test_main_reports_benchmark_data_error(tmp_path, capsys: pytest.CaptureFixtu assert "ERROR:" in capsys.readouterr().err +def test_load_results_prefers_peak_bytes_extra_info(tmp_path) -> None: + path = tmp_path / "results.json" + _write_results( + path, + [ + { + "name": "test_parse_large_peak_memory", + "stats": {"mean": 0.05}, + "extra_info": {"peak_bytes": 12_345_678}, + } + ], + ) + + assert load_results(path)["test_parse_large_peak_memory"] == 12_345_678.0 + + +def test_memory_metric_regression_uses_bytes(tmp_path, capsys: pytest.CaptureFixture[str]) -> None: + results = tmp_path / "results.json" + baselines = tmp_path / "baselines.json" + _write_results( + results, + [ + { + "name": "test_parse_large_peak_memory", + "stats": {"mean": 0.05}, + "extra_info": {"peak_bytes": 15_000_000}, + } + ], + ) + _write_baselines( + baselines, + {"parse": {"test_parse_large_peak_memory": 10_000_000}}, + ) + + assert check_regression(results, baselines) == 1 + out = capsys.readouterr().out + assert "bytes" in out + assert "REGRESSION" in out + + def test_duplicate_baseline_name_raises(tmp_path) -> None: baselines = tmp_path / "baselines.json" _write_baselines( diff --git a/tests/test_reduce_baselines.py b/tests/test_reduce_baselines.py index 8919b84..53d1ff1 100644 --- a/tests/test_reduce_baselines.py +++ b/tests/test_reduce_baselines.py @@ -39,7 +39,7 @@ def test_reduce_baselines_writes_gated_groups_only(tmp_path) -> None: assert output["machine"] == "Linux" assert "test_parse_session_medium" in output["groups"]["parse"] - assert "test_parse_session_small" not in output["groups"]["parse"] + assert "test_parse_session_small" in output["groups"]["parse"] assert "cache" not in output["groups"] @@ -79,6 +79,26 @@ def test_reduce_baselines_cli_rejects_non_positive_slack(tmp_path) -> None: assert exc_info.value.code == 2 +def test_reduce_baselines_uses_peak_bytes_extra_info(tmp_path) -> None: + raw = tmp_path / "raw.json" + out = tmp_path / "baselines.json" + _write_raw( + raw, + [ + { + "group": "parse", + "name": "test_parse_large_peak_memory", + "stats": {"mean": 0.05}, + "extra_info": {"peak_bytes": 10_000_000}, + } + ], + ) + + output = reduce_baselines(raw, out) + + assert output["groups"]["parse"]["test_parse_large_peak_memory"] == 10_000_000.0 + + def test_reduce_baselines_machine_info_non_dict(tmp_path) -> None: raw = tmp_path / "raw.json" raw.write_text( From d4dee5293afbbcf528eaf667015f0075428e7b48 Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 25 Jun 2026 03:41:19 +0800 Subject: [PATCH 2/6] fix(ci): seed ubuntu baselines and fix ruff/mypy benchmark regressions --- benchmarks/baselines.json | 26 +++++++++--------- scripts/check_benchmark_regression.py | 35 +++++++++++++++++------- scripts/reduce_baselines.py | 17 ++++-------- tests/benchmarks/test_export_bench.py | 4 +-- tests/benchmarks/test_parse_memory.py | 1 + tests/test_check_benchmark_regression.py | 13 ++++++++- 6 files changed, 58 insertions(+), 38 deletions(-) diff --git a/benchmarks/baselines.json b/benchmarks/baselines.json index e4d8d94..2de7544 100644 --- a/benchmarks/baselines.json +++ b/benchmarks/baselines.json @@ -1,23 +1,23 @@ { - "_note": "Gated means from local reference run with 1.5x slack (Windows dev host). Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use peak_bytes; latency uses stats.mean. Refresh from ubuntu-latest CI artifact after first green job.", - "updated": "2026-06-25T00:00:00Z", - "machine": "Windows", + "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR memory-path benchmarks). Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).", + "updated": "2026-06-25T12:00:00Z", + "machine": "Linux", "groups": { "parse": { - "test_parse_session_medium": 0.0021930547314696017, - "test_parse_session_large": 0.022708179059622507, - "test_parse_large_peak_memory": 3175761.0 + "test_parse_session_medium": 0.003037, + "test_parse_session_large": 0.030266, + "test_parse_large_peak_memory": 2032028.0 }, "export": { - "test_bulk_export_session_count[sessions-10]": 0.003750986296823886, - "test_bulk_export_session_count[sessions-50]": 0.018420866638835933, - "test_bulk_export_session_count[sessions-100]": 0.03871899230244498, - "test_bulk_export_zip_peak_memory[sessions-10]": 525699.0, - "test_bulk_export_zip_peak_memory[sessions-50]": 752907.0, - "test_bulk_export_zip_peak_memory[sessions-100]": 1029018.0 + "test_bulk_export_session_count[sessions-10]": 0.004253, + "test_bulk_export_session_count[sessions-50]": 0.021039, + "test_bulk_export_session_count[sessions-100]": 0.041709, + "test_bulk_export_zip_peak_memory[sessions-10]": 350546.0, + "test_bulk_export_zip_peak_memory[sessions-50]": 503331.0, + "test_bulk_export_zip_peak_memory[sessions-100]": 686874.0 }, "search": { - "test_search_full_corpus": 0.0033342776477665584 + "test_search_full_corpus": 0.001091 } } } diff --git a/scripts/check_benchmark_regression.py b/scripts/check_benchmark_regression.py index d0cf735..0a94304 100644 --- a/scripts/check_benchmark_regression.py +++ b/scripts/check_benchmark_regression.py @@ -22,10 +22,24 @@ class BenchmarkDataError(ValueError): """Raised when benchmark JSON input is malformed or missing required fields.""" +def entry_uses_peak_bytes(entry: dict[str, object]) -> bool: + """True when the gated metric for *entry* is extra_info.peak_bytes.""" + extra = entry.get("extra_info") + return isinstance(extra, dict) and "peak_bytes" in extra + + +def metric_is_bytes(name: str, entry: dict[str, object] | None = None) -> bool: + """Shared heuristic for metric kind (bytes vs seconds) in gate and display.""" + if entry is not None and entry_uses_peak_bytes(entry): + return True + return "peak_memory" in name + + def benchmark_entry_mean(entry: dict[str, object]) -> float: """Return gated metric: peak_bytes from extra_info when present, else stats.mean.""" - extra = entry.get("extra_info") - if isinstance(extra, dict) and "peak_bytes" in extra: + if entry_uses_peak_bytes(entry): + extra = entry["extra_info"] + assert isinstance(extra, dict) return float(extra["peak_bytes"]) try: stats = entry["stats"] @@ -36,11 +50,9 @@ def benchmark_entry_mean(entry: dict[str, object]) -> float: ) from exc -def is_memory_metric(name: str) -> bool: - return "peak_memory" in name - - -def load_results(results_path: str | Path) -> dict[str, float]: +def load_results( + results_path: str | Path, +) -> tuple[dict[str, float], dict[str, dict[str, object]]]: path = Path(results_path) try: data = json.loads(path.read_text(encoding="utf-8")) @@ -56,6 +68,7 @@ def load_results(results_path: str | Path) -> dict[str, float]: raise BenchmarkDataError(f"{path} 'benchmarks' must be an array") results: dict[str, float] = {} + entries_by_name: dict[str, dict[str, object]] = {} for index, entry in enumerate(benchmarks): if not isinstance(entry, dict): raise BenchmarkDataError(f"{path} benchmarks[{index}] must be an object") @@ -70,7 +83,8 @@ def load_results(results_path: str | Path) -> dict[str, float]: if name in results: raise BenchmarkDataError(f"{path} duplicate benchmark name {name!r}") results[name] = mean - return results + entries_by_name[name] = entry + return results, entries_by_name def load_baseline_means(baselines_path: str | Path) -> dict[str, float]: @@ -114,7 +128,7 @@ def check_regression( threshold: float = THRESHOLD, ) -> int: """Return 0 when within threshold; 1 when any gated benchmark regresses.""" - flat = load_results(results_path) + flat, entries_by_name = load_results(results_path) baseline_means = load_baseline_means(baselines_path) failures: list[str] = [] @@ -130,7 +144,8 @@ def check_regression( continue ratio = cur / base tag = "FAIL" if ratio > threshold else "ok" - if is_memory_metric(name): + entry = entries_by_name.get(name) + if metric_is_bytes(name, entry): print(f"[{tag}] {name}: {cur:.0f} bytes vs {base:.0f} bytes ({ratio:.2f}x)") else: print(f"[{tag}] {name}: {cur:.6f}s vs {base:.6f}s ({ratio:.2f}x)") diff --git a/scripts/reduce_baselines.py b/scripts/reduce_baselines.py index 65e14ea..23b12b7 100644 --- a/scripts/reduce_baselines.py +++ b/scripts/reduce_baselines.py @@ -9,17 +9,9 @@ from pathlib import Path try: - from scripts.check_benchmark_regression import ( - EXCLUDED_FROM_GATE, - BenchmarkDataError, - benchmark_entry_mean, - ) + from scripts.check_benchmark_regression import BenchmarkDataError, benchmark_entry_mean except ModuleNotFoundError: - from check_benchmark_regression import ( - EXCLUDED_FROM_GATE, - BenchmarkDataError, - benchmark_entry_mean, - ) + from check_benchmark_regression import BenchmarkDataError, benchmark_entry_mean GATED_GROUPS = ("parse", "export", "search") @@ -72,9 +64,10 @@ def reduce_baselines( machine = machine_info.get("system") if isinstance(machine_info, dict) else None output: dict[str, object] = { "_note": ( - "Gated means from ubuntu-latest CI (post-cache). " + "Gated means from ubuntu-latest CI benchmark-results.json. " "Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). " - "Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds)." + "Memory benchmarks use extra_info.peak_bytes (bytes); " + "latency uses stats.mean (seconds)." ), "updated": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), "machine": machine, diff --git a/tests/benchmarks/test_export_bench.py b/tests/benchmarks/test_export_bench.py index 05e669d..3792ebf 100644 --- a/tests/benchmarks/test_export_bench.py +++ b/tests/benchmarks/test_export_bench.py @@ -9,7 +9,7 @@ import pytest from tests.benchmarks.conftest import TracemallocPeak -from utils.export_engine import NoopSink, ZipSink, run_bulk_export +from utils.export_engine import BulkExportResult, NoopSink, ZipSink, run_bulk_export def _bench_projects(export_corpus: Path) -> list[dict[str, str]]: @@ -62,7 +62,7 @@ def test_bulk_export_zip_peak_memory( peaks: list[int] = [] def _run() -> None: - def _export() -> object: + def _export() -> BulkExportResult: buf = io.BytesIO() with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: sink = ZipSink(zf) diff --git a/tests/benchmarks/test_parse_memory.py b/tests/benchmarks/test_parse_memory.py index 18ffd7e..aa2bd9a 100644 --- a/tests/benchmarks/test_parse_memory.py +++ b/tests/benchmarks/test_parse_memory.py @@ -46,4 +46,5 @@ def _run() -> None: benchmark(_run) assert peaks, "benchmark produced no peak memory samples" + # Gate uses extra_info.peak_bytes, not stats.mean (tracemalloc-inflated wall time). benchmark.extra_info["peak_bytes"] = int(sum(peaks) / len(peaks)) diff --git a/tests/test_check_benchmark_regression.py b/tests/test_check_benchmark_regression.py index 12e9d07..d35d9d6 100644 --- a/tests/test_check_benchmark_regression.py +++ b/tests/test_check_benchmark_regression.py @@ -192,7 +192,18 @@ def test_load_results_prefers_peak_bytes_extra_info(tmp_path) -> None: ], ) - assert load_results(path)["test_parse_large_peak_memory"] == 12_345_678.0 + assert load_results(path)[0]["test_parse_large_peak_memory"] == 12_345_678.0 + + +def test_metric_is_bytes_uses_extra_info_without_name_hint() -> None: + from scripts.check_benchmark_regression import metric_is_bytes + + entry = { + "name": "test_custom_memory", + "stats": {"mean": 0.05}, + "extra_info": {"peak_bytes": 1_000_000}, + } + assert metric_is_bytes("test_custom_memory", entry) def test_memory_metric_regression_uses_bytes(tmp_path, capsys: pytest.CaptureFixture[str]) -> None: From 717d1ea471ae7bc93e218f3d1d423e28d5dedc60 Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 25 Jun 2026 04:09:24 +0800 Subject: [PATCH 3/6] fix(test): harden benchmark gate and tracemalloc fixtures (PR review) --- Makefile | 19 +++++++---- benchmarks/README.md | 22 +++++++++---- benchmarks/baselines.json | 5 +-- scripts/check_benchmark_regression.py | 21 ++++++++++-- scripts/reduce_baselines.py | 26 +++++++++++---- tests/benchmarks/conftest.py | 15 ++++++--- tests/benchmarks/test_export_bench.py | 7 ++-- tests/benchmarks/test_parse_memory.py | 20 ++++-------- tests/test_check_benchmark_regression.py | 41 ++++++++++++++++++------ tests/test_reduce_baselines.py | 19 ++++++++++- 10 files changed, 139 insertions(+), 56 deletions(-) diff --git a/Makefile b/Makefile index b14f3d6..951775b 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,19 @@ -.PHONY: update-baselines check-benchmarks clean-benchmark-artifacts +.PHONY: seed-baselines-local update-baselines check-benchmarks clean-benchmark-artifacts -update-baselines: - pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts= - python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json +# WARNING: captures timings on THIS machine. Production baselines must match ubuntu-latest CI. +# Prefer downloading benchmark-results.json from a CI artifact, then: +# python scripts/reduce_baselines.py benchmark-results.json benchmarks/baselines.json --slack 1.5 +seed-baselines-local: + @echo "WARNING: seed-baselines-local uses this host's timings; CI gates on ubuntu-latest." >&2 + PYTHONPATH=. pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts= + PYTHONPATH=. python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json + +# Deprecated alias — kept for muscle memory; see seed-baselines-local warning above. +update-baselines: seed-baselines-local check-benchmarks: - pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmark-results.json -o addopts= - python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json + PYTHONPATH=. pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmark-results.json -o addopts= + PYTHONPATH=. python scripts/check_benchmark_regression.py benchmark-results.json benchmarks/baselines.json clean-benchmark-artifacts: rm -f benchmarks/_raw.json benchmark-results.json diff --git a/benchmarks/README.md b/benchmarks/README.md index b72bf80..6182b85 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -40,21 +40,31 @@ The `benchmarks` job on **ubuntu-latest** runs pytest-benchmark (`--benchmark-js **Gated:** parse medium/large + large peak memory; export 10/50/100 session latency + ZIP peak memory. -**Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. Benchmarks without a baseline entry print a warning and do not fail the gate. +**Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. These names are omitted from `baselines.json` when using `reduce_baselines.py`. Benchmarks without a baseline entry print a warning and do not fail the gate. + +Missing gated benchmarks (renamed or removed tests still listed in `baselines.json`) fail the gate. ## Refresh baselines -After intentional performance work, capture on **ubuntu-latest** (same OS as the gated CI job): +After intentional performance work, capture on **ubuntu-latest** (same OS as the gated CI job). Download `benchmark-results.json` from a CI artifact when possible: + +```bash +python scripts/reduce_baselines.py benchmark-results.json benchmarks/baselines.json --slack 1.5 +``` + +For a quick local snapshot only (may not match CI timings): ```bash -make update-baselines +make seed-baselines-local ``` +`make update-baselines` is a deprecated alias for `seed-baselines-local` and prints a warning. Do not commit baselines from macOS/Windows unless you accept cross-OS gate skew. + Or manually: ```bash -pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts= -python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json +PYTHONPATH=. pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts= +PYTHONPATH=. python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json ``` -Baselines must be captured on **ubuntu-latest** to match the gated CI runner. Cross-OS variance causes spurious failures. Download `benchmark-results.json` from a CI artifact to seed baselines if needed. +Baselines must be captured on **ubuntu-latest** to match the gated CI runner. Cross-OS variance causes spurious failures. diff --git a/benchmarks/baselines.json b/benchmarks/baselines.json index 2de7544..b25973e 100644 --- a/benchmarks/baselines.json +++ b/benchmarks/baselines.json @@ -1,5 +1,5 @@ { - "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR memory-path benchmarks). Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).", + "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR memory-path benchmarks). Values multiplied by 1.5× slack at seeding time. Excluded from gate (not in this file): test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).", "updated": "2026-06-25T12:00:00Z", "machine": "Linux", "groups": { @@ -15,9 +15,6 @@ "test_bulk_export_zip_peak_memory[sessions-10]": 350546.0, "test_bulk_export_zip_peak_memory[sessions-50]": 503331.0, "test_bulk_export_zip_peak_memory[sessions-100]": 686874.0 - }, - "search": { - "test_search_full_corpus": 0.001091 } } } diff --git a/scripts/check_benchmark_regression.py b/scripts/check_benchmark_regression.py index 0a94304..26726ad 100644 --- a/scripts/check_benchmark_regression.py +++ b/scripts/check_benchmark_regression.py @@ -39,8 +39,16 @@ def benchmark_entry_mean(entry: dict[str, object]) -> float: """Return gated metric: peak_bytes from extra_info when present, else stats.mean.""" if entry_uses_peak_bytes(entry): extra = entry["extra_info"] - assert isinstance(extra, dict) - return float(extra["peak_bytes"]) + if not isinstance(extra, dict): + raise BenchmarkDataError( + f"extra_info for {entry.get('name')!r} is not a dict" + ) + try: + return float(extra["peak_bytes"]) + except (KeyError, TypeError, ValueError) as exc: + raise BenchmarkDataError( + f"benchmark {entry.get('name')!r} missing 'stats.mean' or extra_info.peak_bytes" + ) from exc try: stats = entry["stats"] return float(stats["mean"]) # type: ignore[index] @@ -75,6 +83,8 @@ def load_results( try: name = entry["name"] mean = benchmark_entry_mean(entry) + except BenchmarkDataError: + raise except (KeyError, TypeError, ValueError) as exc: raise BenchmarkDataError( f"{path} benchmarks[{index}] missing 'name' or measurable value" @@ -132,12 +142,14 @@ def check_regression( baseline_means = load_baseline_means(baselines_path) failures: list[str] = [] + missing: list[str] = [] for name, base in baseline_means.items(): if name in EXCLUDED_FROM_GATE: continue cur = flat.get(name) if cur is None: - print(f"WARN: no current result for baseline {name!r}; skipping") + print(f"FAIL: no current result for gated baseline {name!r}") + missing.append(name) continue if base == 0: print(f"WARN: baseline for {name!r} is zero; skipping ratio check") @@ -160,6 +172,9 @@ def check_regression( if failures: print(f"\nREGRESSION: {len(failures)} benchmark(s) exceeded {threshold:.0%}") + if missing: + print(f"\nMISSING: {len(missing)} gated benchmark(s) absent from current results") + if failures or missing: return 1 return 0 diff --git a/scripts/reduce_baselines.py b/scripts/reduce_baselines.py index 23b12b7..e1570cf 100644 --- a/scripts/reduce_baselines.py +++ b/scripts/reduce_baselines.py @@ -8,10 +8,11 @@ from datetime import UTC, datetime from pathlib import Path -try: - from scripts.check_benchmark_regression import BenchmarkDataError, benchmark_entry_mean -except ModuleNotFoundError: - from check_benchmark_regression import BenchmarkDataError, benchmark_entry_mean +from scripts.check_benchmark_regression import ( + EXCLUDED_FROM_GATE, + BenchmarkDataError, + benchmark_entry_mean, +) GATED_GROUPS = ("parse", "export", "search") @@ -51,21 +52,32 @@ def reduce_baselines( try: name = entry["name"] mean = benchmark_entry_mean(entry) + except BenchmarkDataError: + raise except (KeyError, TypeError, ValueError) as exc: raise BenchmarkDataError( f"{path} benchmarks[{index}] missing 'name' or measurable value" ) from exc + bench_name = str(name) + if bench_name in EXCLUDED_FROM_GATE: + continue group = entry.get("group") if group not in GATED_GROUPS: continue - groups[group][str(name)] = mean * slack + groups[group][bench_name] = mean * slack + + # Drop empty groups (e.g. search when only excluded benchmarks ran). + groups = {name: values for name, values in groups.items() if values} + slack_note = f" Values multiplied by {slack}× slack at generation time." if slack != 1.0 else "" machine_info = raw.get("machine_info") machine = machine_info.get("system") if isinstance(machine_info, dict) else None output: dict[str, object] = { "_note": ( - "Gated means from ubuntu-latest CI benchmark-results.json. " - "Excluded from gate: test_parse_session_small, test_search_full_corpus (CI noise). " + "Gated means from ubuntu-latest CI benchmark-results.json." + f"{slack_note} " + "Excluded from gate (not written here): test_parse_session_small, " + "test_search_full_corpus (CI noise). " "Memory benchmarks use extra_info.peak_bytes (bytes); " "latency uses stats.mean (seconds)." ), diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 7a43b31..52683a2 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -23,6 +23,7 @@ class TracemallocPeak: """Measure peak Python heap bytes for one callable invocation.""" def measure(self, func: Callable[..., T], /, *args: Any, **kwargs: Any) -> tuple[T, int]: + was_tracing = tracemalloc.is_tracing() tracemalloc.start() tracemalloc.clear_traces() try: @@ -30,7 +31,8 @@ def measure(self, func: Callable[..., T], /, *args: Any, **kwargs: Any) -> tuple _, peak = tracemalloc.get_traced_memory() return result, peak finally: - tracemalloc.stop() + if not was_tracing: + tracemalloc.stop() @pytest.fixture @@ -38,13 +40,16 @@ def tracemalloc_peak() -> TracemallocPeak: return TracemallocPeak() -def write_jsonl(path: Path, line_count: int) -> Path: +def write_jsonl(path: Path, line_count: int, *, first_timestamp: str | None = None) -> Path: """Write a JSONL session file with *line_count* rows derived from the template fixture.""" template = json.loads(TEMPLATE_LINE) with path.open("w", encoding="utf-8") as f: for i in range(line_count): entry = deepcopy(template) - entry["timestamp"] = f"2026-06-12T10:{i % 60:02d}:00Z" + if i == 0 and first_timestamp is not None: + entry["timestamp"] = first_timestamp + else: + entry["timestamp"] = f"2026-06-12T10:{i % 60:02d}:00Z" if i % 3 == 1: msg = entry.setdefault("message", {}) if isinstance(msg, dict) and "content" in msg: @@ -96,7 +101,9 @@ def export_corpus(tmp_path: Path, request: pytest.FixtureRequest) -> Path: project = tmp_path / "bench-project" project.mkdir() for i in range(count): - write_jsonl(project / f"session_{i:04d}.jsonl", 20) + # Unique first_timestamp per session so export filenames do not collide in ZIP benches. + first_ts = f"2026-06-12T{i % 24:02d}:{i % 60:02d}:00Z" + write_jsonl(project / f"session_{i:04d}.jsonl", 20, first_timestamp=first_ts) return project diff --git a/tests/benchmarks/test_export_bench.py b/tests/benchmarks/test_export_bench.py index 3792ebf..39abcbb 100644 --- a/tests/benchmarks/test_export_bench.py +++ b/tests/benchmarks/test_export_bench.py @@ -8,7 +8,6 @@ import pytest -from tests.benchmarks.conftest import TracemallocPeak from utils.export_engine import BulkExportResult, NoopSink, ZipSink, run_bulk_export @@ -56,10 +55,11 @@ def _run() -> object: def test_bulk_export_zip_peak_memory( benchmark, export_corpus: Path, - tracemalloc_peak: TracemallocPeak, + tracemalloc_peak, ) -> None: projects = _bench_projects(export_corpus) peaks: list[int] = [] + results: list[BulkExportResult] = [] def _run() -> None: def _export() -> BulkExportResult: @@ -78,9 +78,10 @@ def _export() -> BulkExportResult: ) result, peak = tracemalloc_peak.measure(_export) - assert result.exported_session_count > 0 + results.append(result) peaks.append(peak) benchmark(_run) + assert results and results[-1].exported_session_count > 0 assert peaks, "benchmark produced no peak memory samples" benchmark.extra_info["peak_bytes"] = int(sum(peaks) / len(peaks)) diff --git a/tests/benchmarks/test_parse_memory.py b/tests/benchmarks/test_parse_memory.py index aa2bd9a..46738cd 100644 --- a/tests/benchmarks/test_parse_memory.py +++ b/tests/benchmarks/test_parse_memory.py @@ -2,16 +2,17 @@ from __future__ import annotations -import tracemalloc from pathlib import Path import pytest -from tests.benchmarks.conftest import TracemallocPeak from utils.jsonl_parser import parse_session -def test_large_parse_peak_memory_under_ceiling(parse_large_file: Path) -> None: +def test_large_parse_peak_memory_under_ceiling( + parse_large_file: Path, + tracemalloc_peak, +) -> None: path = parse_large_file file_bytes = path.stat().st_size # Issue #7 ceiling: Python heap peak (tracemalloc) vs on-disk JSONL size. Parsed @@ -19,15 +20,8 @@ def test_large_parse_peak_memory_under_ceiling(parse_large_file: Path) -> None: # a comment here if the parser legitimately grows. ceiling = file_bytes * 10 - tracemalloc.start() - tracemalloc.clear_traces() - try: - result = parse_session(str(path)) - assert len(result["messages"]) > 0, "parse_session returned no messages" - _, peak = tracemalloc.get_traced_memory() - finally: - tracemalloc.stop() - + result, peak = tracemalloc_peak.measure(parse_session, str(path)) + assert len(result["messages"]) > 0, "parse_session returned no messages" assert peak < ceiling, f"peak {peak} bytes exceeds 10x file size {file_bytes}" @@ -35,7 +29,7 @@ def test_large_parse_peak_memory_under_ceiling(parse_large_file: Path) -> None: def test_parse_large_peak_memory( benchmark, parse_large_file: Path, - tracemalloc_peak: TracemallocPeak, + tracemalloc_peak, ) -> None: path = str(parse_large_file) peaks: list[int] = [] diff --git a/tests/test_check_benchmark_regression.py b/tests/test_check_benchmark_regression.py index d35d9d6..891766f 100644 --- a/tests/test_check_benchmark_regression.py +++ b/tests/test_check_benchmark_regression.py @@ -39,12 +39,12 @@ def test_missing_baseline_warns_without_failing( results, [ {"name": "test_new_bench", "stats": {"mean": 0.01}}, - {"name": "test_parse_session_small", "stats": {"mean": 0.0001}}, + {"name": GATED_BENCH, "stats": {"mean": 0.002}}, ], ) _write_baselines( baselines, - {"parse": {"test_parse_session_small": 0.0001}}, + {"parse": {GATED_BENCH: 0.002}}, ) assert check_regression(results, baselines) == 0 @@ -152,9 +152,7 @@ def test_excluded_benchmark_in_baselines_is_not_gated( assert "REGRESSION" not in capsys.readouterr().out -def test_missing_current_result_warns_without_failing( - tmp_path, capsys: pytest.CaptureFixture[str] -) -> None: +def test_missing_current_result_fails(tmp_path, capsys: pytest.CaptureFixture[str]) -> None: results = tmp_path / "results.json" baselines = tmp_path / "baselines.json" _write_results(results, []) @@ -163,8 +161,10 @@ def test_missing_current_result_warns_without_failing( {"parse": {GATED_BENCH: 0.002}}, ) - assert check_regression(results, baselines) == 0 - assert "no current result for baseline" in capsys.readouterr().out + assert check_regression(results, baselines) == 1 + out = capsys.readouterr().out + assert "MISSING" in out + assert "no current result for gated baseline" in out def test_main_reports_benchmark_data_error(tmp_path, capsys: pytest.CaptureFixture[str]) -> None: @@ -199,11 +199,11 @@ def test_metric_is_bytes_uses_extra_info_without_name_hint() -> None: from scripts.check_benchmark_regression import metric_is_bytes entry = { - "name": "test_custom_memory", + "name": "test_export_latency", "stats": {"mean": 0.05}, "extra_info": {"peak_bytes": 1_000_000}, } - assert metric_is_bytes("test_custom_memory", entry) + assert metric_is_bytes("test_export_latency", entry) def test_memory_metric_regression_uses_bytes(tmp_path, capsys: pytest.CaptureFixture[str]) -> None: @@ -230,6 +230,29 @@ def test_memory_metric_regression_uses_bytes(tmp_path, capsys: pytest.CaptureFix assert "REGRESSION" in out +def test_benchmark_entry_mean_rejects_non_dict_extra_info() -> None: + from scripts.check_benchmark_regression import benchmark_entry_mean + + with pytest.raises(BenchmarkDataError, match="extra_info"): + benchmark_entry_mean( + { + "name": "test_parse_large_peak_memory", + "extra_info": "not-a-dict", + } + ) + + +def test_load_results_preserves_benchmark_data_error_message(tmp_path) -> None: + path = tmp_path / "results.json" + _write_results( + path, + [{"name": "test_parse_large_peak_memory", "extra_info": {"peak_bytes": "bad"}}], + ) + + with pytest.raises(BenchmarkDataError, match="extra_info.peak_bytes"): + load_results(path) + + def test_duplicate_baseline_name_raises(tmp_path) -> None: baselines = tmp_path / "baselines.json" _write_baselines( diff --git a/tests/test_reduce_baselines.py b/tests/test_reduce_baselines.py index 53d1ff1..40710bb 100644 --- a/tests/test_reduce_baselines.py +++ b/tests/test_reduce_baselines.py @@ -39,10 +39,27 @@ def test_reduce_baselines_writes_gated_groups_only(tmp_path) -> None: assert output["machine"] == "Linux" assert "test_parse_session_medium" in output["groups"]["parse"] - assert "test_parse_session_small" in output["groups"]["parse"] + assert "test_parse_session_small" not in output["groups"]["parse"] assert "cache" not in output["groups"] +def test_reduce_baselines_skips_excluded_from_gate(tmp_path) -> None: + raw = tmp_path / "raw.json" + out = tmp_path / "baselines.json" + _write_raw( + raw, + [ + {"group": "search", "name": "test_search_full_corpus", "stats": {"mean": 0.001}}, + {"group": "parse", "name": "test_parse_session_medium", "stats": {"mean": 0.002}}, + ], + ) + + output = reduce_baselines(raw, out) + + assert "search" not in output["groups"] + assert "test_search_full_corpus" not in json.loads(out.read_text(encoding="utf-8"))["groups"] + + def test_reduce_baselines_applies_slack(tmp_path) -> None: raw = tmp_path / "raw.json" out = tmp_path / "baselines.json" From 0f5e1ab9b6f1c5bb96a884cf5481ed0e5555a0da Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 25 Jun 2026 04:14:43 +0800 Subject: [PATCH 4/6] fix(ci): ruff format check_benchmark_regression.py --- scripts/check_benchmark_regression.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/check_benchmark_regression.py b/scripts/check_benchmark_regression.py index 26726ad..3a27dab 100644 --- a/scripts/check_benchmark_regression.py +++ b/scripts/check_benchmark_regression.py @@ -40,9 +40,7 @@ def benchmark_entry_mean(entry: dict[str, object]) -> float: if entry_uses_peak_bytes(entry): extra = entry["extra_info"] if not isinstance(extra, dict): - raise BenchmarkDataError( - f"extra_info for {entry.get('name')!r} is not a dict" - ) + raise BenchmarkDataError(f"extra_info for {entry.get('name')!r} is not a dict") try: return float(extra["peak_bytes"]) except (KeyError, TypeError, ValueError) as exc: From 2fb39bc0841f568c4e4f6b872f7e02a5303d938e Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 25 Jun 2026 04:58:57 +0800 Subject: [PATCH 5/6] =?UTF-8?q?tests/benchmarks/conftest.py=20=20=20=20=20?= =?UTF-8?q?for=20i=20in=20range(count):=20=20=20=20=20=20=20=20=20write=5F?= =?UTF-8?q?jsonl(project=20/=20f"session=5F{i:04d}.jsonl",=2020)=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20#=20Unique=20first=5Ftimestamp=20per=20sessio?= =?UTF-8?q?n=20so=20export=20filenames=20do=20not=20collide=20in=20ZIP=20b?= =?UTF-8?q?enches.=20=20=20=20=20=20=20=20=20first=5Fts=20=3D=20f"2026-06-?= =?UTF-8?q?12T{i=20%=2024:02d}:{i=20%=2060:02d}:00Z"=20@timon0305=20timon0?= =?UTF-8?q?305=201=20minute=20ago=20Member=20first=5Fts=20=3D=20f"2026-06-?= =?UTF-8?q?12T{i=20%=2024:02d}:{i=20%=2060:02d}:00Z"=20is=20only=20unique?= =?UTF-8?q?=20for=20count=20=E2=89=A4=20120=20(collides=20at=20lcm(24,60))?= =?UTF-8?q?.=20Add=20assert=20count=20<=3D=20120=20(or=20a=20wider=20forma?= =?UTF-8?q?t)=20so=20a=20future=20corpus=20bump=20can't=20produce=20collid?= =?UTF-8?q?ing=20ZIP=20entries.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + benchmarks/README.md | 2 +- benchmarks/baselines.json | 24 ++++++++++++++---------- scripts/reduce_baselines.py | 10 ++-------- tests/benchmarks/conftest.py | 2 +- tests/test_reduce_baselines.py | 9 +++++---- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 27a84ea..10bcf41 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ coverage/ coverage.xml benchmark-results.json benchmarks/_raw.json +benchmarks/_ci/ diff --git a/benchmarks/README.md b/benchmarks/README.md index 6182b85..9d3bc74 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -40,7 +40,7 @@ The `benchmarks` job on **ubuntu-latest** runs pytest-benchmark (`--benchmark-js **Gated:** parse medium/large + large peak memory; export 10/50/100 session latency + ZIP peak memory. -**Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. These names are omitted from `baselines.json` when using `reduce_baselines.py`. Benchmarks without a baseline entry print a warning and do not fail the gate. +**Not gated (informational only):** `test_parse_session_small`, `test_search_full_corpus` (sub-ms CI noise), and the `cache` group. These may appear in `baselines.json` for reference but are skipped by `check_benchmark_regression.py`. Benchmarks without a baseline entry print a warning and do not fail the gate. Missing gated benchmarks (renamed or removed tests still listed in `baselines.json`) fail the gate. diff --git a/benchmarks/baselines.json b/benchmarks/baselines.json index b25973e..07fb84e 100644 --- a/benchmarks/baselines.json +++ b/benchmarks/baselines.json @@ -1,20 +1,24 @@ { - "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR memory-path benchmarks). Values multiplied by 1.5× slack at seeding time. Excluded from gate (not in this file): test_parse_session_small, test_search_full_corpus (CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).", - "updated": "2026-06-25T12:00:00Z", + "_note": "Gated means from ubuntu-latest CI benchmark-results.json (PR #97, run 28126772276). Excluded from gate (recorded for reference): test_parse_session_small, test_search_full_corpus (sub-ms CI noise). Memory benchmarks use extra_info.peak_bytes (bytes); latency uses stats.mean (seconds).", + "updated": "2026-06-24T20:15:37Z", "machine": "Linux", "groups": { "parse": { - "test_parse_session_medium": 0.003037, - "test_parse_session_large": 0.030266, + "test_parse_session_small": 0.00010518068718225604, + "test_parse_session_medium": 0.002991333112179635, + "test_parse_session_large": 0.032311203818181436, "test_parse_large_peak_memory": 2032028.0 }, "export": { - "test_bulk_export_session_count[sessions-10]": 0.004253, - "test_bulk_export_session_count[sessions-50]": 0.021039, - "test_bulk_export_session_count[sessions-100]": 0.041709, - "test_bulk_export_zip_peak_memory[sessions-10]": 350546.0, - "test_bulk_export_zip_peak_memory[sessions-50]": 503331.0, - "test_bulk_export_zip_peak_memory[sessions-100]": 686874.0 + "test_bulk_export_session_count[sessions-10]": 0.0042825538530803925, + "test_bulk_export_session_count[sessions-50]": 0.021406330209302382, + "test_bulk_export_session_count[sessions-100]": 0.04229194749999898, + "test_bulk_export_zip_peak_memory[sessions-10]": 350628.0, + "test_bulk_export_zip_peak_memory[sessions-50]": 506454.0, + "test_bulk_export_zip_peak_memory[sessions-100]": 694088.0 + }, + "search": { + "test_search_full_corpus": 0.0011120838654706596 } } } diff --git a/scripts/reduce_baselines.py b/scripts/reduce_baselines.py index e1570cf..a3b4114 100644 --- a/scripts/reduce_baselines.py +++ b/scripts/reduce_baselines.py @@ -9,7 +9,6 @@ from pathlib import Path from scripts.check_benchmark_regression import ( - EXCLUDED_FROM_GATE, BenchmarkDataError, benchmark_entry_mean, ) @@ -59,16 +58,11 @@ def reduce_baselines( f"{path} benchmarks[{index}] missing 'name' or measurable value" ) from exc bench_name = str(name) - if bench_name in EXCLUDED_FROM_GATE: - continue group = entry.get("group") if group not in GATED_GROUPS: continue groups[group][bench_name] = mean * slack - # Drop empty groups (e.g. search when only excluded benchmarks ran). - groups = {name: values for name, values in groups.items() if values} - slack_note = f" Values multiplied by {slack}× slack at generation time." if slack != 1.0 else "" machine_info = raw.get("machine_info") machine = machine_info.get("system") if isinstance(machine_info, dict) else None @@ -76,8 +70,8 @@ def reduce_baselines( "_note": ( "Gated means from ubuntu-latest CI benchmark-results.json." f"{slack_note} " - "Excluded from gate (not written here): test_parse_session_small, " - "test_search_full_corpus (CI noise). " + "Excluded from gate (recorded for reference): test_parse_session_small, " + "test_search_full_corpus (sub-ms CI noise). " "Memory benchmarks use extra_info.peak_bytes (bytes); " "latency uses stats.mean (seconds)." ), diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 52683a2..353628a 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -102,7 +102,7 @@ def export_corpus(tmp_path: Path, request: pytest.FixtureRequest) -> Path: project.mkdir() for i in range(count): # Unique first_timestamp per session so export filenames do not collide in ZIP benches. - first_ts = f"2026-06-12T{i % 24:02d}:{i % 60:02d}:00Z" + first_ts = f"2026-06-12T{i // 60:02d}:{i % 60:02d}:00Z" write_jsonl(project / f"session_{i:04d}.jsonl", 20, first_timestamp=first_ts) return project diff --git a/tests/test_reduce_baselines.py b/tests/test_reduce_baselines.py index 40710bb..bad0a56 100644 --- a/tests/test_reduce_baselines.py +++ b/tests/test_reduce_baselines.py @@ -38,12 +38,14 @@ def test_reduce_baselines_writes_gated_groups_only(tmp_path) -> None: output = reduce_baselines(raw, out) assert output["machine"] == "Linux" + assert set(output["groups"].keys()) == {"parse", "export", "search"} assert "test_parse_session_medium" in output["groups"]["parse"] - assert "test_parse_session_small" not in output["groups"]["parse"] + assert "test_parse_session_small" in output["groups"]["parse"] + assert output["groups"]["search"] == {} assert "cache" not in output["groups"] -def test_reduce_baselines_skips_excluded_from_gate(tmp_path) -> None: +def test_reduce_baselines_includes_search_benchmark(tmp_path) -> None: raw = tmp_path / "raw.json" out = tmp_path / "baselines.json" _write_raw( @@ -56,8 +58,7 @@ def test_reduce_baselines_skips_excluded_from_gate(tmp_path) -> None: output = reduce_baselines(raw, out) - assert "search" not in output["groups"] - assert "test_search_full_corpus" not in json.loads(out.read_text(encoding="utf-8"))["groups"] + assert output["groups"]["search"]["test_search_full_corpus"] == pytest.approx(0.001) def test_reduce_baselines_applies_slack(tmp_path) -> None: From 1c6666608280858017a0d5c1c1dea608c3ed2c56 Mon Sep 17 00:00:00 2001 From: chen Date: Thu, 25 Jun 2026 05:11:36 +0800 Subject: [PATCH 6/6] fix(test): align baseline refresh slack and export timestamp rollover --- Makefile | 2 +- benchmarks/README.md | 2 +- tests/benchmarks/conftest.py | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 951775b..d3d6d9b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ seed-baselines-local: @echo "WARNING: seed-baselines-local uses this host's timings; CI gates on ubuntu-latest." >&2 PYTHONPATH=. pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts= - PYTHONPATH=. python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json + PYTHONPATH=. python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json --slack 1.5 # Deprecated alias — kept for muscle memory; see seed-baselines-local warning above. update-baselines: seed-baselines-local diff --git a/benchmarks/README.md b/benchmarks/README.md index 9d3bc74..5ef54c2 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -64,7 +64,7 @@ Or manually: ```bash PYTHONPATH=. pytest tests/benchmarks/ --benchmark-only --benchmark-json=benchmarks/_raw.json -o addopts= -PYTHONPATH=. python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json +PYTHONPATH=. python scripts/reduce_baselines.py benchmarks/_raw.json benchmarks/baselines.json --slack 1.5 ``` Baselines must be captured on **ubuntu-latest** to match the gated CI runner. Cross-OS variance causes spurious failures. diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py index 353628a..2bc4876 100644 --- a/tests/benchmarks/conftest.py +++ b/tests/benchmarks/conftest.py @@ -6,6 +6,7 @@ import tracemalloc from collections.abc import Callable from copy import deepcopy +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any, TypeVar @@ -18,6 +19,13 @@ T = TypeVar("T") +_EXPORT_SESSION_BASE = datetime(2026, 6, 12, 0, 0, tzinfo=UTC) + + +def export_session_first_timestamp(index: int) -> str: + """Return a unique, valid ISO timestamp for export-corpus session *index*.""" + return (_EXPORT_SESSION_BASE + timedelta(minutes=index)).strftime("%Y-%m-%dT%H:%M:%SZ") + class TracemallocPeak: """Measure peak Python heap bytes for one callable invocation.""" @@ -102,7 +110,7 @@ def export_corpus(tmp_path: Path, request: pytest.FixtureRequest) -> Path: project.mkdir() for i in range(count): # Unique first_timestamp per session so export filenames do not collide in ZIP benches. - first_ts = f"2026-06-12T{i // 60:02d}:{i % 60:02d}:00Z" + first_ts = export_session_first_timestamp(i) write_jsonl(project / f"session_{i:04d}.jsonl", 20, first_timestamp=first_ts) return project