diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 677bc034..71d3683e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,6 +20,7 @@ repos: - id: check-toml - id: check-added-large-files args: [--maxkb=5000] # parity fixture parquets can be larger than default 500KB + exclude: ^tests/fixtures/parity/pinned_cache\.tar\.xz$ # 5.1MB pinned raw-source bundle (deliberate, documented in fixtures/parity/README.md) - repo: local hooks: diff --git a/tests/fixtures/parity/README.md b/tests/fixtures/parity/README.md index 718fa0b6..c36997a3 100644 --- a/tests/fixtures/parity/README.md +++ b/tests/fixtures/parity/README.md @@ -8,6 +8,102 @@ > Do **NOT** regenerate these files in CI. They are settlement-grade > reference data; the bytes ARE the Day 3 hard-gate ground truth. +## Pinned-input redesign (2026-07-15, operator-directed — release 1.17.0 gate) + +The parity gate now runs in TWO variants (see `tests/test_parity.py`): + +- **`test_parity_case_pinned` — the HARD gate.** Runs `research()` fully + OFFLINE (socket-level network block) and **CACHE-COLD** over + `pinned_cache.tar.xz` — a committed bundle of the RAW `v1/sources/` + fetch text (IEM CSV / GHCNh PSV / CLI JSON; 82 MB raw → 5.1 MB xz), + captured live on **2026-07-15** by `capture_pinned_cache.py` from main + @ `02580c3`. Cold replay forces the production parsers, + `merge_observations`/`merge_climate` SOURCE_PRIORITY logic, aggregation, + and the join to execute on every gate run (codex PR-124 CRITICAL: + pinning the derived parquet tiers instead would let the warm-cache path + skip all of them). Deterministic forever; lives in the default fast + suite (every CI push), not behind `-m live`. All-5-case replay ≈ 7 s. +- **`test_parity_case_live` — the upstream-drift monitor** (`-m live`, + pre-publish). A failure here while the pinned gate is green means the + mutable IEM/AWC archive drifted since the pin — audit that the diff is + confined to count-consistent `obs_*` sum-aggregates (`cli_*` + + `obs_high_f`/`obs_low_f` MUST stay clean), then deliberately re-pin via + `capture_pinned_cache.py` and extend the audit table below. + +**Validation of the approach:** replaying the pinned inputs through current +code reproduces the original v0.14.1 fixtures for cases 1/2/5 EXACTLY +(within the standing ULP `atol`) — the code is byte-faithful to 0.14.1 +wherever the archive did not move. + +### Drifted-cell re-base audit (2026-07-15) + +Between the May-2026 capture and the 2026-07-15 pin, the upstream archive +restated 1 KLAX day + 17 KMIA days (late/corrected METARs — every diff is +count-consistent). The 55 cells below were re-based onto the pinned-archive +values; **every other cell in every fixture remains original 0.14.1 +bytes.** Settlement-critical columns (`cli_high_f`/`cli_low_f`/ +`cli_report_type`/`obs_high_f`/`obs_low_f`) were untouched on all cases. +The pre-patch parquets are preserved in git history (last at the +`v1.17.0`-candidate merge `02580c3`). + +| case | date | column | v0.14.1 fixture value | pinned-archive value | +|------|------|--------|-----------------------|----------------------| +| case-3 | 2025-03-12 | obs_mean_f | 56.739999999999995 | 56.782641509433965 | +| case-3 | 2025-03-12 | obs_mean_dewpoint_f | 52.61076923076923 | 52.61924528301887 | +| case-3 | 2025-03-12 | obs_count | 52 | 53 | +| case-4 | 2025-01-31 | obs_mean_f | 73.7235294117647 | 73.81714285714285 | +| case-4 | 2025-02-07 | obs_mean_f | 74.67870967741935 | 74.846875 | +| case-4 | 2025-02-09 | obs_mean_f | 75.6174193548387 | 75.784375 | +| case-4 | 2025-02-10 | obs_mean_f | 75.41085714285714 | 75.54 | +| case-4 | 2025-02-11 | obs_mean_f | 75.39696969696969 | 75.50235294117647 | +| case-4 | 2025-02-20 | obs_mean_f | 71.244 | 71.3488888888889 | +| case-4 | 2025-03-12 | obs_mean_f | 69.67548387096774 | 69.938125 | +| case-4 | 2025-05-12 | obs_mean_f | 75.60444444444444 | 75.72086956521738 | +| case-4 | 2025-06-02 | obs_mean_f | 77.77771428571428 | 77.64611111111111 | +| case-4 | 2025-06-11 | obs_mean_f | 83.93935483870968 | 84.00424242424242 | +| case-4 | 2025-06-16 | obs_mean_f | 84.48451612903226 | 84.565625 | +| case-4 | 2025-06-21 | obs_mean_f | 84.06484848484848 | 84.15352941176471 | +| case-4 | 2025-09-11 | obs_mean_f | 80.73441860465117 | 80.78454545454545 | +| case-4 | 2025-10-19 | obs_mean_f | 80.77483870967743 | 80.8425 | +| case-4 | 2025-10-31 | obs_mean_f | 69.70967741935483 | 69.841875 | +| case-4 | 2025-11-07 | obs_mean_f | 79.54709677419355 | 79.653125 | +| case-4 | 2025-11-08 | obs_mean_f | 80.47625 | 80.58363636363637 | +| case-4 | 2025-01-31 | obs_mean_dewpoint_f | 66.08764705882353 | 66.08571428571429 | +| case-4 | 2025-02-07 | obs_mean_dewpoint_f | 65.6116129032258 | 65.5625 | +| case-4 | 2025-02-09 | obs_mean_dewpoint_f | 68.03225806451613 | 67.969375 | +| case-4 | 2025-02-10 | obs_mean_dewpoint_f | 67.57771428571428 | 67.55944444444444 | +| case-4 | 2025-02-11 | obs_mean_dewpoint_f | 66.38545454545454 | 66.28470588235294 | +| case-4 | 2025-02-20 | obs_mean_dewpoint_f | 63.15657142857143 | 63.07111111111111 | +| case-4 | 2025-03-12 | obs_mean_dewpoint_f | 53.189677419354844 | 53.2475 | +| case-4 | 2025-05-12 | obs_mean_dewpoint_f | 72.41555555555556 | 72.40565217391304 | +| case-4 | 2025-06-02 | obs_mean_dewpoint_f | 73.2782857142857 | 73.21666666666667 | +| case-4 | 2025-06-11 | obs_mean_dewpoint_f | 73.61419354838709 | 73.60666666666667 | +| case-4 | 2025-06-16 | obs_mean_dewpoint_f | 71.54645161290323 | 71.593125 | +| case-4 | 2025-06-21 | obs_mean_dewpoint_f | 72.7248484848485 | 72.67588235294117 | +| case-4 | 2025-09-11 | obs_mean_dewpoint_f | 75.52139534883722 | 75.465 | +| case-4 | 2025-10-19 | obs_mean_dewpoint_f | 72.29290322580646 | 72.31625 | +| case-4 | 2025-10-31 | obs_mean_dewpoint_f | 57.097419354838706 | 57.095 | +| case-4 | 2025-11-07 | obs_mean_dewpoint_f | 68.94193548387096 | 68.9125 | +| case-4 | 2025-11-08 | obs_mean_dewpoint_f | 72.12125 | 72.05636363636364 | +| case-4 | 2025-09-11 | obs_total_precip_in | 2.08 | 2.1784251968503936 | +| case-4 | 2025-01-31 | obs_count | 34 | 35 | +| case-4 | 2025-02-07 | obs_count | 31 | 32 | +| case-4 | 2025-02-09 | obs_count | 31 | 32 | +| case-4 | 2025-02-10 | obs_count | 35 | 36 | +| case-4 | 2025-02-11 | obs_count | 33 | 34 | +| case-4 | 2025-02-20 | obs_count | 35 | 36 | +| case-4 | 2025-03-12 | obs_count | 31 | 32 | +| case-4 | 2025-05-12 | obs_count | 45 | 46 | +| case-4 | 2025-06-02 | obs_count | 35 | 36 | +| case-4 | 2025-06-11 | obs_count | 31 | 33 | +| case-4 | 2025-06-16 | obs_count | 31 | 32 | +| case-4 | 2025-06-21 | obs_count | 33 | 34 | +| case-4 | 2025-09-11 | obs_count | 43 | 44 | +| case-4 | 2025-10-19 | obs_count | 31 | 32 | +| case-4 | 2025-10-31 | obs_count | 31 | 32 | +| case-4 | 2025-11-07 | obs_count | 31 | 32 | +| case-4 | 2025-11-08 | obs_count | 32 | 33 | + ## Case-4 re-baseline (2026-07-09, operator-approved — release 1.13.0 gate) `case_4_KMIA_2024-12-01_2025-11-30.parquet` (and its derived diff --git a/tests/fixtures/parity/capture_pinned_cache.py b/tests/fixtures/parity/capture_pinned_cache.py new file mode 100644 index 00000000..23b9d40b --- /dev/null +++ b/tests/fixtures/parity/capture_pinned_cache.py @@ -0,0 +1,87 @@ +"""Capture the PINNED-INPUT cache bundle for the parity gate. + +Fetches the 5 parity windows live via ``mostlyright.research()`` into a +staging ``MOSTLYRIGHT_CACHE_DIR``, strips the cache to the RAW source tier +(``v1/sources/`` — the fetched IEM CSV / GHCNh PSV / CLI JSON text), and +writes it compressed to ``tests/fixtures/parity/pinned_cache.tar.xz``. + +RAW sources, deliberately NOT the derived parquet tiers (codex PR-124 +CRITICAL): the offline gate replays CACHE-COLD from the raw text, so the +production parsers (``_fetch_iem_month``/GHCNh/CLI), ``merge_observations`` +/ ``merge_climate`` SOURCE_PRIORITY logic, aggregation, and the join all +execute under the network block on every run — a parser or merge +regression fails the gate. Pinning derived parquets instead would let the +warm-cache path skip all of that. (82 MB raw → 5.1 MB xz; cold replay of +all 5 cases ≈ 5 s.) + +Why this exists (2026-07-15): the live parity gate compared "today's code + +today's IEM/AWC archive" against fixtures frozen from ``mostlyright==0.14.1`` +in May 2026. The upstream archive is MUTABLE — late/corrected METARs shift +the sum-then-divide ``obs_*`` aggregates (KLAX 2025-03, KMIA 2024-12..2025-11 +drifted) — and the 0.14.1 oracle can never be re-captured +(api.mostlyright.md was re-pointed in Phase 28). Pinning the INPUTS converts +the gate into a deterministic test of OUR aggregation/join code against the +0.14.1 outputs, which is what it was always meant to prove. + +Run (live network, ~50 min — KMIA is a year window):: + + uv run python tests/fixtures/parity/capture_pinned_cache.py + +Re-running OVERWRITES the pinned bundle from today's archive — only do this +deliberately (it re-bases the input ground truth) and re-run the pinned gate +plus the drifted-cell audit in README.md afterwards. +""" + +from __future__ import annotations + +import os +import shutil +import sys +import tarfile +import tempfile +import time +from pathlib import Path + +FIXTURES = Path(__file__).resolve().parent +PINNED_TARBALL = FIXTURES / "pinned_cache.tar.xz" + +CASES: list[tuple[int, str, str, str]] = [ + (1, "KNYC", "2025-01-06", "2025-01-12"), + (2, "KMDW", "2025-04-01", "2025-04-30"), + (3, "KLAX", "2025-03-01", "2025-03-31"), + (4, "KMIA", "2024-12-01", "2025-11-30"), + (5, "KMSY", "2024-09-08", "2024-09-22"), +] + +#: The RAW source tier the cache-cold offline replay parses. Derived tiers +#: (``v1/observations``/``v1/climate``) and ``provenance/`` are deliberately +#: excluded — they are rebuilt from raw text at test time so the production +#: parse+merge+aggregate code is exercised on every gate run. +_KEEP_TIER = "v1/sources" + + +def main() -> int: + staging = Path(tempfile.mkdtemp(prefix="mr-parity-pin-")) + os.environ["MOSTLYRIGHT_CACHE_DIR"] = str(staging) + import mostlyright + + for num, station, frm, to in CASES: + t0 = time.time() + df = mostlyright.research(station, frm, to, label="cli") + print(f"case-{num} {station} {frm}..{to}: {len(df)} rows in {time.time() - t0:.0f}s") + + src = staging / _KEEP_TIER + if not src.is_dir(): + raise RuntimeError(f"expected raw-source cache tier missing: {src}") + files = [f for f in sorted(src.rglob("*")) if f.is_file()] + with tarfile.open(PINNED_TARBALL, "w:xz") as tar: + for f in files: + tar.add(f, arcname=str(f.relative_to(staging))) + size_mb = PINNED_TARBALL.stat().st_size / 1e6 + print(f"pinned {len(files)} raw-source files -> {PINNED_TARBALL} ({size_mb:.1f} MB)") + shutil.rmtree(staging) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/parity/case_3_KLAX_2025-03-01_2025-03-31.parquet b/tests/fixtures/parity/case_3_KLAX_2025-03-01_2025-03-31.parquet index 1930da37..05ef1799 100644 Binary files a/tests/fixtures/parity/case_3_KLAX_2025-03-01_2025-03-31.parquet and b/tests/fixtures/parity/case_3_KLAX_2025-03-01_2025-03-31.parquet differ diff --git a/tests/fixtures/parity/case_4_KMIA_2024-12-01_2025-11-30.parquet b/tests/fixtures/parity/case_4_KMIA_2024-12-01_2025-11-30.parquet index 4d940421..5754903e 100644 Binary files a/tests/fixtures/parity/case_4_KMIA_2024-12-01_2025-11-30.parquet and b/tests/fixtures/parity/case_4_KMIA_2024-12-01_2025-11-30.parquet differ diff --git a/tests/fixtures/parity/pinned_cache.tar.xz b/tests/fixtures/parity/pinned_cache.tar.xz new file mode 100644 index 00000000..80f74978 Binary files /dev/null and b/tests/fixtures/parity/pinned_cache.tar.xz differ diff --git a/tests/fixtures/parity/ts/case_3_KLAX_2025-03-01_2025-03-31.json b/tests/fixtures/parity/ts/case_3_KLAX_2025-03-01_2025-03-31.json index bd27652d..51cc64ae 100644 --- a/tests/fixtures/parity/ts/case_3_KLAX_2025-03-01_2025-03-31.json +++ b/tests/fixtures/parity/ts/case_3_KLAX_2025-03-01_2025-03-31.json @@ -253,13 +253,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-03-13T00:30:00Z", - "obs_count": 52, + "obs_count": 53, "obs_high_f": 60.08, "obs_low_f": 53.0, "obs_max_gust_kt": 31.0, "obs_max_wind_kt": 17, - "obs_mean_dewpoint_f": 52.61076923076923, - "obs_mean_f": 56.739999999999995, + "obs_mean_dewpoint_f": 52.61924528301887, + "obs_mean_f": 56.782641509433965, "obs_total_precip_in": 0.15000000000000002, "station": "LAX" }, diff --git a/tests/fixtures/parity/ts/case_4_KMIA_2024-12-01_2025-11-30.json b/tests/fixtures/parity/ts/case_4_KMIA_2024-12-01_2025-11-30.json index 06bd8029..652e53b4 100644 --- a/tests/fixtures/parity/ts/case_4_KMIA_2024-12-01_2025-11-30.json +++ b/tests/fixtures/parity/ts/case_4_KMIA_2024-12-01_2025-11-30.json @@ -1353,13 +1353,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-01-31T21:30:00Z", - "obs_count": 34, + "obs_count": 35, "obs_high_f": 78.08, "obs_low_f": 69.0, "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 15, - "obs_mean_dewpoint_f": 66.08764705882353, - "obs_mean_f": 73.7235294117647, + "obs_mean_dewpoint_f": 66.08571428571429, + "obs_mean_f": 73.81714285714285, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -1507,13 +1507,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-02-07T21:30:00Z", - "obs_count": 31, + "obs_count": 32, "obs_high_f": 81.0, "obs_low_f": 68.0, "obs_max_gust_kt": 16.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 65.6116129032258, - "obs_mean_f": 74.67870967741935, + "obs_mean_dewpoint_f": 65.5625, + "obs_mean_f": 74.846875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -1551,13 +1551,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-02-09T21:30:00Z", - "obs_count": 31, + "obs_count": 32, "obs_high_f": 82.04, "obs_low_f": 71.0, "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 68.03225806451613, - "obs_mean_f": 75.6174193548387, + "obs_mean_dewpoint_f": 67.969375, + "obs_mean_f": 75.784375, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -1573,13 +1573,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-02-10T21:30:00Z", - "obs_count": 35, + "obs_count": 36, "obs_high_f": 82.04, "obs_low_f": 68.0, "obs_max_gust_kt": 17.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 67.57771428571428, - "obs_mean_f": 75.41085714285714, + "obs_mean_dewpoint_f": 67.55944444444444, + "obs_mean_f": 75.54, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -1595,13 +1595,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-02-11T21:30:00Z", - "obs_count": 33, + "obs_count": 34, "obs_high_f": 81.0, "obs_low_f": 71.96, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 66.38545454545454, - "obs_mean_f": 75.39696969696969, + "obs_mean_dewpoint_f": 66.28470588235294, + "obs_mean_f": 75.50235294117647, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -1793,13 +1793,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-02-20T21:30:00Z", - "obs_count": 35, + "obs_count": 36, "obs_high_f": 76.0, "obs_low_f": 63.0, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 63.15657142857143, - "obs_mean_f": 71.244, + "obs_mean_dewpoint_f": 63.07111111111111, + "obs_mean_f": 71.3488888888889, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2233,13 +2233,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-03-12T21:30:00Z", - "obs_count": 31, + "obs_count": 32, "obs_high_f": 80.06, "obs_low_f": 60.980000000000004, "obs_max_gust_kt": 16.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 53.189677419354844, - "obs_mean_f": 69.67548387096774, + "obs_mean_dewpoint_f": 53.2475, + "obs_mean_f": 69.938125, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3575,13 +3575,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-05-12T21:30:00Z", - "obs_count": 45, + "obs_count": 46, "obs_high_f": 84.0, "obs_low_f": 71.0, "obs_max_gust_kt": 31.0, "obs_max_wind_kt": 21, - "obs_mean_dewpoint_f": 72.41555555555556, - "obs_mean_f": 75.60444444444444, + "obs_mean_dewpoint_f": 72.40565217391304, + "obs_mean_f": 75.72086956521738, "obs_total_precip_in": 9.68, "station": "MIA" }, @@ -4037,13 +4037,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-06-02T21:30:00Z", - "obs_count": 35, + "obs_count": 36, "obs_high_f": 89.0, "obs_low_f": 72.0, "obs_max_gust_kt": 43.0, "obs_max_wind_kt": 15, - "obs_mean_dewpoint_f": 73.2782857142857, - "obs_mean_f": 77.77771428571428, + "obs_mean_dewpoint_f": 73.21666666666667, + "obs_mean_f": 77.64611111111111, "obs_total_precip_in": 6.92, "station": "MIA" }, @@ -4235,13 +4235,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-06-11T21:30:00Z", - "obs_count": 31, + "obs_count": 33, "obs_high_f": 89.06, "obs_low_f": 80.0, "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 73.61419354838709, - "obs_mean_f": 83.93935483870968, + "obs_mean_dewpoint_f": 73.60666666666667, + "obs_mean_f": 84.00424242424242, "obs_total_precip_in": 0.4, "station": "MIA" }, @@ -4345,13 +4345,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-06-16T21:30:00Z", - "obs_count": 31, + "obs_count": 32, "obs_high_f": 89.06, "obs_low_f": 78.0, "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 71.54645161290323, - "obs_mean_f": 84.48451612903226, + "obs_mean_dewpoint_f": 71.593125, + "obs_mean_f": 84.565625, "obs_total_precip_in": 0.12, "station": "MIA" }, @@ -4455,13 +4455,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-06-21T21:30:00Z", - "obs_count": 33, + "obs_count": 34, "obs_high_f": 87.08000000000001, "obs_low_f": 79.0, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 72.7248484848485, - "obs_mean_f": 84.06484848484848, + "obs_mean_dewpoint_f": 72.67588235294117, + "obs_mean_f": 84.15352941176471, "obs_total_precip_in": 0.11, "station": "MIA" }, @@ -6259,14 +6259,14 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-09-11T21:30:00Z", - "obs_count": 43, + "obs_count": 44, "obs_high_f": 92.0, "obs_low_f": 76.0, "obs_max_gust_kt": 24.0, "obs_max_wind_kt": 16, - "obs_mean_dewpoint_f": 75.52139534883722, - "obs_mean_f": 80.73441860465117, - "obs_total_precip_in": 2.08, + "obs_mean_dewpoint_f": 75.465, + "obs_mean_f": 80.78454545454545, + "obs_total_precip_in": 2.1784251968503936, "station": "MIA" }, { @@ -7095,13 +7095,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-10-19T21:30:00Z", - "obs_count": 31, + "obs_count": 32, "obs_high_f": 85.0, "obs_low_f": 77.0, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 15, - "obs_mean_dewpoint_f": 72.29290322580646, - "obs_mean_f": 80.77483870967743, + "obs_mean_dewpoint_f": 72.31625, + "obs_mean_f": 80.8425, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7359,13 +7359,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-10-31T21:30:00Z", - "obs_count": 31, + "obs_count": 32, "obs_high_f": 76.0, "obs_low_f": 64.94, "obs_max_gust_kt": 15.0, "obs_max_wind_kt": 8, - "obs_mean_dewpoint_f": 57.097419354838706, - "obs_mean_f": 69.70967741935483, + "obs_mean_dewpoint_f": 57.095, + "obs_mean_f": 69.841875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7513,13 +7513,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-11-07T21:30:00Z", - "obs_count": 31, + "obs_count": 32, "obs_high_f": 87.0, "obs_low_f": 75.0, "obs_max_gust_kt": null, "obs_max_wind_kt": 8, - "obs_mean_dewpoint_f": 68.94193548387096, - "obs_mean_f": 79.54709677419355, + "obs_mean_dewpoint_f": 68.9125, + "obs_mean_f": 79.653125, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7535,13 +7535,13 @@ "fcst_pop_6hr_pct": null, "fcst_qpf_6hr_in": null, "market_close_utc": "2025-11-08T21:30:00Z", - "obs_count": 32, + "obs_count": 33, "obs_high_f": 86.0, "obs_low_f": 75.92, "obs_max_gust_kt": 16.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 72.12125, - "obs_mean_f": 80.47625, + "obs_mean_dewpoint_f": 72.05636363636364, + "obs_mean_f": 80.58363636363637, "obs_total_precip_in": 0.0, "station": "MIA" }, diff --git a/tests/fixtures/parity/ts/manifest.json b/tests/fixtures/parity/ts/manifest.json index 1855a1bb..5f512242 100644 --- a/tests/fixtures/parity/ts/manifest.json +++ b/tests/fixtures/parity/ts/manifest.json @@ -82,7 +82,7 @@ }, "from": "2025-03-01", "row_count": 31, - "sha256": "126e868501355c0fc861c8fc77ae16365c8367398bb87f6b3c2a5826c7880078", + "sha256": "4beef03dcc62b11af6c43e96cce9e50bf583dce7435622cbf0d684f30717a9f0", "station": "KLAX", "to": "2025-03-31" }, @@ -111,7 +111,7 @@ }, "from": "2024-12-01", "row_count": 365, - "sha256": "6d38652ecb1d83e6ad4d40b639a90923bf660255ab4f0895fabfa3e181c8d4e6", + "sha256": "9cf030452508f329823b4bdc3524541bb2d93b50a686052b698b098ba924e524", "station": "KMIA", "to": "2025-11-30" }, diff --git a/tests/test_parity.py b/tests/test_parity.py index a8041d05..be80665a 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -1,13 +1,42 @@ -"""Day 3 HARD GATE - 5-fixture byte-equivalent parity vs mostlyright==0.14.1. - -This test IS the Sprint 0 parity gate. Sprint 0 ships only if all 5 -parametrized cases pass against the captured ``case_*.parquet`` fixtures. -Each case calls ``mostlyright.research(station, frm, to)`` against the -real public APIs (IEM ASOS, IEM CLI, GHCNh) and compares the returned -DataFrame to the v0.14.1 ``client.pairs()`` output. - -Marked ``@pytest.mark.live`` per CLAUDE.md testing playbook - the default -``-m "not live"`` pytest invocation skips this; run explicitly with:: +"""HARD GATE - 5-fixture byte-equivalent parity vs mostlyright==0.14.1. + +This test IS the parity gate. A release ships only if all 5 parametrized +cases pass against the captured ``case_*.parquet`` fixtures (the last +output ``mostlyright==0.14.1`` ever produced - the oracle is gone: +``api.mostlyright.md`` was re-pointed in Phase 28, so the fixtures can +never be re-captured). + +TWO variants (pinned-input redesign, 2026-07-15): + +- ``test_parity_case_pinned`` - **the HARD gate.** Runs ``research()`` + fully OFFLINE (network hard-blocked) and CACHE-COLD over the committed + ``pinned_cache.tar.xz`` bundle - the RAW ``v1/sources/`` fetch text + (IEM CSV / GHCNh PSV / CLI JSON, captured 2026-07-15 by + ``fixtures/parity/capture_pinned_cache.py``). Cold replay means the + production parsers, ``merge_observations``/``merge_climate`` + SOURCE_PRIORITY logic, aggregation, and the join ALL execute on every + run (codex PR-124 CRITICAL: pinning derived parquets would let the + warm-cache path skip them). Deterministic forever; runs in the default + fast suite. This tests OUR parse-merge-aggregate-join code against the + 0.14.1 outputs - which is what the gate was always meant to prove. +- ``test_parity_case_live`` - **the upstream-drift monitor** (marked + ``live``; pre-publish runbook only). Same assertions against the LIVE + IEM/AWC/GHCNh/CLI APIs. Because the upstream archive is MUTABLE + (late/corrected METARs move the sum-then-divide ``obs_*`` aggregates), + a failure here WHILE the pinned gate is green means the ARCHIVE + drifted, not the code: audit the diff (must be confined to ``obs_*`` + count-consistent cells; ``cli_*``/``obs_high_f``/``obs_low_f`` clean), + then deliberately re-pin via ``capture_pinned_cache.py`` and document + the re-based cells in ``fixtures/parity/README.md``. + +The KLAX 2025-03-12 + 17 KMIA days drifted upstream between the May-2026 +fixture capture and 2026-07-15; those 55 cells (obs_mean_f / +obs_mean_dewpoint_f / obs_total_precip_in / obs_count ONLY) were re-based +onto the pinned archive - the per-cell audit lives in +``fixtures/parity/README.md``. Every other cell remains original 0.14.1 +bytes, and cases 1/2/5 reproduce 0.14.1 EXACTLY from the pinned inputs. + +Run the live monitor explicitly with:: uv run pytest tests/test_parity.py -m live -v @@ -152,6 +181,18 @@ def _pd3_translate_expected(expected: dict[str, str], df: pd.DataFrame) -> dict[ return expected out = dict(expected) for col, dtype_str in expected.items(): + # pandas 3 default datetime resolution inference is MICROSECOND for + # freshly-CONSTRUCTED frames (research() output) but parquet-read + # frames stay nanosecond — the documented, lossless ns → us shift + # from coerce_pd3.py (second-resolution timestamps only in the + # v0.14.1 contract). Conditional on the actual frame, exactly like + # the string translation below: only ns and us are accepted — any + # other dtype still hard-fails. + if dtype_str.startswith("datetime64[ns") and col in df.columns: + actual_dt = str(df[col].dtype) + if actual_dt.startswith("datetime64[us"): + out[col] = dtype_str.replace("[ns", "[us", 1) + continue if dtype_str != "object" or col not in df.columns: continue non_null = df[col].dropna().head(5) @@ -160,14 +201,115 @@ def _pd3_translate_expected(expected: dict[str, str], df: pd.DataFrame) -> dict[ return out +def _pd3_align_expected_frame(df: pd.DataFrame) -> pd.DataFrame: + """Coerce a pd2-written fixture frame's datetime columns ns → us under pd3. + + The DATETIME half of the documented ``coerce_pd3.py`` bridge, applied + inline: a live/pinned ``research()`` frame under pandas 3 infers + microsecond resolution while the pd2-written parquet reads back as + nanoseconds. Lossless (the contract carries second-resolution + timestamps). The bridge's ``object → string`` half is deliberately NOT + applied - pandas 3's parquet reader already lands string columns on the + ``str`` dtype that live frames carry, and ``_pd3_translate_expected`` + pins that expectation. + """ + if not _PD3: + return df + out = df.copy() + for col in out.columns: + dtype_str = str(out[col].dtype) + if dtype_str.startswith("datetime64[ns"): + out[col] = out[col].astype(dtype_str.replace("[ns", "[us", 1)) + return out + + +PINNED_TARBALL = FIXTURES / "pinned_cache.tar.xz" + + +@pytest.fixture(scope="module") +def _pinned_cache_root(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Unpack the pinned RAW-source bundle once for the module. + + The bundle carries ONLY ``v1/sources/`` (the raw IEM CSV / GHCNh PSV / + CLI JSON text captured 2026-07-15) - NO derived parquet tiers - so every + gate run replays CACHE-COLD through the production parsers, + ``merge_observations``/``merge_climate`` SOURCE_PRIORITY logic, + aggregation, and the join (codex PR-124 CRITICAL: pinning the derived + tiers instead would let the warm-cache path skip all of that). Derived + rows written during replay land in this shared per-module root: within a + single case the 2nd/3rd ``research()`` calls go cache-warm (production + behavior), and the 5 cases never share a station. + """ + import tarfile + + assert PINNED_TARBALL.is_file(), ( + "tests/fixtures/parity/pinned_cache.tar.xz missing - run " + "`uv run python tests/fixtures/parity/capture_pinned_cache.py`" + ) + root = tmp_path_factory.mktemp("pinned-cache") + with tarfile.open(PINNED_TARBALL, "r:xz") as tar: + try: + tar.extractall(root, filter="data") + except TypeError: # Python 3.11: no filter= kwarg + tar.extractall(root) + return root + + +@pytest.fixture +def _pinned_inputs(_pinned_cache_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Point the cache at the pinned raw-source root and hard-block the network. + + The socket-level block makes any accidental fetch a loud ``RuntimeError`` + instead of a silent live-dependency - a missing/incomplete bundle can + never quietly turn this gate back into a live test. + """ + import socket + + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(_pinned_cache_root)) + + def _no_net(self: object, *args: object, **kwargs: object) -> None: + raise RuntimeError( + "NETWORK BLOCKED - the pinned parity gate must run fully offline; " + "a fetch here means the pinned_cache bundle is incomplete" + ) + + monkeypatch.setattr(socket.socket, "connect", _no_net) + monkeypatch.setattr( + socket, + "create_connection", + lambda *a, **k: (_ for _ in ()).throw(RuntimeError("NETWORK BLOCKED")), + ) + + +@pytest.mark.parametrize( + "case_num,station,frm,to", + CASES, + ids=[f"case-{n}-{s}-{f}-{t}" for n, s, f, t in CASES], +) +def test_parity_case_pinned( + _pinned_inputs: None, case_num: int, station: str, frm: str, to: str +) -> None: + """PARITY-01 + PARITY-02 (HARD gate): byte-equivalent vs v0.14.1 + ``client.pairs()`` over the PINNED input archive - deterministic, + network-blocked, fast-suite resident.""" + _assert_parity_case(case_num, station, frm, to) + + @pytest.mark.live @pytest.mark.parametrize( "case_num,station,frm,to", CASES, ids=[f"case-{n}-{s}-{f}-{t}" for n, s, f, t in CASES], ) -def test_parity_case(case_num: int, station: str, frm: str, to: str) -> None: - """PARITY-01 + PARITY-02: byte-equivalent vs v0.14.1 client.pairs().""" +def test_parity_case_live(case_num: int, station: str, frm: str, to: str) -> None: + """Upstream-drift MONITOR: the same assertions against the LIVE public + APIs. A failure here while ``test_parity_case_pinned`` is green means + the upstream archive drifted since the pin - see the module docstring + for the audit + deliberate re-pin procedure.""" + _assert_parity_case(case_num, station, frm, to) + + +def _assert_parity_case(case_num: int, station: str, frm: str, to: str) -> None: expected = pd.read_parquet(FIXTURES / f"case_{case_num}_{station}_{frm}_{to}.parquet") # Phase 32 (32-02 / D-22) parity pin: pin ``label="cli"`` EXPLICITLY on both # parity calls. The D-22 default-label bridge flips the implicit default to @@ -192,7 +334,7 @@ def test_parity_case(case_num: int, station: str, frm: str, to: str) -> None: assert_frame_equal(_canon(actual_daily), _canon(actual), check_dtype=True) actual_c = _canon(actual) - expected_c = _canon(expected) + expected_c = _pd3_align_expected_frame(_canon(expected)) # PARITY-03: dtype ground truth. actual_dtypes = {col: str(dtype) for col, dtype in actual_c.dtypes.items()}