bench(refresh): SDID weight gate requires id alignment - no positional fallback

The registry documents the SDID weight-parity gate as id-aligned and
fail-closed, but compare_weight_vectors() silently fell back to
positional comparison when ids were absent, and the runner never checked
aligned_by_ids - a benchmark-script regression that stopped emitting ids
would have quietly weakened the documented contract.

- compare_weight_vectors(require_ids=True): missing ids on either side
  now FAILS the comparison; the SDID publication gate passes it.
  Positional fallback remains available (require_ids=False) for legacy
  artifacts only.
- ensure_mpdta() integrity checks use explicit raises instead of assert
  (asserts vanish under python -O).
- TODO row tracks the intentionally deferred phase-2 artifact/prose push
  (removed by that push itself).
- Tests: 48 total (missing-ids-fail-closed with positionally identical
  weights, valid-ids pass, non-required fallback preserved).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GPX5Rv8ozQXPdUV23QTfjr

Author: igerber

TODO.md

@@ -45,6 +45,10 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m
 
 ### Testing / docs
 
+| Issue | Location | Origin | Effort | Priority |
+|-------|----------|--------|--------|----------|
+| Benchmark refresh phase 2 (same PR #672): run the gated timed refresh on an idle machine, commit `benchmarks/refresh_2026_07/results/refresh_results.json`, regenerate the marker-bounded regions of `docs/benchmarks.rst` via `gen_benchmark_tables.py`, and reconcile the remaining pre-refresh prose (protocol bullets "3 replications / mean ± std", combined BasicDiD/TWFE wording, SDID note under the perf table, Key Observations, "Reproducing Benchmarks" section, `llms.txt` speedup cross-references). Row removed by the phase-2 push itself. | `benchmarks/refresh_2026_07/`, `docs/benchmarks.rst` | #672 | Mid | Medium |
+
 | Issue | Location | Origin | Effort | Priority |
 |-------|----------|--------|--------|----------|
 

benchmarks/refresh_2026_07/refresh_common.py

@@ -131,14 +131,18 @@ def compare_weight_vectors(
     atol: float = 1e-8,
     py_ids: Any = None,
     r_ids: Any = None,
+    require_ids: bool = False,
 ) -> Dict[str, Any]:
     """
     Compare SDID unit/time weight vectors, aligning by unit/period id when
     both sides provide ids (ordering-robust: Python's
     get_unit_weights_df() sorts by descending weight, R emits panel order).
-    Falls back to positional comparison when ids are absent. Enforces the
-    docs' identical-weights claim with an auditable committed metric;
-    fail-closed on length/key mismatch, duplicates, or non-finite entries.
+    With require_ids=True (the SDID publication gate), missing ids on
+    either side FAILS the comparison - the documented id-alignment contract
+    can never silently degrade to positional order if a benchmark script
+    regresses. Without it, positional comparison is a permitted fallback
+    for legacy artifacts. Fail-closed on length/key mismatch, duplicates,
+    or non-finite entries; metrics are committed for audit.
     """
     py = list(py_weights or [])
     r = list(r_weights or [])
@@ -149,6 +153,8 @@ def compare_weight_vectors(
         "aligned_by_ids": False,
         "ok": False,
     }
+    if require_ids and (py_ids is None or r_ids is None):
+        return metrics
     if not py or len(py) != len(r):
         return metrics
 
@@ -847,9 +853,12 @@ def ensure_mpdta() -> Path:
             "first.treat": "first_treat",
         }
     )[["unit", "time", "outcome", "first_treat"]]
-    assert len(df) == 2500, f"MPDTA rows: {len(df)} != 2500"
-    assert df["unit"].nunique() == 500, "MPDTA counties != 500"
-    assert set(df["first_treat"].unique()) == {0, 2004, 2006, 2007}, "MPDTA cohorts unexpected"
+    if len(df) != 2500:
+        raise RuntimeError(f"MPDTA rows: {len(df)} != 2500")
+    if df["unit"].nunique() != 500:
+        raise RuntimeError("MPDTA counties != 500")
+    if set(df["first_treat"].unique()) != {0, 2004, 2006, 2007}:
+        raise RuntimeError("MPDTA cohorts unexpected")
     save_benchmark_data(df, path)
     return path
 

benchmarks/refresh_2026_07/run_refresh.py

@@ -368,6 +368,9 @@ def _att(arm: str) -> float:
                     atol=1e-8,
                     py_ids=py_res.get(id_field),
                     r_ids=r_res.get(id_field),
+                    # Documented contract: the publication gate is id-aligned
+                    # and may never silently degrade to positional order.
+                    require_ids=True,
                 )
                 weight_detail[f"{surface}:{py_arm}"] = metrics
                 if not metrics["ok"]:

tests/test_benchmark_refresh_gates.py

@@ -428,6 +428,27 @@ def test_string_ids_from_r_align_with_int_ids(self):
         m = rc.compare_weight_vectors([0.6, 0.4], [0.4, 0.6], py_ids=[10, 3], r_ids=["3", "10"])
         assert m["aligned_by_ids"] and m["ok"]
 
+    def test_missing_ids_fail_closed_when_required(self):
+        # The SDID publication gate documents id alignment; positionally
+        # identical weights must still FAIL if either side stops emitting
+        # ids (a script regression cannot silently weaken the contract).
+        m = rc.compare_weight_vectors([0.5, 0.5], [0.5, 0.5], require_ids=True)
+        assert not m["ok"] and not m["aligned_by_ids"]
+        m2 = rc.compare_weight_vectors(
+            [0.5, 0.5], [0.5, 0.5], py_ids=[1, 2], r_ids=None, require_ids=True
+        )
+        assert not m2["ok"]
+
+    def test_require_ids_passes_with_valid_ids(self):
+        m = rc.compare_weight_vectors(
+            [0.5, 0.5], [0.5, 0.5], py_ids=[1, 2], r_ids=[1, 2], require_ids=True
+        )
+        assert m["ok"] and m["aligned_by_ids"]
+
+    def test_positional_fallback_still_allowed_when_not_required(self):
+        m = rc.compare_weight_vectors([0.5, 0.5], [0.5, 0.5])
+        assert m["ok"] and not m["aligned_by_ids"]
+
     def test_mismatched_id_sets_fail_closed(self):
         m = rc.compare_weight_vectors([0.5, 0.5], [0.5, 0.5], py_ids=[1, 2], r_ids=[1, 3])
         assert not m["ok"] and m["max_abs_diff"] is None