From d7146c01328c82aa2743404ac116d5e9d6d088e0 Mon Sep 17 00:00:00 2001 From: Charles Shaw Date: Thu, 23 Jul 2026 14:12:25 +0100 Subject: [PATCH 01/17] fix: make legacy dataset provenance explicit --- CHANGELOG.md | 11 + diff_diff/datasets.py | 824 +++++++++++++++++++++++------ diff_diff/guides/llms-full.txt | 2 +- tests/test_datasets.py | 387 +++++++++++++- tests/test_doc_snippets.py | 2 +- tests/test_methodology_callaway.py | 5 +- 6 files changed, 1065 insertions(+), 166 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b216cd042..35a2a004f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -394,6 +394,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 event-study surface) and M-093 (4.0 sentinel retirement, phase 5) are added. **Estimator equations, weighting, variance, and numerical output are unchanged.** +### Fixed +- **Legacy dataset loaders no longer silently present synthetic observations as + canonical data.** `load_card_krueger()`, `load_castle_doctrine()`, and + `load_mpdta()` now use commit-pinned, SHA-256-verified sources with schema and + panel-invariant validation; successful frames expose a dataset-specific + `df.attrs["source"]`. Download, checksum, parsing, or validation failures emit + exactly one `UserWarning` containing `SYNTHETIC` and mark the fallback as + `source="synthetic_fallback"`. `load_divorce_laws()` now follows that explicit + fallback path because no verified source currently reproduces its composite + public schema without additional analytical choices. + ### Changed - **dCDH Phase 1 (`L_max=None`) event-study row is now re-synced by the final-df refresh.** Under replicate-weight designs the refresh recomputes `overall_*` with the diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 1be024011..4bce8c22e 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -4,15 +4,18 @@ This module provides functions to load classic econometrics datasets commonly used for teaching and demonstrating DiD methods. -All datasets are downloaded from public sources and cached locally -for subsequent use. +Canonical data are downloaded from checksum-pinned public sources and cached +locally. If a source cannot be verified, the loader warns and returns an +explicitly provenance-marked synthetic fallback. """ import hashlib +import os import warnings from io import BytesIO, StringIO from pathlib import Path -from typing import Dict, cast +from tempfile import NamedTemporaryFile +from typing import Callable, Dict, Optional, cast from urllib.error import HTTPError, URLError from urllib.request import urlopen @@ -21,6 +24,42 @@ # Cache directory for downloaded datasets _CACHE_DIR = Path.home() / ".cache" / "diff_diff" / "datasets" +_MAX_DATASET_BYTES = 50 * 1024 * 1024 + +# These commit-pinned mirrors were checked against the dataset authors' package +# artifacts. Every cached and downloaded byte sequence is verified below. +_CARD_KRUEGER_SOURCE_URL = ( + "https://raw.githubusercontent.com/rafiash/CardKrueger-stata-sample/" + "07bc929f1d6552db117bd27a7cf0d881d16e9494/public.dat" +) +_CARD_KRUEGER_SOURCE_SHA256 = "04bde0cad5540980f32ce099c6dad369e2f05494698071d8a65b3e1cbe9ca53a" +_CASTLE_DOCTRINE_SOURCE_URL = ( + "https://raw.githubusercontent.com/scunning1975/mixtape/" + "ca4279a87a6f0759f6b6f02841a53bdd68e27d3c/castle.dta" +) +_CASTLE_DOCTRINE_SOURCE_SHA256 = "804633c161827b6c0824f86f239046386d1a8266a866f83bf5ddb2aa762a5f29" +_MPDTA_SOURCE_URL = ( + "https://raw.githubusercontent.com/d2cml-ai/csdid/" + "7ad707385354cb3924b8da94ef7e62a76bf55a4d/data/mpdta.csv" +) +_MPDTA_SOURCE_SHA256 = "2283bea1221a152420f98dfa20f633c5d054ea51d881115c8cd702a97bcd3167" + +# ``sid`` follows alphabetical state order with 9 reserved for Washington, DC, +# which is absent from the source panel. +_CASTLE_STATE_BY_SID = dict( + enumerate( + """ + AL AK AZ AR CA CO CT DE _ FL GA HI ID IL IN IA KS KY LA ME MD MA MI MN + MS MO MT NE NV NH NJ NM NY NC ND OH OK OR PA RI SC SD TN TX UT VT VA WA + WV WI WY + """.split(), + start=1, + ) +) + + +class _DatasetSourceError(RuntimeError): + """Expected failure while fetching, parsing, or validating canonical data.""" def _get_cache_path(name: str) -> Path: @@ -32,28 +71,97 @@ def _get_cache_path(name: str) -> Path: def _download_with_cache( url: str, name: str, + sha256: str, force_download: bool = False, ) -> str: - """Download a file and cache it locally.""" + """Download UTF-8 text, verify its checksum, and cache it.""" cache_path = _get_cache_path(name) + content = _download_verified_bytes(url, name, sha256, cache_path, force_download) + try: + return content.decode("utf-8") + except UnicodeDecodeError as e: + raise _DatasetSourceError( + f"Dataset '{name}' passed its byte checksum but is not valid UTF-8 text." + ) from e - if cache_path.exists() and not force_download: - return cache_path.read_text() + +def _read_verified_cache(cache_path: Path, sha256: str) -> Optional[bytes]: + """Return a bounded, checksum-valid cache entry or None.""" + try: + if not cache_path.exists() or cache_path.stat().st_size > _MAX_DATASET_BYTES: + return None + content = cache_path.read_bytes() + except OSError: + return None + if hashlib.sha256(content).hexdigest() == sha256: + return content + return None + + +def _write_cache_atomically(cache_path: Path, content: bytes, name: str) -> None: + """Replace a cache entry only after a complete same-directory write.""" + temp_path: Optional[Path] = None + try: + with NamedTemporaryFile( + mode="wb", + dir=cache_path.parent, + prefix=f".{cache_path.name}.", + delete=False, + ) as temp_file: + temp_file.write(content) + temp_path = Path(temp_file.name) + os.replace(temp_path, cache_path) + except OSError as e: + if temp_path is not None: + try: + temp_path.unlink() + except OSError: + pass + raise _DatasetSourceError(f"Failed to cache dataset '{name}': {e}") from e + + +def _download_verified_bytes( + url: str, + name: str, + sha256: str, + cache_path: Path, + force_download: bool = False, +) -> bytes: + """Return checksum-verified bytes from cache or a fresh download.""" + if not force_download: + cached = _read_verified_cache(cache_path, sha256) + if cached is not None: + return cached try: with urlopen(url, timeout=30) as response: - content = response.read().decode("utf-8") - cache_path.write_text(content) - return content - except (HTTPError, URLError) as e: - if cache_path.exists(): - # Use cached version if download fails - return cache_path.read_text() - raise RuntimeError( + content = response.read(_MAX_DATASET_BYTES + 1) + except (HTTPError, OSError, TimeoutError, URLError) as e: + cached = _read_verified_cache(cache_path, sha256) + if cached is not None: + return cached + raise _DatasetSourceError( f"Failed to download dataset '{name}' from {url}: {e}\n" "Check your internet connection or try again later." ) from e + if len(content) > _MAX_DATASET_BYTES: + raise _DatasetSourceError( + f"Dataset '{name}' downloaded from {url} exceeds the " + f"{_MAX_DATASET_BYTES}-byte safety limit." + ) + + if hashlib.sha256(content).hexdigest() != sha256: + raise _DatasetSourceError( + f"Checksum mismatch for dataset '{name}' downloaded from {url}.\n" + "The upstream file differs from the pinned SHA-256. Verify the " + "source revision before updating the pinned checksum; otherwise " + "treat the download as untrusted." + ) + + _write_cache_atomically(cache_path, content, name) + return content + def _get_cache_path_binary(name: str) -> Path: """Get the cache path for a binary (Stata .dta) dataset.""" @@ -69,43 +177,491 @@ def _download_with_cache_binary( ) -> bytes: """Download a binary file (e.g. Stata .dta), verify its checksum, and cache it. - The source host serves plain HTTP, so every byte-load (cache or fresh - download) is verified against a pinned SHA-256. A stale/corrupt cache - triggers one re-download; a checksum mismatch on freshly downloaded - bytes raises. + Every byte-load (cache or fresh download) is verified against a pinned + SHA-256. A stale/corrupt cache triggers one re-download; a checksum + mismatch on freshly downloaded bytes raises. """ - cache_path = _get_cache_path_binary(name) + return _download_verified_bytes( + url, + name, + sha256, + _get_cache_path_binary(name), + force_download, + ) - if cache_path.exists() and not force_download: - content = cache_path.read_bytes() - if hashlib.sha256(content).hexdigest() == sha256: - return content - # Cached copy is stale or corrupt: fall through to re-download +def _load_verified_dataset( + *, + cache_name: str, + source: str, + force_download: bool, + load_source: Optional[Callable[[bool], pd.DataFrame]], + prepare: Callable[[pd.DataFrame], pd.DataFrame], + validate_source: Callable[[pd.DataFrame], None], + validate_fallback: Callable[[pd.DataFrame], None], + fallback: Callable[[], pd.DataFrame], +) -> pd.DataFrame: + """Load and validate canonical data or return a loud synthetic fallback.""" try: - with urlopen(url, timeout=30) as response: - content = response.read() - except (HTTPError, URLError) as e: - if cache_path.exists(): - content = cache_path.read_bytes() - if hashlib.sha256(content).hexdigest() == sha256: - # Use cached version if download fails - return content - raise RuntimeError( - f"Failed to download dataset '{name}' from {url}: {e}\n" - "Check your internet connection or try again later." - ) from e + if load_source is None: + raise _DatasetSourceError("no verified canonical source is configured") + df = prepare(load_source(force_download)) + validate_source(df) + except _DatasetSourceError as exc: + warnings.warn( + f"{cache_name} canonical data are unavailable ({exc}); returning a " + "SYNTHETIC fallback. Check `df.attrs['source']` before treating " + "the result as replication data.", + UserWarning, + stacklevel=2, + ) + df = fallback() + validate_fallback(df) + df.attrs["source"] = "synthetic_fallback" + return df - if hashlib.sha256(content).hexdigest() != sha256: - raise RuntimeError( - f"Checksum mismatch for dataset '{name}' downloaded from {url}.\n" - "The upstream file differs from the pinned SHA-256. If the lwdid " - "Stata package published a new data revision, verify the new file " - "and update the pinned checksum; otherwise treat the download as " - "untrusted." + df.attrs["source"] = source + return df + + +def _load_card_krueger_source(force_download: bool) -> pd.DataFrame: + """Load the checksum-pinned Card-Krueger public flat file.""" + content = _download_with_cache( + _CARD_KRUEGER_SOURCE_URL, + "card_krueger", + _CARD_KRUEGER_SOURCE_SHA256, + force_download, + ) + columns = """ + sheet chain co_owned state southj centralj northj pa1 pa2 shore + ncalls empft emppt nmgrs wage_st inctime firstinc bonus pctaff meals + open hrsopen psoda pfry pentree nregs nregs11 type2 status2 date2 + ncalls2 empft2 emppt2 nmgrs2 wage_st2 inctime2 firstin2 special2 + meals2 open2r hrsopen2 psoda2 pfry2 pentree2 nregs2 nregs112 + """.split() + try: + return pd.read_csv( + StringIO(content), + sep=r"\s+", + names=columns, + na_values=".", ) - cache_path.write_bytes(content) - return content + except (TypeError, ValueError) as e: + raise _DatasetSourceError(f"Failed to parse Card-Krueger source data: {e}") from e + + +def _load_castle_doctrine_source(force_download: bool) -> pd.DataFrame: + """Load the checksum-pinned Cheng-Hoekstra Stata data.""" + content = _download_with_cache_binary( + _CASTLE_DOCTRINE_SOURCE_URL, + "castle_doctrine", + _CASTLE_DOCTRINE_SOURCE_SHA256, + force_download, + ) + try: + return pd.read_stata(BytesIO(content)) + except (OSError, TypeError, ValueError) as e: + raise _DatasetSourceError(f"Failed to parse Castle Doctrine source data: {e}") from e + + +def _load_mpdta_source(force_download: bool) -> pd.DataFrame: + """Load the checksum-pinned Callaway-Sant'Anna example data.""" + content = _download_with_cache( + _MPDTA_SOURCE_URL, + "mpdta", + _MPDTA_SOURCE_SHA256, + force_download, + ) + try: + return pd.read_csv(StringIO(content)) + except (TypeError, ValueError) as e: + raise _DatasetSourceError(f"Failed to parse mpdta source data: {e}") from e + + +def _require_columns(df: pd.DataFrame, dataset: str, columns: set) -> None: + """Reject empty or structurally incomplete downloaded datasets.""" + if df.empty: + raise _DatasetSourceError(f"{dataset} source is empty") + missing = columns - set(df.columns) + if missing: + raise _DatasetSourceError(f"{dataset} source is missing columns: {sorted(missing)}") + + +def _identity_dataset(df: pd.DataFrame) -> pd.DataFrame: + """Return an already-normalized dataset unchanged.""" + return df + + +def _require_complete(df: pd.DataFrame, dataset: str, columns: set) -> None: + """Reject missing values in columns whose public contract is complete.""" + missing = df[list(columns)].isna().sum() + missing = missing[missing > 0] + if not missing.empty: + raise _DatasetSourceError( + f"{dataset} source has missing required values: {missing.to_dict()}" + ) + + +def _require_finite(df: pd.DataFrame, dataset: str, columns: set) -> None: + """Reject non-numeric or non-finite values in numeric contract columns.""" + try: + values = df[list(columns)].to_numpy(dtype=float) + except (TypeError, ValueError) as e: + raise _DatasetSourceError(f"{dataset} source has non-numeric values") from e + if not np.isfinite(values).all(): + raise _DatasetSourceError(f"{dataset} source has non-finite values in {sorted(columns)}") + + +def _validate_panel_keys(df: pd.DataFrame, dataset: str, unit: str) -> None: + """Validate the common unit-time and cohort invariants for panel datasets.""" + if df.duplicated([unit, "year"]).any(): + raise _DatasetSourceError(f"{dataset} source has duplicate {unit}-year rows") + cohort_counts = df.groupby(unit)["first_treat"].nunique(dropna=False) + if not (cohort_counts == 1).all(): + raise _DatasetSourceError(f"{dataset} first_treat is not constant within {unit}") + if not (df["cohort"] == df["first_treat"]).all(): + raise _DatasetSourceError(f"{dataset} cohort does not match first_treat") + expected_treated = ((df["first_treat"] > 0) & (df["year"] >= df["first_treat"])).astype(int) + if not (df["treated"] == expected_treated).all(): + raise _DatasetSourceError(f"{dataset} treated indicator is inconsistent with first_treat") + + +def _prepare_card_krueger(df: pd.DataFrame) -> pd.DataFrame: + """Normalize a Card-Krueger source frame to the public loader schema.""" + raw_columns = { + "sheet", + "state", + "chain", + "empft", + "emppt", + "nmgrs", + "wage_st", + "empft2", + "emppt2", + "nmgrs2", + "wage_st2", + } + if raw_columns <= set(df.columns): + store_id = df["sheet"].copy() + _require_complete(df, "card_krueger", {"sheet", "state", "chain"}) + _require_finite(df, "card_krueger", {"sheet", "state", "chain"}) + if not set(df["state"].unique()) <= {0, 1}: + raise _DatasetSourceError("card_krueger source has unknown state codes") + if not set(df["chain"].unique()) <= {1, 2, 3, 4}: + raise _DatasetSourceError("card_krueger source has unknown chain codes") + for column in ( + "empft", + "emppt", + "nmgrs", + "wage_st", + "empft2", + "emppt2", + "nmgrs2", + "wage_st2", + ): + converted = pd.to_numeric(df[column], errors="coerce") + if converted.notna().sum() != df[column].notna().sum(): + raise _DatasetSourceError(f"card_krueger source has non-numeric values in {column}") + df[column] = converted + duplicate_407 = store_id == 407 + if ( + duplicate_407.sum() != 2 + or set(df.loc[duplicate_407, "state"]) != {0, 1} + or (store_id == 408).any() + ): + raise _DatasetSourceError( + "card_krueger source does not match the documented duplicate-407 convention" + ) + store_id.loc[duplicate_407 & (df["state"] == 1)] = 408 + emp_pre = df["empft"] + df["nmgrs"] + 0.5 * df["emppt"] + emp_post = df["empft2"] + df["nmgrs2"] + 0.5 * df["emppt2"] + return pd.DataFrame( + { + "store_id": store_id.astype(int), + "state": np.where(df["state"] == 1, "NJ", "PA"), + "chain": df["chain"].map({1: "bk", 2: "kfc", 3: "roys", 4: "wendys"}), + "emp_pre": emp_pre, + "emp_post": emp_post, + "wage_pre": df["wage_st"], + "wage_post": df["wage_st2"], + "treated": (df["state"] == 1).astype(int), + "emp_change": emp_post - emp_pre, + } + ) + + df = df.rename(columns={"sheet": "store_id"}).copy() + if "state" not in df.columns and "nj" in df.columns: + df["state"] = np.where(df["nj"] == 1, "NJ", "PA") + if "treated" not in df.columns and "state" in df.columns: + df["treated"] = (df["state"] == "NJ").astype(int) + if "emp_change" not in df.columns and {"emp_post", "emp_pre"} <= set(df.columns): + df["emp_change"] = df["emp_post"] - df["emp_pre"] + return df + + +def _validate_card_krueger(df: pd.DataFrame) -> None: + """Validate the documented Card-Krueger wide-data contract.""" + _require_columns( + df, + "card_krueger", + { + "store_id", + "state", + "chain", + "emp_pre", + "emp_post", + "wage_pre", + "wage_post", + "treated", + "emp_change", + }, + ) + _require_complete(df, "card_krueger", {"store_id", "state", "chain", "treated"}) + _require_finite(df, "card_krueger", {"store_id", "treated"}) + if df["store_id"].duplicated().any(): + raise _DatasetSourceError("card_krueger source has duplicate store_id values") + if set(df["state"].dropna().unique()) != {"NJ", "PA"}: + raise _DatasetSourceError("card_krueger source must contain both NJ and PA only") + if set(df["chain"].dropna().unique()) != {"bk", "kfc", "roys", "wendys"}: + raise _DatasetSourceError("card_krueger source has unexpected restaurant chains") + for column in ("emp_pre", "emp_post", "wage_pre", "wage_post"): + values = pd.to_numeric(df[column], errors="coerce") + if ( + values.notna().sum() != df[column].notna().sum() + or not np.isfinite(values.dropna()).all() + or (values.dropna() < 0).any() + ): + raise _DatasetSourceError( + f"card_krueger source has invalid non-negative values in {column}" + ) + emp_change = pd.to_numeric(df["emp_change"], errors="coerce") + if ( + emp_change.notna().sum() != df["emp_change"].notna().sum() + or not np.isfinite(emp_change.dropna()).all() + ): + raise _DatasetSourceError("card_krueger source has invalid emp_change values") + expected_treated = (df["state"] == "NJ").astype(int) + if not (df["treated"] == expected_treated).all(): + raise _DatasetSourceError("card_krueger treated indicator is inconsistent with state") + expected_change = df["emp_post"] - df["emp_pre"] + if not np.allclose(df["emp_change"], expected_change, equal_nan=True): + raise _DatasetSourceError( + "card_krueger emp_change is inconsistent with emp_pre and emp_post" + ) + + +def _validate_card_krueger_source(df: pd.DataFrame) -> None: + """Validate source-specific Card-Krueger counts and categories.""" + _validate_card_krueger(df) + if len(df) != 410: + raise _DatasetSourceError("card_krueger source must contain 410 stores") + if df.groupby("state").size().to_dict() != {"NJ": 331, "PA": 79}: + raise _DatasetSourceError("card_krueger source has unexpected state counts") + expected_missing = { + "emp_pre": 12, + "emp_post": 14, + "wage_pre": 20, + "wage_post": 21, + "emp_change": 26, + } + if df[list(expected_missing)].isna().sum().to_dict() != expected_missing: + raise _DatasetSourceError("card_krueger source has unexpected missing-value counts") + + +def _prepare_castle_doctrine(df: pd.DataFrame) -> pd.DataFrame: + """Normalize a Castle Doctrine source frame to the public loader schema.""" + df = df.copy() + if "sid" in df.columns: + state_codes = df["sid"].map(_CASTLE_STATE_BY_SID) + if state_codes.notna().all() and not (state_codes == "_").any(): + df["state"] = state_codes + if "first_treat" not in df.columns and "effyear" in df.columns: + try: + df["first_treat"] = df["effyear"].fillna(0).astype(int) + except (TypeError, ValueError) as e: + raise _DatasetSourceError("castle_doctrine source has invalid effyear values") from e + if "cohort" not in df.columns and "first_treat" in df.columns: + df["cohort"] = df["first_treat"] + if {"first_treat", "year"} <= set(df.columns): + df["treated"] = ((df["first_treat"] > 0) & (df["year"] >= df["first_treat"])).astype(int) + if "homicide_rate" not in df.columns and "homicide" in df.columns: + df["homicide_rate"] = df["homicide"] + if { + "state", + "year", + "first_treat", + "homicide_rate", + "population", + "income", + "treated", + "cohort", + } <= set(df.columns): + return df[ + [ + "state", + "year", + "first_treat", + "homicide_rate", + "population", + "income", + "treated", + "cohort", + ] + ].copy() + return df + + +def _validate_castle_doctrine(df: pd.DataFrame) -> None: + """Validate the documented Castle Doctrine panel contract.""" + _require_columns( + df, + "castle_doctrine", + { + "state", + "year", + "first_treat", + "homicide_rate", + "population", + "income", + "treated", + "cohort", + }, + ) + _require_complete( + df, + "castle_doctrine", + { + "state", + "year", + "first_treat", + "homicide_rate", + "population", + "income", + "treated", + "cohort", + }, + ) + _require_finite( + df, + "castle_doctrine", + {"year", "first_treat", "homicide_rate", "population", "income", "treated", "cohort"}, + ) + if ( + (df["homicide_rate"] < 0).any() + or (df["population"] <= 0).any() + or (df["income"] <= 0).any() + ): + raise _DatasetSourceError("castle_doctrine source has invalid outcome or covariate values") + if not df["state"].astype(str).str.fullmatch(r"[A-Z]{2}").all(): + raise _DatasetSourceError("castle_doctrine source has invalid state abbreviations") + _validate_panel_keys(df, "castle_doctrine", "state") + + +def _validate_castle_doctrine_source(df: pd.DataFrame) -> None: + """Validate source-specific Castle Doctrine panel dimensions.""" + _validate_castle_doctrine(df) + if len(df) != 550 or df["state"].nunique() != 50: + raise _DatasetSourceError("castle_doctrine source must contain 50 states and 550 rows") + if set(df["year"].unique()) != set(range(2000, 2011)): + raise _DatasetSourceError("castle_doctrine source has unexpected years") + if set(df["first_treat"].unique()) != {0, 2005, 2006, 2007, 2008, 2009}: + raise _DatasetSourceError("castle_doctrine source has unexpected treatment cohorts") + + +def _validate_divorce_laws(df: pd.DataFrame) -> None: + """Validate the documented divorce-laws panel contract.""" + _require_columns( + df, + "divorce_laws", + { + "state", + "year", + "first_treat", + "divorce_rate", + "female_lfp", + "suicide_rate", + "treated", + "cohort", + }, + ) + _require_complete( + df, + "divorce_laws", + { + "state", + "year", + "first_treat", + "divorce_rate", + "female_lfp", + "suicide_rate", + "treated", + "cohort", + }, + ) + _require_finite( + df, + "divorce_laws", + { + "year", + "first_treat", + "divorce_rate", + "female_lfp", + "suicide_rate", + "treated", + "cohort", + }, + ) + if (df["divorce_rate"] < 0).any() or (df["suicide_rate"] < 0).any(): + raise _DatasetSourceError("divorce_laws source has negative outcome values") + if not df["female_lfp"].between(0, 1).all(): + raise _DatasetSourceError("divorce_laws female_lfp must be between 0 and 1") + _validate_panel_keys(df, "divorce_laws", "state") + + +def _prepare_mpdta(df: pd.DataFrame) -> pd.DataFrame: + """Normalize an mpdta source frame to the public loader schema.""" + if "first.treat" in df.columns: + df = df.rename(columns={"first.treat": "first_treat"}) + if "cohort" not in df.columns and "first_treat" in df.columns: + df["cohort"] = df["first_treat"] + columns = ["countyreal", "year", "lpop", "lemp", "first_treat", "treat", "cohort"] + if set(columns) <= set(df.columns): + return df[columns].copy() + return df + + +def _validate_mpdta(df: pd.DataFrame) -> None: + """Validate the canonical R did::mpdta panel structure.""" + _require_columns( + df, + "mpdta", + {"countyreal", "year", "lpop", "lemp", "first_treat", "treat", "cohort"}, + ) + _require_complete( + df, + "mpdta", + {"countyreal", "year", "lpop", "lemp", "first_treat", "treat", "cohort"}, + ) + _require_finite( + df, + "mpdta", + {"countyreal", "year", "lpop", "lemp", "first_treat", "treat", "cohort"}, + ) + if df.duplicated(["countyreal", "year"]).any(): + raise _DatasetSourceError("mpdta source has duplicate county-year rows") + if len(df) != 2500 or df["countyreal"].nunique() != 500: + raise _DatasetSourceError("mpdta source must contain 500 counties and 2500 rows") + if set(df["year"].unique()) != {2003, 2004, 2005, 2006, 2007}: + raise _DatasetSourceError("mpdta source has unexpected years") + if set(df["first_treat"].unique()) != {0, 2004, 2006, 2007}: + raise _DatasetSourceError("mpdta source has unexpected treatment cohorts") + cohort_counts = df.groupby("countyreal")["first_treat"].nunique(dropna=False) + if not (cohort_counts == 1).all(): + raise _DatasetSourceError("mpdta first_treat is not constant within county") + if not (df["cohort"] == df["first_treat"]).all(): + raise _DatasetSourceError("mpdta cohort does not match first_treat") + if not (df["treat"] == (df["first_treat"] > 0).astype(int)).all(): + raise _DatasetSourceError("mpdta treat indicator is inconsistent with first_treat") def clear_cache() -> None: @@ -154,6 +710,11 @@ def load_card_krueger(force_download: bool = False) -> pd.DataFrame: Original finding: No significant negative effect of minimum wage increase on employment (ATT ≈ +2.8 FTE employees). + The canonical data are checksum-verified and returned with + ``df.attrs["source"] == "card_krueger_public_data"``. Any download, parse, + or validation failure emits one ``UserWarning`` containing ``SYNTHETIC`` + and returns ``df.attrs["source"] == "synthetic_fallback"``. + References ---------- Card, D., & Krueger, A. B. (1994). Minimum Wages and Employment: A Case Study @@ -178,36 +739,17 @@ def load_card_krueger(force_download: bool = False) -> pd.DataFrame: >>> did = DifferenceInDifferences() >>> results = did.fit(ck_long, outcome='employment', treatment='treated', time='post') """ - # Card-Krueger data hosted at multiple academic sources - # Using Princeton data archive mirror - url = "https://raw.githubusercontent.com/causaldata/causal_datasets/main/card_krueger/card_krueger.csv" - - try: - content = _download_with_cache(url, "card_krueger", force_download) - df = pd.read_csv(StringIO(content)) - except RuntimeError: - # Fallback: construct from embedded data - df = _construct_card_krueger_data() - - # Standardize column names and add convenience columns - df = df.rename( - columns={ - "sheet": "store_id", - } + return _load_verified_dataset( + cache_name="card_krueger", + source="card_krueger_public_data", + force_download=force_download, + load_source=_load_card_krueger_source, + prepare=_prepare_card_krueger, + validate_source=_validate_card_krueger_source, + validate_fallback=_validate_card_krueger, + fallback=_construct_card_krueger_data, ) - # Ensure proper types - if "state" not in df.columns and "nj" in df.columns: - df["state"] = np.where(df["nj"] == 1, "NJ", "PA") - - if "treated" not in df.columns: - df["treated"] = (df["state"] == "NJ").astype(int) - - if "emp_change" not in df.columns and "emp_post" in df.columns and "emp_pre" in df.columns: - df["emp_change"] = df["emp_post"] - df["emp_pre"] - - return df - def _construct_card_krueger_data() -> pd.DataFrame: """ @@ -309,6 +851,11 @@ def load_castle_doctrine(force_download: bool = False) -> pd.DataFrame: in self-defense. States adopted these laws at different times between 2005 and 2009, creating a staggered treatment design. + The canonical data are checksum-verified and returned with + ``df.attrs["source"] == "cheng_hoekstra_castle_data"``. Any download, + parse, or validation failure emits one ``UserWarning`` containing + ``SYNTHETIC`` and marks the returned frame as ``"synthetic_fallback"``. + References ---------- Cheng, C., & Hoekstra, M. (2013). Does Strengthening Self-Defense Law Deter @@ -330,34 +877,16 @@ def load_castle_doctrine(force_download: bool = False) -> pd.DataFrame: ... first_treat="first_treat" ... ) """ - url = "https://raw.githubusercontent.com/causaldata/causal_datasets/main/castle/castle.csv" - - try: - content = _download_with_cache(url, "castle_doctrine", force_download) - df = pd.read_csv(StringIO(content)) - except RuntimeError: - # Fallback: construct from documented patterns - df = _construct_castle_doctrine_data() - - # Standardize column names - rename_map = { - "sid": "state_id", - "cdl": "treated", - } - df = df.rename(columns={k: v for k, v in rename_map.items() if k in df.columns}) - - # Add convenience columns - if "first_treat" not in df.columns and "effyear" in df.columns: - df["first_treat"] = df["effyear"].fillna(0).astype(int) - - if "cohort" not in df.columns and "first_treat" in df.columns: - df["cohort"] = df["first_treat"] - - # Ensure treated indicator exists - if "treated" not in df.columns and "first_treat" in df.columns: - df["treated"] = ((df["first_treat"] > 0) & (df["year"] >= df["first_treat"])).astype(int) - - return df + return _load_verified_dataset( + cache_name="castle_doctrine", + source="cheng_hoekstra_castle_data", + force_download=force_download, + load_source=_load_castle_doctrine_source, + prepare=_prepare_castle_doctrine, + validate_source=_validate_castle_doctrine_source, + validate_fallback=_validate_castle_doctrine, + fallback=_construct_castle_doctrine_data, + ) def _construct_castle_doctrine_data() -> pd.DataFrame: @@ -477,7 +1006,8 @@ def load_divorce_laws(force_download: bool = False) -> pd.DataFrame: Parameters ---------- force_download : bool, default=False - If True, re-download the dataset even if cached. + Retained for API compatibility. No verified source currently satisfies + the loader's composite schema. Returns ------- @@ -498,6 +1028,11 @@ def load_divorce_laws(force_download: bool = False) -> pd.DataFrame: other's consent. States adopted these laws at different times, primarily between 1969 and 1985. + No verified source currently reproduces all documented columns without + deriving new variables or changing pre-panel treatment semantics. This + loader therefore emits one ``UserWarning`` containing ``SYNTHETIC`` and + returns ``df.attrs["source"] == "synthetic_fallback"``. + References ---------- Stevenson, B., & Wolfers, J. (2006). Bargaining in the Shadow of the Law: @@ -522,39 +1057,16 @@ def load_divorce_laws(force_download: bool = False) -> pd.DataFrame: ... first_treat="first_treat" ... ) """ - # Try to load from causaldata repository - url = "https://raw.githubusercontent.com/causaldata/causal_datasets/main/divorce/divorce.csv" - - try: - content = _download_with_cache(url, "divorce_laws", force_download) - df = pd.read_csv(StringIO(content)) - except RuntimeError: - # Fallback to constructed data - df = _construct_divorce_laws_data() - - # Standardize column names - if "stfips" in df.columns: - df = df.rename(columns={"stfips": "state_id"}) - - if "first_treat" not in df.columns and "unilateral" in df.columns: - # Determine first treatment year from the unilateral indicator - first_treat = df.groupby("state").apply( - lambda x: x.loc[x["unilateral"] == 1, "year"].min() if x["unilateral"].sum() > 0 else 0 - ) - df["first_treat"] = df["state"].map(first_treat).fillna(0).astype(int) - - if "cohort" not in df.columns and "first_treat" in df.columns: - df["cohort"] = df["first_treat"] - - if "treated" not in df.columns: - if "unilateral" in df.columns: - df["treated"] = df["unilateral"] - elif "first_treat" in df.columns: - df["treated"] = ((df["first_treat"] > 0) & (df["year"] >= df["first_treat"])).astype( - int - ) - - return df + return _load_verified_dataset( + cache_name="divorce_laws", + source="stevenson_wolfers_divorce_data", + force_download=force_download, + load_source=None, + prepare=_identity_dataset, + validate_source=_validate_divorce_laws, + validate_fallback=_validate_divorce_laws, + fallback=_construct_divorce_laws_data, + ) def _construct_divorce_laws_data() -> pd.DataFrame: @@ -689,9 +1201,9 @@ def load_mpdta(force_download: bool = False) -> pd.DataFrame: """ Load the Minimum Wage Panel Dataset for DiD Analysis (mpdta). - This is a simulated dataset from the R `did` package that mimics - county-level employment data under staggered minimum wage increases. - It's designed specifically for teaching the Callaway-Sant'Anna estimator. + This example dataset from the R `did` package contains county-level teen + employment data under staggered minimum wage increases. It is commonly + used to teach the Callaway-Sant'Anna estimator. Parameters ---------- @@ -714,6 +1226,11 @@ def load_mpdta(force_download: bool = False) -> pd.DataFrame: This dataset is included in the R `did` package and is commonly used in tutorials demonstrating the Callaway-Sant'Anna estimator. + The canonical data are checksum-verified and returned with + ``df.attrs["source"] == "callaway_santanna_mpdta"``. Any download, parse, + or validation failure emits one ``UserWarning`` containing ``SYNTHETIC`` + and marks the returned frame as ``"synthetic_fallback"``. + References ---------- Callaway, B., & Sant'Anna, P. H. (2021). Difference-in-differences with @@ -734,25 +1251,16 @@ def load_mpdta(force_download: bool = False) -> pd.DataFrame: ... first_treat="first_treat" ... ) """ - # mpdta is available from the did package documentation - url = "https://raw.githubusercontent.com/bcallaway11/did/master/data-raw/mpdta.csv" - - try: - content = _download_with_cache(url, "mpdta", force_download) - df = pd.read_csv(StringIO(content)) - except RuntimeError: - # Fallback to constructed data matching the R package - df = _construct_mpdta_data() - - # Standardize column names - if "first.treat" in df.columns: - df = df.rename(columns={"first.treat": "first_treat"}) - - # Ensure cohort column exists - if "cohort" not in df.columns and "first_treat" in df.columns: - df["cohort"] = df["first_treat"] - - return df + return _load_verified_dataset( + cache_name="mpdta", + source="callaway_santanna_mpdta", + force_download=force_download, + load_source=_load_mpdta_source, + prepare=_prepare_mpdta, + validate_source=_validate_mpdta, + validate_fallback=_validate_mpdta, + fallback=_construct_mpdta_data, + ) def _construct_mpdta_data() -> pd.DataFrame: @@ -1175,7 +1683,7 @@ def list_datasets() -> Dict[str, str]: "card_krueger": "Card & Krueger (1994) minimum wage dataset - classic 2x2 DiD", "castle_doctrine": "Castle Doctrine laws - staggered adoption across states", "divorce_laws": "Unilateral divorce laws - staggered adoption (Stevenson-Wolfers)", - "mpdta": "Minimum wage panel data - simulated CS example from R `did` package", + "mpdta": "County teen-employment panel - Callaway-Sant'Anna example from R `did`", "prop99": "California Prop 99 smoking panel - single treated unit (Lee-Wooldridge format)", "walmart": "Walmart entry county panel - staggered adoption (Lee-Wooldridge sample)", } diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 67e20e633..13e878096 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -2373,7 +2373,7 @@ data = load_dataset("card_krueger") ck = load_card_krueger() # Card & Krueger (1994) minimum wage castle = load_castle_doctrine() # Castle Doctrine / Stand Your Ground laws divorce = load_divorce_laws() # Unilateral divorce laws (staggered) -mpdta = load_mpdta() # Minimum wage panel (simulated, from R did package) +mpdta = load_mpdta() # County teen-employment panel from R did package prop99 = load_prop99() # California Prop 99 smoking (single treated unit) walmart = load_walmart() # Walmart entry county panel (staggered, 1,277 counties) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index a5fbfa515..19dadfaf2 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -21,7 +21,10 @@ clear_cache, list_datasets, load_card_krueger, + load_castle_doctrine, load_dataset, + load_divorce_laws, + load_mpdta, load_prop99, load_walmart, ) @@ -63,8 +66,11 @@ def test_load_by_name(self): """load_dataset should load datasets by name.""" # Use fallback data to avoid network dependency with patch("diff_diff.datasets._download_with_cache") as mock: - mock.side_effect = RuntimeError("No network") - df = load_dataset("card_krueger") + from diff_diff.datasets import _DatasetSourceError + + mock.side_effect = _DatasetSourceError("No network") + with pytest.warns(UserWarning, match="SYNTHETIC"): + df = load_dataset("card_krueger") assert isinstance(df, pd.DataFrame) def test_load_by_name_binary(self): @@ -125,9 +131,13 @@ def test_fallback_data_values(self): def test_load_uses_fallback_on_network_error(self): """load_card_krueger should use fallback when network fails.""" with patch("diff_diff.datasets._download_with_cache") as mock: - mock.side_effect = RuntimeError("Network error") - df = load_card_krueger() + from diff_diff.datasets import _DatasetSourceError + + mock.side_effect = _DatasetSourceError("Network error") + with pytest.warns(UserWarning, match="SYNTHETIC"): + df = load_card_krueger() assert isinstance(df, pd.DataFrame) + assert df.attrs["source"] == "synthetic_fallback" assert "treated" in df.columns @@ -242,6 +252,258 @@ def test_fallback_data_size(self): assert df["countyreal"].nunique() == 500 +class TestLegacyLoaderProvenance: + """Legacy loaders must never silently present synthetic data as canonical.""" + + LOADERS = ( + ( + load_card_krueger, + _construct_card_krueger_data, + "_load_card_krueger_source", + "card_krueger_public_data", + ), + ( + load_castle_doctrine, + _construct_castle_doctrine_data, + "_load_castle_doctrine_source", + "cheng_hoekstra_castle_data", + ), + ( + load_divorce_laws, + _construct_divorce_laws_data, + None, + None, + ), + ( + load_mpdta, + _construct_mpdta_data, + "_load_mpdta_source", + "callaway_santanna_mpdta", + ), + ) + + @staticmethod + def _valid_card_source_frame(): + n = 410 + df = pd.DataFrame( + { + "store_id": np.arange(1, n + 1), + "state": ["NJ"] * 331 + ["PA"] * 79, + "chain": np.resize(["bk", "kfc", "roys", "wendys"], n), + "emp_pre": np.full(n, 20.0), + "emp_post": np.full(n, 21.0), + "wage_pre": np.full(n, 4.5), + "wage_post": np.full(n, 5.0), + } + ) + df.loc[:11, "emp_pre"] = np.nan + df.loc[12:25, "emp_post"] = np.nan + df.loc[:19, "wage_pre"] = np.nan + df.loc[20:40, "wage_post"] = np.nan + df["treated"] = (df["state"] == "NJ").astype(int) + df["emp_change"] = df["emp_post"] - df["emp_pre"] + return df + + @staticmethod + def _valid_castle_source_frame(): + import diff_diff.datasets as datasets_mod + + states = [code for code in datasets_mod._CASTLE_STATE_BY_SID.values() if code != "_"] + cohorts = dict(zip(states[:5], [2005, 2006, 2007, 2008, 2009])) + rows = [] + for state in states: + first_treat = cohorts.get(state, 0) + for year in range(2000, 2011): + rows.append( + { + "state": state, + "year": year, + "first_treat": first_treat, + "homicide_rate": 5.0, + "population": 1_000_000, + "income": 40_000, + "treated": int(first_treat > 0 and year >= first_treat), + "cohort": first_treat, + } + ) + return pd.DataFrame(rows) + + @pytest.mark.parametrize(("loader", "fallback", "source_loader", "_source"), LOADERS) + def test_network_failure_warns_and_marks_synthetic_fallback( + self, loader, fallback, source_loader, _source, monkeypatch + ): + import diff_diff.datasets as datasets_mod + + if source_loader is not None: + monkeypatch.setattr( + datasets_mod, + source_loader, + MagicMock(side_effect=datasets_mod._DatasetSourceError("Network error")), + ) + with pytest.warns(UserWarning, match="SYNTHETIC") as caught: + result = loader() + + assert len(caught) == 1 + assert result.attrs["source"] == "synthetic_fallback" + assert result.shape == fallback().shape + + @pytest.mark.parametrize(("loader", "fallback", "source_loader", "_source"), LOADERS) + def test_malformed_download_warns_and_uses_marked_fallback( + self, loader, fallback, source_loader, _source, monkeypatch + ): + import diff_diff.datasets as datasets_mod + + if source_loader is not None: + monkeypatch.setattr( + datasets_mod, + source_loader, + lambda _force_download: pd.DataFrame({"bad": [1]}), + ) + with pytest.warns(UserWarning, match="SYNTHETIC") as caught: + result = loader() + + assert len(caught) == 1 + assert result.attrs["source"] == "synthetic_fallback" + assert result.shape == fallback().shape + + @pytest.mark.parametrize( + ("loader", "fallback", "source_loader", "source"), + [case for case in LOADERS if case[2] is not None], + ) + def test_verified_download_is_marked_with_source( + self, loader, fallback, source_loader, source, monkeypatch + ): + import diff_diff.datasets as datasets_mod + + valid_frame = { + load_card_krueger: self._valid_card_source_frame, + load_castle_doctrine: self._valid_castle_source_frame, + load_mpdta: _construct_mpdta_data, + }[loader]() + monkeypatch.setattr( + datasets_mod, + source_loader, + lambda _force_download: valid_frame, + ) + result = loader() + + assert result.attrs["source"] == source + + def test_source_specific_dimensions_are_enforced(self): + """Synthetic frames cannot pass as Card or Castle canonical data.""" + from diff_diff.datasets import ( + _validate_card_krueger_source, + _validate_castle_doctrine_source, + ) + + with pytest.raises(RuntimeError, match="410 stores"): + _validate_card_krueger_source(_construct_card_krueger_data()) + with pytest.raises(RuntimeError, match="50 states and 550 rows"): + _validate_castle_doctrine_source(_construct_castle_doctrine_data()) + + def test_card_source_transform_uses_fte_and_stable_duplicate_ids(self): + """The public flat-file projection follows the published FTE formula.""" + from diff_diff.datasets import _prepare_card_krueger + + raw = pd.DataFrame( + { + "sheet": [407, 407], + "state": [0, 1], + "chain": [2, 4], + "empft": [2.0, 5.0], + "emppt": [10.0, 8.0], + "nmgrs": [1.0, 2.0], + "wage_st": [4.75, 5.75], + "empft2": [1.0, 8.0], + "emppt2": [12.0, 6.0], + "nmgrs2": [2.0, 2.0], + "wage_st2": [4.25, 5.50], + } + ) + + result = _prepare_card_krueger(raw) + + assert result["store_id"].tolist() == [407, 408] + assert result["state"].tolist() == ["PA", "NJ"] + assert result["chain"].tolist() == ["kfc", "wendys"] + assert result["emp_pre"].tolist() == [8.0, 11.0] + assert result["emp_post"].tolist() == [9.0, 13.0] + + def test_castle_source_transform_ignores_fractional_cdl(self): + """The public treated field is binary, not fractional-year exposure.""" + from diff_diff.datasets import _prepare_castle_doctrine + + raw = pd.DataFrame( + { + "state": ["Alabama", "Alabama"], + "sid": [1, 1], + "year": [2005, 2006], + "effyear": [2006.0, 2006.0], + "cdl": [0.0, 0.580822], + "homicide": [7.0, 7.5], + "population": [4_300_000, 4_350_000], + "income": [44_000, 45_000], + } + ) + + result = _prepare_castle_doctrine(raw) + + assert result["state"].tolist() == ["AL", "AL"] + assert result["treated"].tolist() == [0, 1] + assert result["first_treat"].tolist() == [2006, 2006] + + def test_card_source_rejects_unknown_chain_code(self): + from diff_diff.datasets import _DatasetSourceError, _prepare_card_krueger + + raw = pd.DataFrame( + { + "sheet": [407, 407], + "state": [0, 1], + "chain": [2, 99], + "empft": [2.0, 5.0], + "emppt": [10.0, 8.0], + "nmgrs": [1.0, 2.0], + "wage_st": [4.75, 5.75], + "empft2": [1.0, 8.0], + "emppt2": [12.0, 6.0], + "nmgrs2": [2.0, 2.0], + "wage_st2": [4.25, 5.50], + } + ) + + with pytest.raises(_DatasetSourceError, match="unknown chain"): + _prepare_card_krueger(raw) + + def test_semantically_invalid_source_values_are_rejected(self): + from diff_diff.datasets import ( + _DatasetSourceError, + _validate_card_krueger_source, + _validate_castle_doctrine_source, + _validate_mpdta, + ) + + card = self._valid_card_source_frame() + card.loc[100, "emp_change"] += 1 + with pytest.raises(_DatasetSourceError, match="emp_change"): + _validate_card_krueger_source(card) + + card = self._valid_card_source_frame() + card.loc[100, "emp_pre"] = np.inf + card.loc[100, "emp_change"] = card.loc[100, "emp_post"] - np.inf + with pytest.raises(_DatasetSourceError, match="emp_pre"): + _validate_card_krueger_source(card) + + castle = self._valid_castle_source_frame() + castle.loc[0, "homicide_rate"] = -1 + with pytest.raises(_DatasetSourceError, match="invalid outcome"): + _validate_castle_doctrine_source(castle) + + mpdta = _construct_mpdta_data() + mpdta["lemp"] = np.nan + with pytest.raises(_DatasetSourceError, match="missing required"): + _validate_mpdta(mpdta) + + class TestProp99: """Tests for California Prop 99 smoking dataset.""" @@ -514,6 +776,123 @@ def test_stale_cache_triggers_redownload(self, tmp_path, monkeypatch): assert (tmp_path / "x.dta").read_bytes() == good +class TestCsvDownloadIntegrity: + """CSV downloads receive the same trust-on-bytes contract as binary data.""" + + def test_checksum_mismatch_raises_without_caching(self, tmp_path, monkeypatch): + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + + fake_response = MagicMock() + fake_response.read.return_value = b"tampered bytes" + fake_response.__enter__ = lambda self: self + fake_response.__exit__ = lambda self, *a: False + + with patch("diff_diff.datasets.urlopen", return_value=fake_response): + with pytest.raises(RuntimeError, match="Checksum mismatch"): + datasets_mod._download_with_cache( + "https://example.invalid/x.csv", + "x", + sha256="0" * 64, + ) + + assert not (tmp_path / "x.csv").exists() + + def test_stale_cache_triggers_verified_redownload(self, tmp_path, monkeypatch): + import hashlib + + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + good = b"a,b\n1,2\n" + good_sha = hashlib.sha256(good).hexdigest() + (tmp_path / "x.csv").write_bytes(b"stale bytes") + + fake_response = MagicMock() + fake_response.read.return_value = good + fake_response.__enter__ = lambda self: self + fake_response.__exit__ = lambda self, *a: False + + with patch("diff_diff.datasets.urlopen", return_value=fake_response): + content = datasets_mod._download_with_cache( + "https://example.invalid/x.csv", + "x", + sha256=good_sha, + ) + + assert content == good.decode("utf-8") + assert (tmp_path / "x.csv").read_bytes() == good + + def test_oversized_response_is_rejected_without_caching(self, tmp_path, monkeypatch): + """A source cannot bypass checksum handling with an unbounded response.""" + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + monkeypatch.setattr(datasets_mod, "_MAX_DATASET_BYTES", 4) + fake_response = MagicMock() + fake_response.read.return_value = b"12345" + fake_response.__enter__ = lambda self: self + fake_response.__exit__ = lambda self, *a: False + + with patch("diff_diff.datasets.urlopen", return_value=fake_response): + with pytest.raises(RuntimeError, match="safety limit"): + datasets_mod._download_with_cache( + "https://example.invalid/x.csv", + "x", + sha256="0" * 64, + ) + + assert not (tmp_path / "x.csv").exists() + + def test_oversized_cache_is_not_read(self, tmp_path, monkeypatch): + """An oversized local cache cannot bypass the response-size limit.""" + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + monkeypatch.setattr(datasets_mod, "_MAX_DATASET_BYTES", 4) + (tmp_path / "x.csv").write_bytes(b"12345") + + with patch("diff_diff.datasets.urlopen", side_effect=TimeoutError("offline")): + with pytest.raises(datasets_mod._DatasetSourceError, match="Failed to download"): + datasets_mod._download_with_cache( + "https://example.invalid/x.csv", + "x", + sha256="0" * 64, + ) + + def test_failed_atomic_replace_preserves_existing_cache(self, tmp_path, monkeypatch): + """An interrupted replacement must not truncate a valid cache entry.""" + import hashlib + + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + good = b"a,b\n1,2\n" + good_sha = hashlib.sha256(good).hexdigest() + cache_path = tmp_path / "x.csv" + cache_path.write_bytes(good) + fake_response = MagicMock() + fake_response.read.return_value = good + fake_response.__enter__ = lambda self: self + fake_response.__exit__ = lambda self, *a: False + + with ( + patch("diff_diff.datasets.urlopen", return_value=fake_response), + patch("diff_diff.datasets.os.replace", side_effect=OSError("interrupted")), + ): + with pytest.raises(datasets_mod._DatasetSourceError, match="Failed to cache"): + datasets_mod._download_with_cache( + "https://example.invalid/x.csv", + "x", + sha256=good_sha, + force_download=True, + ) + + assert cache_path.read_bytes() == good + assert list(tmp_path.glob(".x.csv.*")) == [] + + class TestClearCache: """Tests for cache management.""" diff --git a/tests/test_doc_snippets.py b/tests/test_doc_snippets.py index efc1e58a2..f9df47a18 100644 --- a/tests/test_doc_snippets.py +++ b/tests/test_doc_snippets.py @@ -345,7 +345,7 @@ def _mock_list_datasets(): "card_krueger": "Card & Krueger (1994) minimum wage dataset", "castle_doctrine": "Castle Doctrine laws - staggered adoption", "divorce_laws": "Unilateral divorce laws - staggered adoption", - "mpdta": "Minimum wage panel data - simulated CS example", + "mpdta": "County teen-employment panel - Callaway-Sant'Anna example", "prop99": "California Prop 99 smoking panel - single treated unit", "walmart": "Walmart entry county panel - staggered adoption", } diff --git a/tests/test_methodology_callaway.py b/tests/test_methodology_callaway.py index 0aa3c91ac..34655bbf9 100644 --- a/tests/test_methodology_callaway.py +++ b/tests/test_methodology_callaway.py @@ -1655,8 +1655,9 @@ class TestMPDTARComparison: the same data exported from R. We export R's mpdta dataset to a temp file and load it in Python to ensure identical input data. - Note: Python's load_mpdta() downloads from a different source than R's - packaged data, which has different values. These tests use R's data directly. + ``load_mpdta()`` now uses a checksum-pinned mirror matching R's packaged data. + These tests still export from R directly to preserve an independent + cross-language input and result path. Expected tolerances (based on benchmark analysis): - Overall ATT: <1% difference From f9b875b0f3235f136579f066331003c430fbebf0 Mon Sep 17 00:00:00 2001 From: Charles Shaw Date: Thu, 23 Jul 2026 14:55:31 +0100 Subject: [PATCH 02/17] fix: preserve verified dataset downloads on cache write failure --- CHANGELOG.md | 3 ++- diff_diff/datasets.py | 14 +++++++++--- tests/test_datasets.py | 49 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a2a004f..627a5fbd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -403,7 +403,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 exactly one `UserWarning` containing `SYNTHETIC` and mark the fallback as `source="synthetic_fallback"`. `load_divorce_laws()` now follows that explicit fallback path because no verified source currently reproduces its composite - public schema without additional analytical choices. + public schema without additional analytical choices. Verified fresh downloads + are returned even when best-effort cache persistence fails. ### Changed - **dCDH Phase 1 (`L_max=None`) event-study row is now re-synced by the final-df diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 4bce8c22e..8411df702 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -159,7 +159,12 @@ def _download_verified_bytes( "treat the download as untrusted." ) - _write_cache_atomically(cache_path, content, name) + try: + _write_cache_atomically(cache_path, content, name) + except _DatasetSourceError: + # Cache persistence is best-effort after the downloaded bytes have + # already passed the pinned SHA-256 check. + pass return content @@ -213,7 +218,7 @@ def _load_verified_dataset( "SYNTHETIC fallback. Check `df.attrs['source']` before treating " "the result as replication data.", UserWarning, - stacklevel=2, + stacklevel=3, ) df = fallback() validate_fallback(df) @@ -1682,7 +1687,10 @@ def list_datasets() -> Dict[str, str]: return { "card_krueger": "Card & Krueger (1994) minimum wage dataset - classic 2x2 DiD", "castle_doctrine": "Castle Doctrine laws - staggered adoption across states", - "divorce_laws": "Unilateral divorce laws - staggered adoption (Stevenson-Wolfers)", + "divorce_laws": ( + "Unilateral divorce laws - synthetic fallback only; no verified " + "Stevenson-Wolfers source is configured" + ), "mpdta": "County teen-employment panel - Callaway-Sant'Anna example from R `did`", "prop99": "California Prop 99 smoking panel - single treated unit (Lee-Wooldridge format)", "walmart": "Walmart entry county panel - staggered adoption (Lee-Wooldridge sample)", diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 19dadfaf2..e4a45da82 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -58,6 +58,11 @@ def test_descriptions_are_strings(self): assert isinstance(desc, str) assert len(desc) > 0 + def test_divorce_laws_catalogue_marks_synthetic_only(self): + """Discovery metadata should not imply divorce_laws is canonical data.""" + result = list_datasets() + assert "synthetic fallback only" in result["divorce_laws"] + class TestLoadDataset: """Tests for load_dataset function.""" @@ -344,6 +349,7 @@ def test_network_failure_warns_and_marks_synthetic_fallback( result = loader() assert len(caught) == 1 + assert caught[0].filename.endswith("test_datasets.py") assert result.attrs["source"] == "synthetic_fallback" assert result.shape == fallback().shape @@ -389,6 +395,31 @@ def test_verified_download_is_marked_with_source( assert result.attrs["source"] == source + def test_verified_download_survives_cache_write_failure(self, tmp_path, monkeypatch): + """A verified mpdta download should be returned even if caching fails.""" + import hashlib + + import diff_diff.datasets as datasets_mod + + content = _construct_mpdta_data().to_csv(index=False).encode("utf-8") + sha256 = hashlib.sha256(content).hexdigest() + fake_response = MagicMock() + fake_response.read.return_value = content + fake_response.__enter__ = lambda self: self + fake_response.__exit__ = lambda self, *a: False + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + monkeypatch.setattr(datasets_mod, "_MPDTA_SOURCE_SHA256", sha256) + with ( + patch("diff_diff.datasets.urlopen", return_value=fake_response), + patch("diff_diff.datasets.os.replace", side_effect=OSError("disk full")), + ): + result = load_mpdta(force_download=True) + + assert result.attrs["source"] == "callaway_santanna_mpdta" + assert result.shape == _construct_mpdta_data().shape + assert not (tmp_path / "mpdta.csv").exists() + def test_source_specific_dimensions_are_enforced(self): """Synthetic frames cannot pass as Card or Castle canonical data.""" from diff_diff.datasets import ( @@ -861,8 +892,8 @@ def test_oversized_cache_is_not_read(self, tmp_path, monkeypatch): sha256="0" * 64, ) - def test_failed_atomic_replace_preserves_existing_cache(self, tmp_path, monkeypatch): - """An interrupted replacement must not truncate a valid cache entry.""" + def test_failed_atomic_replace_returns_verified_download(self, tmp_path, monkeypatch): + """An interrupted replacement must not discard verified fresh bytes.""" import hashlib import diff_diff.datasets as datasets_mod @@ -881,14 +912,14 @@ def test_failed_atomic_replace_preserves_existing_cache(self, tmp_path, monkeypa patch("diff_diff.datasets.urlopen", return_value=fake_response), patch("diff_diff.datasets.os.replace", side_effect=OSError("interrupted")), ): - with pytest.raises(datasets_mod._DatasetSourceError, match="Failed to cache"): - datasets_mod._download_with_cache( - "https://example.invalid/x.csv", - "x", - sha256=good_sha, - force_download=True, - ) + content = datasets_mod._download_with_cache( + "https://example.invalid/x.csv", + "x", + sha256=good_sha, + force_download=True, + ) + assert content == good.decode("utf-8") assert cache_path.read_bytes() == good assert list(tmp_path.glob(".x.csv.*")) == [] From d730c0b513cc05526fa5c5c36edbbb41a5991119 Mon Sep 17 00:00:00 2001 From: Charles Shaw Date: Mon, 27 Jul 2026 10:37:16 +0100 Subject: [PATCH 03/17] fix: complete dataset loader provenance safeguards --- CHANGELOG.md | 4 ++ diff_diff/datasets.py | 28 ++++++++++--- diff_diff/guides/llms-full.txt | 2 +- tests/test_datasets.py | 76 +++++++++++++++++++++++++++++++++- 4 files changed, 102 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 627a5fbd7..0c34ac914 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -407,6 +407,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 are returned even when best-effort cache persistence fails. ### Changed +- **Castle Doctrine now exposes first-year treatment intensity.** The loader + retains the source's fractional adoption-year `cdl` value as + `treatment_exposure`, while preserving the existing binary `treated` field. + - **dCDH Phase 1 (`L_max=None`) event-study row is now re-synced by the final-df refresh.** Under replicate-weight designs the refresh recomputes `overall_*` with the final effective df, but the Phase 1 event-study row is a VALUE COPY of that inference diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 8411df702..3b2bedecb 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -12,6 +12,7 @@ import hashlib import os import warnings +from http.client import IncompleteRead from io import BytesIO, StringIO from pathlib import Path from tempfile import NamedTemporaryFile @@ -64,7 +65,6 @@ class _DatasetSourceError(RuntimeError): def _get_cache_path(name: str) -> Path: """Get the cache path for a dataset.""" - _CACHE_DIR.mkdir(parents=True, exist_ok=True) return _CACHE_DIR / f"{name}.csv" @@ -102,6 +102,7 @@ def _write_cache_atomically(cache_path: Path, content: bytes, name: str) -> None """Replace a cache entry only after a complete same-directory write.""" temp_path: Optional[Path] = None try: + cache_path.parent.mkdir(parents=True, exist_ok=True) with NamedTemporaryFile( mode="wb", dir=cache_path.parent, @@ -136,7 +137,7 @@ def _download_verified_bytes( try: with urlopen(url, timeout=30) as response: content = response.read(_MAX_DATASET_BYTES + 1) - except (HTTPError, OSError, TimeoutError, URLError) as e: + except (HTTPError, IncompleteRead, OSError, TimeoutError, URLError) as e: cached = _read_verified_cache(cache_path, sha256) if cached is not None: return cached @@ -170,7 +171,6 @@ def _download_verified_bytes( def _get_cache_path_binary(name: str) -> Path: """Get the cache path for a binary (Stata .dta) dataset.""" - _CACHE_DIR.mkdir(parents=True, exist_ok=True) return _CACHE_DIR / f"{name}.dta" @@ -489,6 +489,8 @@ def _prepare_castle_doctrine(df: pd.DataFrame) -> pd.DataFrame: df["cohort"] = df["first_treat"] if {"first_treat", "year"} <= set(df.columns): df["treated"] = ((df["first_treat"] > 0) & (df["year"] >= df["first_treat"])).astype(int) + if "treatment_exposure" not in df.columns and "cdl" in df.columns: + df["treatment_exposure"] = df["cdl"] if "homicide_rate" not in df.columns and "homicide" in df.columns: df["homicide_rate"] = df["homicide"] if { @@ -499,6 +501,7 @@ def _prepare_castle_doctrine(df: pd.DataFrame) -> pd.DataFrame: "population", "income", "treated", + "treatment_exposure", "cohort", } <= set(df.columns): return df[ @@ -510,6 +513,7 @@ def _prepare_castle_doctrine(df: pd.DataFrame) -> pd.DataFrame: "population", "income", "treated", + "treatment_exposure", "cohort", ] ].copy() @@ -529,6 +533,7 @@ def _validate_castle_doctrine(df: pd.DataFrame) -> None: "population", "income", "treated", + "treatment_exposure", "cohort", }, ) @@ -549,7 +554,16 @@ def _validate_castle_doctrine(df: pd.DataFrame) -> None: _require_finite( df, "castle_doctrine", - {"year", "first_treat", "homicide_rate", "population", "income", "treated", "cohort"}, + { + "year", + "first_treat", + "homicide_rate", + "population", + "income", + "treated", + "treatment_exposure", + "cohort", + }, ) if ( (df["homicide_rate"] < 0).any() @@ -559,6 +573,8 @@ def _validate_castle_doctrine(df: pd.DataFrame) -> None: raise _DatasetSourceError("castle_doctrine source has invalid outcome or covariate values") if not df["state"].astype(str).str.fullmatch(r"[A-Z]{2}").all(): raise _DatasetSourceError("castle_doctrine source has invalid state abbreviations") + if not df["treatment_exposure"].between(0, 1).all(): + raise _DatasetSourceError("castle_doctrine treatment_exposure must be between 0 and 1") _validate_panel_keys(df, "castle_doctrine", "state") @@ -848,6 +864,7 @@ def load_castle_doctrine(force_download: bool = False) -> pd.DataFrame: - population : int - State population - income : float - Per capita income - treated : int - 1 if law in effect, 0 otherwise + - treatment_exposure : float - Fraction of the year the law was in effect - cohort : int - Alias for first_treat Notes @@ -992,6 +1009,7 @@ def _construct_castle_doctrine_data() -> pd.DataFrame: base_income * (1 + 0.02 * (year - 2000)) + np.random.normal(0, 1000), 0 ), "treated": int(first_treat > 0 and year >= first_treat), + "treatment_exposure": float(first_treat > 0 and year >= first_treat), } ) @@ -1002,7 +1020,7 @@ def _construct_castle_doctrine_data() -> pd.DataFrame: def load_divorce_laws(force_download: bool = False) -> pd.DataFrame: """ - Load unilateral divorce laws dataset. + Load the synthetic-only unilateral divorce-laws dataset. This dataset tracks the staggered adoption of unilateral (no-fault) divorce laws across U.S. states. It's a classic example for studying staggered diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 13e878096..27554a88e 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -2372,7 +2372,7 @@ data = load_dataset("card_krueger") # Named loaders ck = load_card_krueger() # Card & Krueger (1994) minimum wage castle = load_castle_doctrine() # Castle Doctrine / Stand Your Ground laws -divorce = load_divorce_laws() # Unilateral divorce laws (staggered) +divorce = load_divorce_laws() # Synthetic-only unilateral divorce-law fallback mpdta = load_mpdta() # County teen-employment panel from R did package prop99 = load_prop99() # California Prop 99 smoking (single treated unit) walmart = load_walmart() # Walmart entry county panel (staggered, 1,277 counties) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index e4a45da82..ee2995906 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -5,6 +5,7 @@ including both the download/cache mechanism and the fallback data generation. """ +from pathlib import Path from unittest.mock import MagicMock, patch import numpy as np @@ -328,6 +329,7 @@ def _valid_castle_source_frame(): "population": 1_000_000, "income": 40_000, "treated": int(first_treat > 0 and year >= first_treat), + "treatment_exposure": float(first_treat > 0 and year >= first_treat), "cohort": first_treat, } ) @@ -420,6 +422,54 @@ def test_verified_download_survives_cache_write_failure(self, tmp_path, monkeypa assert result.shape == _construct_mpdta_data().shape assert not (tmp_path / "mpdta.csv").exists() + def test_verified_download_survives_unwritable_cache_directory(self, tmp_path, monkeypatch): + """A cache-directory failure must not discard verified download bytes.""" + import hashlib + + import diff_diff.datasets as datasets_mod + + content = _construct_mpdta_data().to_csv(index=False).encode("utf-8") + cache_dir = tmp_path / "unwritable-cache" + sha256 = hashlib.sha256(content).hexdigest() + fake_response = MagicMock() + fake_response.read.return_value = content + fake_response.__enter__ = lambda self: self + fake_response.__exit__ = lambda self, *a: False + original_mkdir = Path.mkdir + + def deny_cache_directory(path, *args, **kwargs): + if path == cache_dir: + raise PermissionError("read-only cache directory") + return original_mkdir(path, *args, **kwargs) + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", cache_dir) + monkeypatch.setattr(datasets_mod, "_MPDTA_SOURCE_SHA256", sha256) + monkeypatch.setattr(Path, "mkdir", deny_cache_directory) + with patch("diff_diff.datasets.urlopen", return_value=fake_response): + result = load_mpdta(force_download=True) + + assert result.attrs["source"] == "callaway_santanna_mpdta" + assert result.shape == _construct_mpdta_data().shape + assert not cache_dir.exists() + + def test_incomplete_download_warns_and_uses_marked_fallback(self, tmp_path, monkeypatch): + """Truncated responses must follow the same explicit fallback path.""" + from http.client import IncompleteRead + + import diff_diff.datasets as datasets_mod + + fake_response = MagicMock() + fake_response.read.side_effect = IncompleteRead(b"partial", 100) + fake_response.__enter__ = lambda self: self + fake_response.__exit__ = lambda self, *a: False + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + with patch("diff_diff.datasets.urlopen", return_value=fake_response): + with pytest.warns(UserWarning, match="SYNTHETIC"): + result = load_mpdta(force_download=True) + + assert result.attrs["source"] == "synthetic_fallback" + def test_source_specific_dimensions_are_enforced(self): """Synthetic frames cannot pass as Card or Castle canonical data.""" from diff_diff.datasets import ( @@ -460,8 +510,8 @@ def test_card_source_transform_uses_fte_and_stable_duplicate_ids(self): assert result["emp_pre"].tolist() == [8.0, 11.0] assert result["emp_post"].tolist() == [9.0, 13.0] - def test_castle_source_transform_ignores_fractional_cdl(self): - """The public treated field is binary, not fractional-year exposure.""" + def test_castle_source_transform_preserves_fractional_cdl_exposure(self): + """The binary treatment flag and fractional source exposure are distinct.""" from diff_diff.datasets import _prepare_castle_doctrine raw = pd.DataFrame( @@ -481,8 +531,30 @@ def test_castle_source_transform_ignores_fractional_cdl(self): assert result["state"].tolist() == ["AL", "AL"] assert result["treated"].tolist() == [0, 1] + assert result["treatment_exposure"].tolist() == [0.0, 0.580822] assert result["first_treat"].tolist() == [2006, 2006] + def test_castle_adoption_year_has_binary_treatment_and_partial_exposure(self): + """Real-source adoption rows retain their partial first-year exposure.""" + from diff_diff.datasets import _prepare_castle_doctrine + + raw = pd.DataFrame( + { + "sid": [1] * 11, + "year": list(range(2000, 2011)), + "effyear": [2006.0] * 11, + "cdl": [0.0] * 6 + [0.580822] + [1.0] * 4, + "homicide": [7.0] * 11, + "population": [4_300_000] * 11, + "income": [44_000] * 11, + } + ) + result = _prepare_castle_doctrine(raw) + adoption_year = result.loc[result["year"] == 2006].iloc[0] + + assert adoption_year["treated"] == 1 + assert 0 < adoption_year["treatment_exposure"] < 1 + def test_card_source_rejects_unknown_chain_code(self): from diff_diff.datasets import _DatasetSourceError, _prepare_card_krueger From 5270086d21c95fbc8c05bb265c7a19dd53e69ce1 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 08:04:30 -0400 Subject: [PATCH 04/17] docs: castle treatment-coding registry note + dataset provenance doc refresh Records the Cheng-Hoekstra (2013) treatment-variable convention in REGISTRY: the paper's regressor is CDL_it, the proportion of the year the law was in effect (p. 830), preserved verbatim as treatment_exposure. The binary treated column follows the year >= first_treat staggered-DiD convention, which matches neither the paper's main specification nor its drop-the-adoption-year robustness check, and differs from the source file's own post indicator on 21 of 550 rows. Also refreshes surfaces the loader rewrite left stale: the datasets.rst intro still described the pre-checksum cache-fallback behaviour, the doc-snippet stub catalog still described divorce_laws as staggered-adoption data, and the backlog row this work closes was still open. --- TODO.md | 1 - diff_diff/datasets.py | 16 ++++++++++++++-- docs/api/datasets.rst | 19 +++++++++++++------ docs/methodology/REGISTRY.md | 16 ++++++++++++++++ tests/test_doc_snippets.py | 2 +- 5 files changed, 44 insertions(+), 10 deletions(-) diff --git a/TODO.md b/TODO.md index 6621adf7d..21e052604 100644 --- a/TODO.md +++ b/TODO.md @@ -56,7 +56,6 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | Issue | Location | Origin | Effort | Priority | |-------|----------|--------|--------|----------| -| Align the four legacy dataset loaders (`load_card_krueger`, `load_castle_doctrine`, `load_divorce_laws`, `load_mpdta`) with the loud-fallback pattern of `load_prop99`/`load_walmart`: `UserWarning` + `df.attrs["source"]` marker on synthetic fallback (currently silent), plus optional checksum pinning for the CSV downloads. **Upgraded to a live defect 2026-07-13: the `causaldata/causal_datasets` GitHub repo backing castle/card_krueger/divorce is dead (404), so those loaders silently serve synthetic data everywhere - needs loud fallback + replacement sources.** | `diff_diff/datasets.py` | LWDiD precursor | Quick | Medium | | Tighten the mypy suppressions that back the enforced-zero posture: burn down `prep_dgp`'s per-module `[index]` override (needs a None-vs-array restructure that preserves the seeded RNG stream), and evaluate re-enabling the globally disabled codes (`arg-type`, `return-value`, `var-annotated`, `assignment`) one at a time — `assignment` alone hid several real annotation drifts found during the 2026-07 triage. | `pyproject.toml` `[tool.mypy]`, `diff_diff/prep_dgp.py` | lint-CI | Mid | Low | | MMM interop follow-up: Meridian `roi_calibration_period` mask builder - accept the MMM's time index + channel order and emit the boolean `(n_media_times, n_media_channels)` mask so `.to_code()` scopes the prior to the experiment window automatically (today the caller passes a mask expression / `full_model_window=True`). | `diff_diff/mmm.py` | mmm-interop | Quick | Low | | MMM interop PR-B: calibration tutorial notebook (fit DiD/CS -> scope -> `to_pymc_marketing_lift_test` / `to_meridian_roi_prior`) + a `llms-practitioner.txt` Step 8 pointer to the exporters as the MMM hand-off. | `docs/tutorials/`, `diff_diff/guides/llms-practitioner.txt` | mmm-interop | Mid | Low | diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 3b2bedecb..86088dd26 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -27,8 +27,13 @@ _CACHE_DIR = Path.home() / ".cache" / "diff_diff" / "datasets" _MAX_DATASET_BYTES = 50 * 1024 * 1024 -# These commit-pinned mirrors were checked against the dataset authors' package -# artifacts. Every cached and downloaded byte sequence is verified below. +# These commit-pinned mirrors were verified byte-equivalent to their authoritative +# sources at pinning time: ``public.dat`` is byte-identical to the copy in +# ``njmin.zip`` from David Card's data archive, and ``mpdta.csv`` matches +# ``did::mpdta`` (R package 2.5.1) to CSV round-trip precision (max absolute +# difference 1.07e-14 on ``lemp``). Every cached and downloaded byte sequence is +# verified against the pinned SHA-256 below, so a later change at any mirror fails +# closed to the loud synthetic fallback rather than substituting different data. _CARD_KRUEGER_SOURCE_URL = ( "https://raw.githubusercontent.com/rafiash/CardKrueger-stata-sample/" "07bc929f1d6552db117bd27a7cf0d881d16e9494/public.dat" @@ -865,6 +870,8 @@ def load_castle_doctrine(force_download: bool = False) -> pd.DataFrame: - income : float - Per capita income - treated : int - 1 if law in effect, 0 otherwise - treatment_exposure : float - Fraction of the year the law was in effect + (canonical data only; synthetic fallback frames set this to a binary + 0/1 copy of ``treated``) - cohort : int - Alias for first_treat Notes @@ -878,6 +885,11 @@ def load_castle_doctrine(force_download: bool = False) -> pd.DataFrame: parse, or validation failure emits one ``UserWarning`` containing ``SYNTHETIC`` and marks the returned frame as ``"synthetic_fallback"``. + Replicating Cheng-Hoekstra (2013) requires the paper's regressor and outcome: + regress ``log(homicide_rate)`` on ``treatment_exposure`` (their ``CDL_it``, + the proportion of the year the law was in effect), not the binary ``treated``. + See "Castle Doctrine treatment coding" in ``docs/methodology/REGISTRY.md``. + References ---------- Cheng, C., & Hoekstra, M. (2013). Does Strengthening Self-Defense Law Deter diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index ccd694483..0c2ff8b1d 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -1,12 +1,19 @@ Datasets ======== -Built-in real-world datasets from published studies for examples, tutorials, and testing. - -All datasets are downloaded from public sources on first use and cached locally -at ``~/.cache/diff_diff/datasets/``. Pass ``force_download=True`` to any loader -to refresh the cache. If the download fails and a cached copy exists, the cached -version is used automatically. +Built-in datasets from published studies for examples, tutorials, and testing. + +Canonical data are downloaded from commit-pinned public sources and verified against a +pinned SHA-256 on every download and cache read, then cached locally at +``~/.cache/diff_diff/datasets/``. Pass ``force_download=True`` to any loader to refresh +the cache. + +Every loader reports its provenance through ``df.attrs["source"]``. If a source cannot be +downloaded, parsed, or validated, the loader emits one ``UserWarning`` containing +``SYNTHETIC`` and returns a generated fallback frame marked +``df.attrs["source"] == "synthetic_fallback"``. Numbers computed on a fallback frame are +not replications of the published study. ``load_divorce_laws()`` has no verified source +configured and returns synthetic data on every call. Dataset Loaders --------------- diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 036853aa8..853c26d37 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -5837,6 +5837,22 @@ estimator-focused: --- +# Replication Data Conventions + +## Castle Doctrine treatment coding (`load_castle_doctrine`) + +**Primary source:** Cheng, C., & Hoekstra, M. (2013). Does Strengthening Self-Defense Law Deter Crime or Escalate Violence? Evidence from Expansions to Castle Doctrine. *Journal of Human Resources*, 48(3), 821-854. + +**Module:** `diff_diff/datasets.py` + +**Scope:** Treatment-variable coding in `load_castle_doctrine()` only - no estimator, variance, or inference behavior. Loader provenance, checksum pinning, and the synthetic-fallback contract are documented in the loader docstrings and `docs/api/datasets.rst`. + +- **Note:** The paper's estimating equation is `Outcome_it = b0*CDL_it + b1*X_it + c_i + u_t + e_it`, "where `CDL_it` is the treatment variable that equals the proportion of year `t` in which state `i` has an effective Castle Doctrine law" (p. 830, Sec. III). For partial-year adopters it states: "we set CDL equal to the proportion of the year in which the law was in effect, though estimates are almost identical when we exclude the year of adoption" (fn. 21 reports the excluded-year estimates). That regressor is preserved verbatim as `treatment_exposure` (the source file's `cdl`). Verified in the file: 0 before adoption and for never-adopters, exactly 1.0 after, and a day-count fraction in the adoption year - `cdl * 365` is an exact integer for 17 of the 21 adopting states (59 to 306 days) and exactly 182.5 for the other four. +- **Note:** The binary `treated` column follows the standard staggered-DiD convention (`year >= first_treat`), so that `treated`, `first_treat`, and `cohort` stay mutually consistent - which `_validate_panel_keys` enforces and which the cohort estimators (`CallawaySantAnna`, `SunAbraham`) require. This convention matches **neither** of the paper's treatments of the adoption year: not the fractional `CDL_it` of the main specification, nor the drop-the-adoption-year robustness check. It also differs from the source file's own binary `post` indicator (0 in the adoption year, 1 thereafter) on exactly 21 of 550 rows. **Replicating Cheng-Hoekstra requires regressing on `treatment_exposure`, not `treated`.** Synthetic fallback frames carry `treatment_exposure` as a binary 0/1 copy of `treated` and therefore cannot support the paper's specification at all. +- **Note:** The paper's dependent variable is the log of the outcome per 100,000 population (p. 830). `homicide_rate` is the level (the source file's `homicide`, labelled "homicide count per 100,000 state population"), so a replication must take logs. The file's `l_homicide`, which is exactly `log(homicide)`, is not carried into the public schema. + +--- + # Version History - **v1.3** (2026-03-26): Added Replicate Weight Variance, DEFF Diagnostics, diff --git a/tests/test_doc_snippets.py b/tests/test_doc_snippets.py index f9df47a18..4e41ce6e4 100644 --- a/tests/test_doc_snippets.py +++ b/tests/test_doc_snippets.py @@ -344,7 +344,7 @@ def _mock_list_datasets(): return { "card_krueger": "Card & Krueger (1994) minimum wage dataset", "castle_doctrine": "Castle Doctrine laws - staggered adoption", - "divorce_laws": "Unilateral divorce laws - staggered adoption", + "divorce_laws": "Unilateral divorce laws - synthetic fallback only", "mpdta": "County teen-employment panel - Callaway-Sant'Anna example", "prop99": "California Prop 99 smoking panel - single treated unit", "walmart": "Walmart entry county panel - staggered adoption", From 35d1cf3ebd101abdee3df1768f2d6fd08176f444 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 08:16:41 -0400 Subject: [PATCH 05/17] fix: widen download fallback to HTTPException; scope datasets.rst provenance claims The download handler named IncompleteRead explicitly, but BadStatusLine, LineTooLong and the rest of http.client's protocol errors are siblings under HTTPException and none derive from OSError, so they escaped the documented warn-and-fall-back boundary and propagated out of the loaders. Catch the base class instead, with a regression test covering two additional subclasses. The datasets.rst provenance paragraph also overstated the contract: it claimed commit-pinned sources and parse/validation fallback for every loader, but load_prop99/load_walmart read a mutable SSC path over plain HTTP and run their validation outside the try/except, so a validation failure raises rather than falling back. Scoped both claims to the loaders that honour them. --- diff_diff/datasets.py | 8 ++++++-- docs/api/datasets.rst | 25 +++++++++++++++---------- tests/test_datasets.py | 25 +++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 86088dd26..4a84335a9 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -12,7 +12,7 @@ import hashlib import os import warnings -from http.client import IncompleteRead +from http.client import HTTPException from io import BytesIO, StringIO from pathlib import Path from tempfile import NamedTemporaryFile @@ -142,7 +142,11 @@ def _download_verified_bytes( try: with urlopen(url, timeout=30) as response: content = response.read(_MAX_DATASET_BYTES + 1) - except (HTTPError, IncompleteRead, OSError, TimeoutError, URLError) as e: + # ``HTTPException`` is the parent of ``IncompleteRead``, ``BadStatusLine`` and the + # rest of the protocol-level errors ``urlopen`` can surface; none of them derive + # from ``OSError``, so catching the base class keeps the whole family inside the + # documented warn-and-fall-back boundary rather than only the named siblings. + except (HTTPError, HTTPException, OSError, TimeoutError, URLError) as e: cached = _read_verified_cache(cache_path, sha256) if cached is not None: return cached diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index 0c2ff8b1d..3d2ba52fb 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -3,17 +3,22 @@ Datasets Built-in datasets from published studies for examples, tutorials, and testing. -Canonical data are downloaded from commit-pinned public sources and verified against a -pinned SHA-256 on every download and cache read, then cached locally at +Canonical data are downloaded from public sources and verified against a pinned SHA-256 +on every download and cache read, then cached locally at ``~/.cache/diff_diff/datasets/``. Pass ``force_download=True`` to any loader to refresh -the cache. - -Every loader reports its provenance through ``df.attrs["source"]``. If a source cannot be -downloaded, parsed, or validated, the loader emits one ``UserWarning`` containing -``SYNTHETIC`` and returns a generated fallback frame marked -``df.attrs["source"] == "synthetic_fallback"``. Numbers computed on a fallback frame are -not replications of the published study. ``load_divorce_laws()`` has no verified source -configured and returns synthetic data on every call. +the cache. ``load_card_krueger()``, ``load_castle_doctrine()`` and ``load_mpdta()`` pin an +immutable upstream commit, so the URL itself cannot change under them; +``load_prop99()`` and ``load_walmart()`` read a mutable SSC path over plain HTTP, where +the checksum alone establishes integrity. + +Every loader reports its provenance through ``df.attrs["source"]``. When a source cannot +be obtained, the loader emits one ``UserWarning`` containing ``SYNTHETIC`` and returns a +generated fallback frame marked ``df.attrs["source"] == "synthetic_fallback"``. For +``load_card_krueger()``, ``load_castle_doctrine()`` and ``load_mpdta()`` that fallback +also covers parse and schema-validation failures; ``load_prop99()`` and ``load_walmart()`` +run their validation outside the fallback boundary and raise instead. Numbers computed on +a fallback frame are not replications of the published study. ``load_divorce_laws()`` has +no verified source configured and returns synthetic data on every call. Dataset Loaders --------------- diff --git a/tests/test_datasets.py b/tests/test_datasets.py index ee2995906..ec5cde025 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -470,6 +470,31 @@ def test_incomplete_download_warns_and_uses_marked_fallback(self, tmp_path, monk assert result.attrs["source"] == "synthetic_fallback" + def test_protocol_level_http_errors_warn_and_use_marked_fallback(self, tmp_path, monkeypatch): + """Every ``HTTPException``, not just ``IncompleteRead``, stays inside the boundary. + + ``BadStatusLine`` and its siblings derive from ``HTTPException`` rather than + ``OSError``, so naming individual subclasses in the handler would let the rest + escape the documented warn-and-fall-back contract. + """ + from http.client import BadStatusLine, LineTooLong + + import diff_diff.datasets as datasets_mod + + for exc in (BadStatusLine("garbage status"), LineTooLong("header line")): + fake_response = MagicMock() + fake_response.read.side_effect = exc + fake_response.__enter__ = lambda self: self + fake_response.__exit__ = lambda self, *a: False + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path / type(exc).__name__) + with patch("diff_diff.datasets.urlopen", return_value=fake_response): + with pytest.warns(UserWarning, match="SYNTHETIC"): + result = load_mpdta(force_download=True) + + assert result.attrs["source"] == "synthetic_fallback" + assert result.shape == _construct_mpdta_data().shape + def test_source_specific_dimensions_are_enforced(self): """Synthetic frames cannot pass as Card or Castle canonical data.""" from diff_diff.datasets import ( From aa4eb6092807cad8ac933ddb64ee2a801b102601 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 08:28:03 -0400 Subject: [PATCH 06/17] fix: clean up partial cache files on write failure; align stale dataset prose _write_cache_atomically bound temp_path only after the write succeeded, so a mid-write failure (ENOSPC, quota) left the delete=False temporary file stranded in the cache directory, and repeated attempts accumulated them. Bind the path at creation instead, with a regression test that fails the write and asserts an empty cache directory. Also aligns per-dataset prose left contradicting the new provenance contract: the datasets.rst divorce section presented always-synthetic data as Stevenson & Wolfers, the mpdta section still called the now-canonical R data 'simulated', list_datasets() advertised 'real-world datasets', and _construct_mpdta_data's docstring claimed to match the R package it only approximates. --- diff_diff/datasets.py | 13 +++++++++---- docs/api/datasets.rst | 14 ++++++++------ tests/test_datasets.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 4a84335a9..685a478c0 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -114,8 +114,11 @@ def _write_cache_atomically(cache_path: Path, content: bytes, name: str) -> None prefix=f".{cache_path.name}.", delete=False, ) as temp_file: - temp_file.write(content) + # Bind the path before writing: ``delete=False`` means a failed write + # still leaves the file on disk, and the handler below can only clean + # up what it knows about. temp_path = Path(temp_file.name) + temp_file.write(content) os.replace(temp_path, cache_path) except OSError as e: if temp_path is not None: @@ -1304,9 +1307,11 @@ def load_mpdta(force_download: bool = False) -> pd.DataFrame: def _construct_mpdta_data() -> pd.DataFrame: """ - Construct mpdta dataset matching the R `did` package. + Construct a synthetic stand-in for the mpdta dataset. - This replicates the simulated dataset used in Callaway-Sant'Anna tutorials. + Mirrors the schema and panel dimensions of the R `did` package's ``mpdta`` + (500 counties, 2003-2007, cohorts 2004/2006/2007) with generated values. It + is NOT the canonical data and must not be used for replication. """ np.random.seed(2021) # Callaway-Sant'Anna publication year, for reproducibility @@ -1705,7 +1710,7 @@ def _construct_walmart_data() -> pd.DataFrame: def list_datasets() -> Dict[str, str]: """ - List available real-world datasets. + List available built-in datasets. Returns ------- diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index 3d2ba52fb..bfe31f898 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -83,9 +83,11 @@ Example load_divorce_laws ~~~~~~~~~~~~~~~~~ -Unilateral (no-fault) divorce law reforms. Staggered adoption across U.S. -states (1968--1988) from Stevenson & Wolfers (2006), with outcomes for divorce -rate, female labor force participation, and female suicide rate. +Always-synthetic teaching panel of unilateral (no-fault) divorce law reforms, +modelled on the staggered adoption studied by Stevenson & Wolfers (2006), with +outcomes for divorce rate, female labor force participation, and female suicide +rate. No verified source is configured for this loader, so it warns and returns +generated data on every call and is not suitable for replication. .. autofunction:: diff_diff.load_divorce_laws @@ -110,9 +112,9 @@ Example load_mpdta ~~~~~~~~~~ -Minimum wage panel data for training (Callaway & Sant'Anna 2021). Simulated -county-level employment data with staggered minimum wage increases (2003--2007), -from the R ``did`` package. +County teen-employment panel (Callaway & Sant'Anna 2021): the canonical +2,500-row ``mpdta`` dataset distributed with the R ``did`` package, covering +staggered minimum wage increases (2003--2007). .. autofunction:: diff_diff.load_mpdta diff --git a/tests/test_datasets.py b/tests/test_datasets.py index ec5cde025..bbee61b2e 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -495,6 +495,40 @@ def test_protocol_level_http_errors_warn_and_use_marked_fallback(self, tmp_path, assert result.attrs["source"] == "synthetic_fallback" assert result.shape == _construct_mpdta_data().shape + def test_failed_cache_write_leaves_no_partial_file(self, tmp_path, monkeypatch): + """A write that fails mid-flight must not strand a ``delete=False`` temp file.""" + import diff_diff.datasets as datasets_mod + + real_ntf = datasets_mod.NamedTemporaryFile + + class FailingWrite: + """Create the temp file for real, then fail the write (e.g. ENOSPC).""" + + def __init__(self, *args, **kwargs): + self._f = real_ntf(*args, **kwargs) + + def __enter__(self): + self._f.__enter__() + return self + + def __exit__(self, *args): + return self._f.__exit__(*args) + + @property + def name(self): + return self._f.name + + def write(self, data): + raise OSError(28, "No space left on device") + + cache_path = tmp_path / "mpdta.csv" + monkeypatch.setattr(datasets_mod, "NamedTemporaryFile", FailingWrite) + + with pytest.raises(datasets_mod._DatasetSourceError, match="Failed to cache"): + datasets_mod._write_cache_atomically(cache_path, b"x" * 100, "mpdta") + + assert list(tmp_path.iterdir()) == [], "partial cache file was not cleaned up" + def test_source_specific_dimensions_are_enforced(self): """Synthetic frames cannot pass as Card or Castle canonical data.""" from diff_diff.datasets import ( From 90d10edfb33d52f3c6fdbf39293aca9610e47e0a Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 08:43:10 -0400 Subject: [PATCH 07/17] fix: correct castle income label, float32 exactness claim, and LLM-guide provenance The castle income field is documented as per-capita income, but the pinned Stata metadata labels it 'state median income'; the values were never wrong, only the description. (Pre-existing, but now materially misleading since the loader ships the real data.) The REGISTRY note claimed cdl*365 'is an exact integer' for the 17 non-half adoption-year exposures. It is not: cdl is stored as float32, so none of the 17 are bit-exact and the max deviation is 1.03e-5. Reworded to say the products recover integer day counts to within storage rounding. llms.txt still advertised the catalog as 'Real-world datasets' and listed divorce laws among them, and llms-full.txt's section heading likewise, so an agent reading the guide would treat synthetic divorce output as empirical. --- diff_diff/datasets.py | 2 +- diff_diff/guides/llms-full.txt | 2 +- diff_diff/guides/llms.txt | 2 +- docs/methodology/REGISTRY.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 685a478c0..c55757f90 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -874,7 +874,7 @@ def load_castle_doctrine(force_download: bool = False) -> pd.DataFrame: - first_treat : int - Year of law adoption (0 = never adopted) - homicide_rate : float - Homicides per 100,000 population - population : int - State population - - income : float - Per capita income + - income : float - State median income - treated : int - 1 if law in effect, 0 otherwise - treatment_exposure : float - Fraction of the year the law was in effect (canonical data only; synthetic fallback frames set this to a binary diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 27554a88e..6d168454e 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -2355,7 +2355,7 @@ data = generate_synthetic_control_data(n_donors=20, n_pre=60, n_post=5, effect_type="ramp", seed=0) ``` -## Real-World Datasets +## Built-in Datasets ```python from diff_diff import load_card_krueger, load_castle_doctrine, load_divorce_laws, load_mpdta diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index 1465a94dc..b2cc2d934 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -131,6 +131,6 @@ No R or Python package offers design-based variance estimation for modern hetero ## Optional - [Rust Backend](https://diff-diff.readthedocs.io/en/stable/benchmarks.html): Rust backend (bundled in released wheels; `maturin develop --release` for source builds) for order-of-magnitude speedups on compute-intensive estimators - 18-55x vs R synthdid on Synthetic DiD in the published benchmarks - plus TROP and bootstrap acceleration -- [Built-in Datasets](https://diff-diff.readthedocs.io/en/stable/api/datasets.html): Real-world datasets — Card & Krueger (1994), Castle Doctrine, divorce laws, MPDTA, California Prop 99 smoking (single treated unit), Walmart entry county panel (staggered) +- [Built-in Datasets](https://diff-diff.readthedocs.io/en/stable/api/datasets.html): Checksum-pinned replication data — Card & Krueger (1994), Castle Doctrine, MPDTA, California Prop 99 smoking (single treated unit), Walmart entry county panel (staggered); divorce laws is synthetic-only. Every loader marks provenance in `df.attrs["source"]` and warns when it falls back to synthetic data - [Visualization](https://diff-diff.readthedocs.io/en/stable/api/visualization.html): Event study plots, group effects, sensitivity plots, Bacon decomposition plots, power curves - [Data Preparation](https://diff-diff.readthedocs.io/en/stable/api/prep.html): Data generation, panel balancing, wide-to-long conversion, treatment/post indicator creation diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 853c26d37..408b5afa4 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -5847,7 +5847,7 @@ estimator-focused: **Scope:** Treatment-variable coding in `load_castle_doctrine()` only - no estimator, variance, or inference behavior. Loader provenance, checksum pinning, and the synthetic-fallback contract are documented in the loader docstrings and `docs/api/datasets.rst`. -- **Note:** The paper's estimating equation is `Outcome_it = b0*CDL_it + b1*X_it + c_i + u_t + e_it`, "where `CDL_it` is the treatment variable that equals the proportion of year `t` in which state `i` has an effective Castle Doctrine law" (p. 830, Sec. III). For partial-year adopters it states: "we set CDL equal to the proportion of the year in which the law was in effect, though estimates are almost identical when we exclude the year of adoption" (fn. 21 reports the excluded-year estimates). That regressor is preserved verbatim as `treatment_exposure` (the source file's `cdl`). Verified in the file: 0 before adoption and for never-adopters, exactly 1.0 after, and a day-count fraction in the adoption year - `cdl * 365` is an exact integer for 17 of the 21 adopting states (59 to 306 days) and exactly 182.5 for the other four. +- **Note:** The paper's estimating equation is `Outcome_it = b0*CDL_it + b1*X_it + c_i + u_t + e_it`, "where `CDL_it` is the treatment variable that equals the proportion of year `t` in which state `i` has an effective Castle Doctrine law" (p. 830, Sec. III). For partial-year adopters it states: "we set CDL equal to the proportion of the year in which the law was in effect, though estimates are almost identical when we exclude the year of adoption" (fn. 21 reports the excluded-year estimates). That regressor is preserved verbatim as `treatment_exposure` (the source file's `cdl`). Verified in the file: 0 before adoption and for never-adopters, exactly 1.0 after, and a day-count fraction in the adoption year - `cdl * 365` recovers an integer day count for 17 of the 21 adopting states (59 to 306 days) and 182.5 (half a year) for the other four. `cdl` is stored as float32, so those products are integers only to within storage rounding (max deviation 1.03e-5), not bit-exact. - **Note:** The binary `treated` column follows the standard staggered-DiD convention (`year >= first_treat`), so that `treated`, `first_treat`, and `cohort` stay mutually consistent - which `_validate_panel_keys` enforces and which the cohort estimators (`CallawaySantAnna`, `SunAbraham`) require. This convention matches **neither** of the paper's treatments of the adoption year: not the fractional `CDL_it` of the main specification, nor the drop-the-adoption-year robustness check. It also differs from the source file's own binary `post` indicator (0 in the adoption year, 1 thereafter) on exactly 21 of 550 rows. **Replicating Cheng-Hoekstra requires regressing on `treatment_exposure`, not `treated`.** Synthetic fallback frames carry `treatment_exposure` as a binary 0/1 copy of `treated` and therefore cannot support the paper's specification at all. - **Note:** The paper's dependent variable is the log of the outcome per 100,000 population (p. 830). `homicide_rate` is the level (the source file's `homicide`, labelled "homicide count per 100,000 state population"), so a replication must take logs. The file's `l_homicide`, which is exactly `log(homicide)`, is not carried into the public schema. From 35488b4adc903c2859cee633e48fcc222addc342 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 09:02:02 -0400 Subject: [PATCH 08/17] docs: reframe tutorial 09 divorce section as a simulation, not a replication The divorce section ran CallawaySantAnna on load_divorce_laws() output and then told the reader the result showed 'effects near zero (validating parallel trends)' and reproduced the 'spike and fade' Wolfers (2006) documented. That data is generated, so the notebook was presenting fabricated numbers as confirmation of a published empirical finding - the exact failure mode this branch exists to remove. Reframes the section as a teaching simulation: a warning callout at the top, provenance surfaced via df.attrs['source'] in the load cell, the parallel-trends and reproduction claims removed, and the summary marking the panel synthetic. The estimator mechanics it teaches are unchanged. Card-Krueger and Castle Doctrine now say plainly that they run on checksum-verified replication data. Markdown and one print statement only; no outputs were committed and none need relocking. --- docs/tutorials/09_real_world_examples.ipynb | 111 ++------------------ 1 file changed, 7 insertions(+), 104 deletions(-) diff --git a/docs/tutorials/09_real_world_examples.ipynb b/docs/tutorials/09_real_world_examples.ipynb index 93766223e..e274bad79 100644 --- a/docs/tutorials/09_real_world_examples.ipynb +++ b/docs/tutorials/09_real_world_examples.ipynb @@ -4,17 +4,7 @@ "cell_type": "markdown", "id": "cell-0", "metadata": {}, - "source": [ - "# Real-World Data Examples\n", - "\n", - "This notebook demonstrates `diff-diff` using real-world datasets from classic econometric studies. We'll cover:\n", - "\n", - "1. **Card & Krueger (1994)** - Classic 2x2 DiD: Effect of minimum wage on employment\n", - "2. **Castle Doctrine Laws** - Staggered adoption: Effect of self-defense laws on homicide rates\n", - "3. **Unilateral Divorce Laws** - Staggered adoption: Effect of no-fault divorce on divorce rates\n", - "\n", - "These examples show how to apply DiD methods to real policy questions and replicate findings from influential studies." - ] + "source": "# Real-World Data Examples\n\nThis notebook demonstrates `diff-diff` on three classic econometric study designs:\n\n1. **Card & Krueger (1994)** - Classic 2x2 DiD: Effect of minimum wage on employment\n2. **Castle Doctrine Laws** - Staggered adoption: Effect of self-defense laws on homicide rates\n3. **Unilateral Divorce Laws** - Staggered adoption on a long panel (**simulated data**)\n\nThe first two run on checksum-verified replication data and reproduce the published designs. The third does not: `diff-diff` has no verified public source wired up for the Stevenson-Wolfers divorce panel, so `load_divorce_laws()` generates data and warns on every call. Its numbers illustrate estimator mechanics on a long staggered panel - they are not estimates of any real policy effect.\n\nEvery loader records where its data came from in `df.attrs[\"source\"]`. Check it before treating any result as a replication." }, { "cell_type": "code", @@ -57,13 +47,7 @@ "id": "cell-2", "metadata": {}, "outputs": [], - "source": [ - "# List available datasets\n", - "print(\"Available real-world datasets in diff-diff:\")\n", - "print(\"=\" * 60)\n", - "for name, desc in list_datasets().items():\n", - " print(f\" {name}: {desc}\")" - ] + "source": "# List available datasets\nprint(\"Available built-in datasets in diff-diff:\")\nprint(\"=\" * 60)\nfor name, desc in list_datasets().items():\n print(f\" {name}: {desc}\")" }, { "cell_type": "markdown", @@ -603,24 +587,7 @@ "cell_type": "markdown", "id": "cell-29", "metadata": {}, - "source": [ - "---\n", - "\n", - "## Unilateral Divorce Laws: Long Panel with Staggered Adoption\n", - "\n", - "### Background\n", - "\n", - "Unilateral (no-fault) divorce laws allow one spouse to obtain a divorce without the other's consent. These laws were adopted at different times across U.S. states, primarily between 1969 and 1985.\n", - "\n", - "**Research question**: How did unilateral divorce laws affect divorce rates?\n", - "\n", - "**Design**: Staggered DiD with long panel\n", - "- **Treatment**: Adoption of unilateral divorce law\n", - "- **Time period**: 1968-1988\n", - "- **Cohorts**: States adopting in different years\n", - "\n", - "**Key finding**: Wolfers (2006) found an initial spike in divorce rates that faded over time." - ] + "source": "---\n\n## Unilateral Divorce Laws: Long Panel with Staggered Adoption\n\n> **Simulated data.** No verified public source is configured for this dataset, so\n> `load_divorce_laws()` generates a panel and emits a `SYNTHETIC` warning every time it is\n> called. Everything in this section demonstrates estimator mechanics on a long staggered\n> panel. The estimates are properties of the simulation, not findings about divorce law.\n\n### Background\n\nUnilateral (no-fault) divorce laws allow one spouse to obtain a divorce without the other's consent. States adopted them at different times, primarily between 1969 and 1985 - a natural staggered-adoption design, and the motivation for the simulated panel used below.\n\n**Design being illustrated**: Staggered DiD with a long panel\n- **Treatment**: Adoption of unilateral divorce law\n- **Time period**: 1968-1988\n- **Cohorts**: States adopting in different years\n\nThe empirical literature on this question - Wolfers (2006), Stevenson & Wolfers (2006) - is cited at the end of the notebook. Reproducing those results requires the authors' data; this panel will not do it." }, { "cell_type": "code", @@ -628,15 +595,7 @@ "id": "cell-30", "metadata": {}, "outputs": [], - "source": [ - "# Load divorce laws dataset\n", - "divorce = load_divorce_laws()\n", - "\n", - "print(f\"Dataset shape: {divorce.shape}\")\n", - "print(f\"Years: {divorce['year'].min()} to {divorce['year'].max()}\")\n", - "print(f\"States: {divorce['state'].nunique()}\")\n", - "divorce.head()" - ] + "source": "# Load divorce laws dataset (emits a SYNTHETIC warning - see the note above)\ndivorce = load_divorce_laws()\n\nprint(f\"Dataset shape: {divorce.shape}\")\nprint(f\"Provenance: {divorce.attrs['source']}\")\nprint(f\"Years: {divorce['year'].min()} to {divorce['year'].max()}\")\nprint(f\"States: {divorce['state'].nunique()}\")\ndivorce.head()" }, { "cell_type": "code", @@ -729,17 +688,7 @@ "cell_type": "markdown", "id": "cell-35", "metadata": {}, - "source": [ - "### Dynamic Effects Pattern\n", - "\n", - "Notice the pattern in the event study:\n", - "1. **Pre-treatment**: Effects near zero (validating parallel trends)\n", - "2. **Short-run**: Spike in divorce rates immediately after adoption\n", - "3. **Medium-run**: Effects diminish over time\n", - "4. **Long-run**: Effects may return close to zero\n", - "\n", - "This \"spike and fade\" pattern was documented by Wolfers (2006) and suggests that unilateral divorce primarily moved forward divorces that would have happened anyway (\"harvesting effect\")." - ] + "source": "### Dynamic Effects Pattern\n\nThe event study above traces effects by time since adoption: roughly flat before treatment, a jump at adoption, then decay. That shape is built into the simulated data, so recovering it shows the event-study aggregation working as intended. It is not evidence about divorce law, and the flat pre-period is not a parallel-trends test that any real design has passed.\n\nThe shape is loosely modelled on the \"spike and fade\" Wolfers (2006) reported using the actual state panel, where an initial rise in divorce rates diminished over time - attributed to unilateral divorce mostly bringing forward divorces that would have happened anyway. That finding stands on the real data, not on this simulation.\n\nWhat this section genuinely demonstrates is mechanical: how `CallawaySantAnna` reports effects relative to adoption on a long panel with many cohorts, and how to read an event study it produces." }, { "cell_type": "code", @@ -762,53 +711,7 @@ "cell_type": "markdown", "id": "cell-37", "metadata": {}, - "source": [ - "---\n", - "\n", - "## Summary\n", - "\n", - "### Key Takeaways\n", - "\n", - "1. **Card-Krueger (1994)**\n", - " - Classic 2x2 DiD design\n", - " - Simple before/after, treatment/control comparison\n", - " - Key insight: Minimum wage increases don't necessarily reduce employment\n", - "\n", - "2. **Castle Doctrine Laws**\n", - " - Staggered adoption across states\n", - " - TWFE can be biased; use CS or Sun-Abraham\n", - " - Bacon decomposition reveals the problem with TWFE\n", - " - Finding: Laws associated with increased homicide rates\n", - "\n", - "3. **Unilateral Divorce Laws**\n", - " - Long panel with many cohorts\n", - " - Dynamic treatment effects (spike and fade)\n", - " - Event study reveals time-varying patterns\n", - "\n", - "### When to Use Which Estimator\n", - "\n", - "| Design | Recommended Estimator |\n", - "|--------|----------------------|\n", - "| Classic 2x2 | `DifferenceInDifferences` |\n", - "| Panel with 2 periods | `DifferenceInDifferences` or `TwoWayFixedEffects` |\n", - "| Staggered adoption | `CallawaySantAnna` or `SunAbraham` |\n", - "| Heterogeneous timing | Always use `CallawaySantAnna` / `SunAbraham` |\n", - "| Few never-treated | `CallawaySantAnna(control_group='not_yet_treated')` |\n", - "\n", - "### References\n", - "\n", - "- Card, D., & Krueger, A. B. (1994). Minimum Wages and Employment: A Case Study of the Fast-Food Industry in New Jersey and Pennsylvania. *American Economic Review*, 84(4), 772-793.\n", - "\n", - "- Cheng, C., & Hoekstra, M. (2013). Does Strengthening Self-Defense Law Deter Crime or Escalate Violence? Evidence from Expansions to Castle Doctrine. *Journal of Human Resources*, 48(3), 821-854.\n", - "\n", - "- Stevenson, B., & Wolfers, J. (2006). Bargaining in the Shadow of the Law: Divorce Laws and Family Distress. *Quarterly Journal of Economics*, 121(1), 267-288.\n", - "\n", - "- Wolfers, J. (2006). Did Unilateral Divorce Laws Raise Divorce Rates? A Reconciliation and New Results. *American Economic Review*, 96(5), 1802-1820.\n", - "\n", - "- Callaway, B., & Sant'Anna, P. H. (2021). Difference-in-differences with multiple time periods. *Journal of Econometrics*, 225(2), 200-230.\n", - "\n", - "- Goodman-Bacon, A. (2021). Difference-in-differences with variation in treatment timing. *Journal of Econometrics*, 225(2), 254-277." - ] + "source": "---\n\n## Summary\n\n### Key Takeaways\n\n1. **Card-Krueger (1994)** - checksum-verified replication data\n - Classic 2x2 DiD design\n - Simple before/after, treatment/control comparison\n - Key insight: Minimum wage increases don't necessarily reduce employment\n\n2. **Castle Doctrine Laws** - checksum-verified replication data\n - Staggered adoption across states\n - TWFE can be biased; use CS or Sun-Abraham\n - Bacon decomposition reveals the problem with TWFE\n - Finding: Laws associated with increased homicide rates\n\n3. **Unilateral Divorce Laws** - **simulated panel**, `df.attrs[\"source\"] == \"synthetic_fallback\"`\n - Long panel with many cohorts\n - Event study reveals time-varying patterns\n - Demonstrates estimator mechanics only; the numbers are not evidence about divorce law\n\n### Checking provenance\n\nEvery loader marks where its data came from, and warns when it falls back to generated data:\n\n```python\ndf = load_castle_doctrine()\ndf.attrs[\"source\"] # 'cheng_hoekstra_castle_data' -> canonical\n # 'synthetic_fallback' -> generated, not a replication\n```\n\n### When to Use Which Estimator\n\n| Design | Recommended Estimator |\n|--------|----------------------|\n| Classic 2x2 | `DifferenceInDifferences` |\n| Panel with 2 periods | `DifferenceInDifferences` or `TwoWayFixedEffects` |\n| Staggered adoption | `CallawaySantAnna` or `SunAbraham` |\n| Heterogeneous timing | Always use `CallawaySantAnna` / `SunAbraham` |\n| Few never-treated | `CallawaySantAnna(control_group='not_yet_treated')` |\n\n### References\n\nThe divorce-law papers below are cited as context for the design the simulated panel imitates, not as results this notebook reproduces.\n\n- Card, D., & Krueger, A. B. (1994). Minimum Wages and Employment: A Case Study of the Fast-Food Industry in New Jersey and Pennsylvania. *American Economic Review*, 84(4), 772-793.\n\n- Cheng, C., & Hoekstra, M. (2013). Does Strengthening Self-Defense Law Deter Crime or Escalate Violence? Evidence from Expansions to Castle Doctrine. *Journal of Human Resources*, 48(3), 821-854.\n\n- Stevenson, B., & Wolfers, J. (2006). Bargaining in the Shadow of the Law: Divorce Laws and Family Distress. *Quarterly Journal of Economics*, 121(1), 267-288.\n\n- Wolfers, J. (2006). Did Unilateral Divorce Laws Raise Divorce Rates? A Reconciliation and New Results. *American Economic Review*, 96(5), 1802-1820.\n\n- Callaway, B., & Sant'Anna, P. H. (2021). Difference-in-differences with multiple time periods. *Journal of Econometrics*, 225(2), 200-230.\n\n- Goodman-Bacon, A. (2021). Difference-in-differences with variation in treatment timing. *Journal of Econometrics*, 225(2), 254-277." } ], "metadata": { @@ -818,4 +721,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file From f63169f0c48aee993203c032511fb318010a91fe Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 09:16:39 -0400 Subject: [PATCH 09/17] fix: documented Card-Krueger workflow breaks on the canonical incomplete survey The real survey is missing 12 emp_pre and 14 emp_post readings; the synthetic fallback is complete. The load_card_krueger docstring example and the datasets.rst example both melt to long format and fit immediately, so they estimated fine while the loader served synthetic data and raise 'Column employment contains missing values' now that the canonical source resolves. The tutorial already dropped NaNs, which is why it kept passing. Adds the dropna to both examples, documents the missing-value counts in the loader Notes, and adds a regression test that asserts both halves: the raw workflow raises, and the documented one estimates finite output. Also narrows two overclaims in tutorial 09: the intro said the first two sections 'reproduce the published designs', and the Castle section presented its estimates without noting that it regresses the level rate on a binary indicator rather than Cheng-Hoekstra's log rate on fractional CDL_it. --- diff_diff/datasets.py | 10 ++++++ docs/api/datasets.rst | 4 +++ docs/tutorials/09_real_world_examples.ipynb | 23 ++------------ tests/test_datasets.py | 35 +++++++++++++++++++++ 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index c55757f90..2a87c437c 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -743,6 +743,13 @@ def load_card_krueger(force_download: bool = False) -> pd.DataFrame: Original finding: No significant negative effect of minimum wage increase on employment (ATT ≈ +2.8 FTE employees). + The canonical survey is incomplete: 12 stores lack ``emp_pre``, 14 lack + ``emp_post``, and ``wage_pre``/``wage_post`` are missing for 20 and 21 + stores. Drop the missing outcome rows before fitting, as the example below + does; estimators reject missing outcomes rather than dropping them silently. + The synthetic fallback frame is complete, so code that skips this step will + work offline and fail once the canonical source is reachable. + The canonical data are checksum-verified and returned with ``df.attrs["source"] == "card_krueger_public_data"``. Any download, parse, or validation failure emits one ``UserWarning`` containing ``SYNTHETIC`` @@ -768,6 +775,9 @@ def load_card_krueger(force_download: bool = False) -> pd.DataFrame: ... ) >>> ck_long['post'] = (ck_long['period'] == 'emp_post').astype(int) >>> + >>> # 26 store-waves have no employment reading in the source survey + >>> ck_long = ck_long.dropna(subset=['employment']) + >>> >>> # Estimate DiD >>> did = DifferenceInDifferences() >>> results = did.fit(ck_long, outcome='employment', treatment='treated', time='post') diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index bfe31f898..2b78fd760 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -50,6 +50,10 @@ Example ) ck_long['post'] = (ck_long['period'] == 'emp_post').astype(int) + # 26 store-waves have no employment reading in the source survey; estimators + # reject missing outcomes rather than dropping them silently + ck_long = ck_long.dropna(subset=['employment']) + did = DifferenceInDifferences() results = did.fit(ck_long, outcome='employment', treatment='treated', time='post') diff --git a/docs/tutorials/09_real_world_examples.ipynb b/docs/tutorials/09_real_world_examples.ipynb index e274bad79..2d1490052 100644 --- a/docs/tutorials/09_real_world_examples.ipynb +++ b/docs/tutorials/09_real_world_examples.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "id": "cell-0", "metadata": {}, - "source": "# Real-World Data Examples\n\nThis notebook demonstrates `diff-diff` on three classic econometric study designs:\n\n1. **Card & Krueger (1994)** - Classic 2x2 DiD: Effect of minimum wage on employment\n2. **Castle Doctrine Laws** - Staggered adoption: Effect of self-defense laws on homicide rates\n3. **Unilateral Divorce Laws** - Staggered adoption on a long panel (**simulated data**)\n\nThe first two run on checksum-verified replication data and reproduce the published designs. The third does not: `diff-diff` has no verified public source wired up for the Stevenson-Wolfers divorce panel, so `load_divorce_laws()` generates data and warns on every call. Its numbers illustrate estimator mechanics on a long staggered panel - they are not estimates of any real policy effect.\n\nEvery loader records where its data came from in `df.attrs[\"source\"]`. Check it before treating any result as a replication." + "source": "# Real-World Data Examples\n\nThis notebook demonstrates `diff-diff` on three classic econometric study designs:\n\n1. **Card & Krueger (1994)** - Classic 2x2 DiD: Effect of minimum wage on employment\n2. **Castle Doctrine Laws** - Staggered adoption: Effect of self-defense laws on homicide rates\n3. **Unilateral Divorce Laws** - Staggered adoption on a long panel (**simulated data**)\n\nThe first two run on checksum-verified replication data from the original studies. They illustrate `diff-diff`'s estimators on that data rather than reproducing each paper's exact specification - where the two differ, it is noted in the section.\n\nThe third is different: `diff-diff` has no verified public source wired up for the Stevenson-Wolfers divorce panel, so `load_divorce_laws()` generates data and warns on every call. Its numbers illustrate estimator mechanics on a long staggered panel - they are not estimates of any real policy effect.\n\nEvery loader records where its data came from in `df.attrs[\"source\"]`. Check it before treating any result as a replication." }, { "cell_type": "code", @@ -294,24 +294,7 @@ "cell_type": "markdown", "id": "cell-14", "metadata": {}, - "source": [ - "---\n", - "\n", - "## Castle Doctrine Laws: Staggered Adoption\n", - "\n", - "### Background\n", - "\n", - "Castle Doctrine (or \"Stand Your Ground\") laws expand self-defense rights by removing the duty to retreat before using deadly force. These laws were adopted by different U.S. states at different times, creating a **staggered adoption** design.\n", - "\n", - "**Research question**: Do Castle Doctrine laws affect homicide rates?\n", - "\n", - "**Design**: Staggered DiD\n", - "- **Treatment**: Adoption of Castle Doctrine law\n", - "- **Cohorts**: States adopting in 2005, 2006, 2007, 2008, 2009\n", - "- **Control**: States that never adopted during the study period\n", - "\n", - "**Key finding**: Cheng & Hoekstra (2013) found an approximately 8% increase in homicide rates following adoption." - ] + "source": "---\n\n## Castle Doctrine Laws: Staggered Adoption\n\n### Background\n\nCastle Doctrine (or \"Stand Your Ground\") laws expand self-defense rights by removing the duty to retreat before using deadly force. These laws were adopted by different U.S. states at different times, creating a **staggered adoption** design.\n\n**Research question**: Do Castle Doctrine laws affect homicide rates?\n\n**Design**: Staggered DiD\n- **Treatment**: Adoption of Castle Doctrine law\n- **Cohorts**: States adopting in 2005, 2006, 2007, 2008, 2009\n- **Control**: States that never adopted during the study period\n\n**Key finding**: Cheng & Hoekstra (2013) found an approximately 8% increase in homicide rates following adoption.\n\n> **Specification note.** This is the authors' data, but not the authors' regression. Below we\n> use the binary `treated` indicator and the *level* homicide rate, to show what the modern\n> staggered-DiD estimators do. Cheng-Hoekstra regress the *log* rate on `treatment_exposure` -\n> `CDL_it`, the fraction of the year the law was in effect - which the loader also provides.\n> Treat the estimates here as an estimator demonstration, not a reproduction of their table." }, { "cell_type": "code", @@ -711,7 +694,7 @@ "cell_type": "markdown", "id": "cell-37", "metadata": {}, - "source": "---\n\n## Summary\n\n### Key Takeaways\n\n1. **Card-Krueger (1994)** - checksum-verified replication data\n - Classic 2x2 DiD design\n - Simple before/after, treatment/control comparison\n - Key insight: Minimum wage increases don't necessarily reduce employment\n\n2. **Castle Doctrine Laws** - checksum-verified replication data\n - Staggered adoption across states\n - TWFE can be biased; use CS or Sun-Abraham\n - Bacon decomposition reveals the problem with TWFE\n - Finding: Laws associated with increased homicide rates\n\n3. **Unilateral Divorce Laws** - **simulated panel**, `df.attrs[\"source\"] == \"synthetic_fallback\"`\n - Long panel with many cohorts\n - Event study reveals time-varying patterns\n - Demonstrates estimator mechanics only; the numbers are not evidence about divorce law\n\n### Checking provenance\n\nEvery loader marks where its data came from, and warns when it falls back to generated data:\n\n```python\ndf = load_castle_doctrine()\ndf.attrs[\"source\"] # 'cheng_hoekstra_castle_data' -> canonical\n # 'synthetic_fallback' -> generated, not a replication\n```\n\n### When to Use Which Estimator\n\n| Design | Recommended Estimator |\n|--------|----------------------|\n| Classic 2x2 | `DifferenceInDifferences` |\n| Panel with 2 periods | `DifferenceInDifferences` or `TwoWayFixedEffects` |\n| Staggered adoption | `CallawaySantAnna` or `SunAbraham` |\n| Heterogeneous timing | Always use `CallawaySantAnna` / `SunAbraham` |\n| Few never-treated | `CallawaySantAnna(control_group='not_yet_treated')` |\n\n### References\n\nThe divorce-law papers below are cited as context for the design the simulated panel imitates, not as results this notebook reproduces.\n\n- Card, D., & Krueger, A. B. (1994). Minimum Wages and Employment: A Case Study of the Fast-Food Industry in New Jersey and Pennsylvania. *American Economic Review*, 84(4), 772-793.\n\n- Cheng, C., & Hoekstra, M. (2013). Does Strengthening Self-Defense Law Deter Crime or Escalate Violence? Evidence from Expansions to Castle Doctrine. *Journal of Human Resources*, 48(3), 821-854.\n\n- Stevenson, B., & Wolfers, J. (2006). Bargaining in the Shadow of the Law: Divorce Laws and Family Distress. *Quarterly Journal of Economics*, 121(1), 267-288.\n\n- Wolfers, J. (2006). Did Unilateral Divorce Laws Raise Divorce Rates? A Reconciliation and New Results. *American Economic Review*, 96(5), 1802-1820.\n\n- Callaway, B., & Sant'Anna, P. H. (2021). Difference-in-differences with multiple time periods. *Journal of Econometrics*, 225(2), 200-230.\n\n- Goodman-Bacon, A. (2021). Difference-in-differences with variation in treatment timing. *Journal of Econometrics*, 225(2), 254-277." + "source": "---\n\n## Summary\n\n### Key Takeaways\n\n1. **Card-Krueger (1994)** - checksum-verified replication data\n - Classic 2x2 DiD design\n - Simple before/after, treatment/control comparison\n - The canonical survey is incomplete (26 store-waves lack an employment reading), so drop missing outcomes before fitting\n - Key insight: Minimum wage increases don't necessarily reduce employment\n\n2. **Castle Doctrine Laws** - checksum-verified replication data\n - Staggered adoption across states\n - TWFE can be biased; use CS or Sun-Abraham\n - Bacon decomposition reveals the problem with TWFE\n - Estimated here on the level rate with a binary indicator, not the paper's log-rate / fractional-exposure specification\n\n3. **Unilateral Divorce Laws** - **simulated panel**, `df.attrs[\"source\"] == \"synthetic_fallback\"`\n - Long panel with many cohorts\n - Event study reveals time-varying patterns\n - Demonstrates estimator mechanics only; the numbers are not evidence about divorce law\n\n### Checking provenance\n\nEvery loader marks where its data came from, and warns when it falls back to generated data:\n\n```python\ndf = load_castle_doctrine()\ndf.attrs[\"source\"] # 'cheng_hoekstra_castle_data' -> canonical\n # 'synthetic_fallback' -> generated, not a replication\n```\n\n### When to Use Which Estimator\n\n| Design | Recommended Estimator |\n|--------|----------------------|\n| Classic 2x2 | `DifferenceInDifferences` |\n| Panel with 2 periods | `DifferenceInDifferences` or `TwoWayFixedEffects` |\n| Staggered adoption | `CallawaySantAnna` or `SunAbraham` |\n| Heterogeneous timing | Always use `CallawaySantAnna` / `SunAbraham` |\n| Few never-treated | `CallawaySantAnna(control_group='not_yet_treated')` |\n\n### References\n\nThe divorce-law papers below are cited as context for the design the simulated panel imitates, not as results this notebook reproduces.\n\n- Card, D., & Krueger, A. B. (1994). Minimum Wages and Employment: A Case Study of the Fast-Food Industry in New Jersey and Pennsylvania. *American Economic Review*, 84(4), 772-793.\n\n- Cheng, C., & Hoekstra, M. (2013). Does Strengthening Self-Defense Law Deter Crime or Escalate Violence? Evidence from Expansions to Castle Doctrine. *Journal of Human Resources*, 48(3), 821-854.\n\n- Stevenson, B., & Wolfers, J. (2006). Bargaining in the Shadow of the Law: Divorce Laws and Family Distress. *Quarterly Journal of Economics*, 121(1), 267-288.\n\n- Wolfers, J. (2006). Did Unilateral Divorce Laws Raise Divorce Rates? A Reconciliation and New Results. *American Economic Review*, 96(5), 1802-1820.\n\n- Callaway, B., & Sant'Anna, P. H. (2021). Difference-in-differences with multiple time periods. *Journal of Econometrics*, 225(2), 200-230.\n\n- Goodman-Bacon, A. (2021). Difference-in-differences with variation in treatment timing. *Journal of Econometrics*, 225(2), 254-277." } ], "metadata": { diff --git a/tests/test_datasets.py b/tests/test_datasets.py index bbee61b2e..2e286f1e1 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -495,6 +495,41 @@ def test_protocol_level_http_errors_warn_and_use_marked_fallback(self, tmp_path, assert result.attrs["source"] == "synthetic_fallback" assert result.shape == _construct_mpdta_data().shape + def test_documented_card_workflow_runs_on_canonical_frame(self): + """The docstring/API example must survive the canonical data's missing outcomes. + + The real Card-Krueger survey is incomplete (12 missing ``emp_pre``, 14 missing + ``emp_post``), while the synthetic fallback is complete. Without the documented + ``dropna``, the published workflow estimates fine offline and raises as soon as + the canonical source becomes reachable. + """ + from diff_diff import DifferenceInDifferences + + ck = self._valid_card_source_frame() + assert ck["emp_pre"].isna().sum() == 12 + assert ck["emp_post"].isna().sum() == 14 + + ck_long = ck.melt( + id_vars=["store_id", "state", "treated"], + value_vars=["emp_pre", "emp_post"], + var_name="period", + value_name="employment", + ) + ck_long["post"] = (ck_long["period"] == "emp_post").astype(int) + + # Without the documented dropna the estimator rejects the frame outright. + with pytest.raises(ValueError, match="missing values"): + DifferenceInDifferences().fit( + ck_long, outcome="employment", treatment="treated", time="post" + ) + + ck_long = ck_long.dropna(subset=["employment"]) + results = DifferenceInDifferences().fit( + ck_long, outcome="employment", treatment="treated", time="post" + ) + assert np.isfinite(results.att) + assert np.isfinite(results.se) + def test_failed_cache_write_leaves_no_partial_file(self, tmp_path, monkeypatch): """A write that fails mid-flight must not strand a ``delete=False`` temp file.""" import diff_diff.datasets as datasets_mod From c8368cb39028b129082c562d0388ccbbf3960e1e Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 09:44:48 -0400 Subject: [PATCH 10/17] fix: qualify the fallback contract for verified-cache recovery; surface provenance in tutorial The changelog stated categorically that download failures warn and return synthetic data. They do not when a checksum-valid cache entry exists: those bytes already passed the pinned SHA-256, so the loader returns canonical data silently - including under force_download=True, which bypasses the cache on the way in but still falls back to it when the download fails. Qualifies the wording and adds a regression test covering both force_download values. The tutorial intro asserted that the Card and Castle sections run on checksum-verified data, but offline they run on the synthetic fallback and neither cell showed which it got. Both load cells now print df.attrs['source'], and the intro says 'when their canonical sources are reachable'. --- CHANGELOG.md | 23 ++++++++++++-- docs/tutorials/09_real_world_examples.ipynb | 23 ++------------ tests/test_datasets.py | 34 +++++++++++++++++++++ 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c34ac914..001d733d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -399,9 +399,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 canonical data.** `load_card_krueger()`, `load_castle_doctrine()`, and `load_mpdta()` now use commit-pinned, SHA-256-verified sources with schema and panel-invariant validation; successful frames expose a dataset-specific - `df.attrs["source"]`. Download, checksum, parsing, or validation failures emit - exactly one `UserWarning` containing `SYNTHETIC` and mark the fallback as - `source="synthetic_fallback"`. `load_divorce_laws()` now follows that explicit + `df.attrs["source"]`. When neither a verified cache entry nor a verified fresh + download is available, the loader emits exactly one `UserWarning` containing + `SYNTHETIC` and marks the frame `source="synthetic_fallback"`. A download that + fails while a checksum-valid cache entry exists returns that canonical data + silently, including under `force_download=True`. `load_divorce_laws()` now follows that explicit fallback path because no verified source currently reproduces its composite public schema without additional analytical choices. Verified fresh downloads are returned even when best-effort cache persistence fails. @@ -410,6 +412,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Castle Doctrine now exposes first-year treatment intensity.** The loader retains the source's fractional adoption-year `cdl` value as `treatment_exposure`, while preserving the existing binary `treated` field. + Canonical frames carry the fractional value; synthetic fallback frames carry a + binary 0/1 copy of `treated`. Reproducing Cheng-Hoekstra requires regressing + `log(homicide_rate)` on `treatment_exposure`, not on `treated` - see + `docs/methodology/REGISTRY.md`. + +- **`load_card_krueger()` returns the canonical 410-store survey, which is + incomplete.** 12 stores have no `emp_pre`, 14 no `emp_post`, and `wage_pre` / + `wage_post` are missing for 20 and 21 stores. The synthetic frame this loader + previously returned was complete, so code that reshapes to long format and + fits without dropping missing outcomes estimated fine before and now raises + `ValueError: Column '' contains missing values`. Drop the missing + outcome rows before fitting; the loader docstring and + `docs/api/datasets.rst` examples show the pattern. Note that the same code + still succeeds offline, where the loader falls back to the complete synthetic + frame. - **dCDH Phase 1 (`L_max=None`) event-study row is now re-synced by the final-df refresh.** Under replicate-weight designs the refresh recomputes `overall_*` with the diff --git a/docs/tutorials/09_real_world_examples.ipynb b/docs/tutorials/09_real_world_examples.ipynb index 2d1490052..8eb1ce4f7 100644 --- a/docs/tutorials/09_real_world_examples.ipynb +++ b/docs/tutorials/09_real_world_examples.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "id": "cell-0", "metadata": {}, - "source": "# Real-World Data Examples\n\nThis notebook demonstrates `diff-diff` on three classic econometric study designs:\n\n1. **Card & Krueger (1994)** - Classic 2x2 DiD: Effect of minimum wage on employment\n2. **Castle Doctrine Laws** - Staggered adoption: Effect of self-defense laws on homicide rates\n3. **Unilateral Divorce Laws** - Staggered adoption on a long panel (**simulated data**)\n\nThe first two run on checksum-verified replication data from the original studies. They illustrate `diff-diff`'s estimators on that data rather than reproducing each paper's exact specification - where the two differ, it is noted in the section.\n\nThe third is different: `diff-diff` has no verified public source wired up for the Stevenson-Wolfers divorce panel, so `load_divorce_laws()` generates data and warns on every call. Its numbers illustrate estimator mechanics on a long staggered panel - they are not estimates of any real policy effect.\n\nEvery loader records where its data came from in `df.attrs[\"source\"]`. Check it before treating any result as a replication." + "source": "# Real-World Data Examples\n\nThis notebook demonstrates `diff-diff` on three classic econometric study designs:\n\n1. **Card & Krueger (1994)** - Classic 2x2 DiD: Effect of minimum wage on employment\n2. **Castle Doctrine Laws** - Staggered adoption: Effect of self-defense laws on homicide rates\n3. **Unilateral Divorce Laws** - Staggered adoption on a long panel (**simulated data**)\n\nThe first two run on checksum-verified replication data *when their canonical sources are reachable*. If a download fails, the loader warns and falls back to a generated panel with the same schema, so the numbers below become illustrative rather than empirical. Each section prints `df.attrs[\"source\"]` after loading - check it before reading any estimate as a replication. Note also that these sections illustrate `diff-diff`'s estimators on the authors' data rather than reproducing each paper's exact specification; where the two differ, the section says so.\n\nThe third is different in kind: `diff-diff` has no verified public source wired up for the Stevenson-Wolfers divorce panel, so `load_divorce_laws()` generates data and warns on *every* call. Its numbers illustrate estimator mechanics on a long staggered panel - they are not estimates of any real policy effect." }, { "cell_type": "code", @@ -79,16 +79,7 @@ "id": "cell-4", "metadata": {}, "outputs": [], - "source": [ - "# Load the Card-Krueger dataset\n", - "ck = load_card_krueger()\n", - "\n", - "print(f\"Dataset shape: {ck.shape}\")\n", - "print(f\"\\nStores by state:\")\n", - "print(ck.groupby('state').size())\n", - "print(f\"\\nFirst few rows:\")\n", - "ck.head()" - ] + "source": "# Load the Card-Krueger dataset\nck = load_card_krueger()\n\nprint(f\"Dataset shape: {ck.shape}\")\nprint(f\"Provenance: {ck.attrs['source']}\")\nprint(f\" (card_krueger_public_data = canonical; synthetic_fallback = generated, not a replication)\")\nprint(f\"\\nStores by state:\")\nprint(ck.groupby('state').size())\nprint(f\"\\nFirst few rows:\")\nck.head()" }, { "cell_type": "code", @@ -302,15 +293,7 @@ "id": "cell-15", "metadata": {}, "outputs": [], - "source": [ - "# Load the Castle Doctrine dataset\n", - "castle = load_castle_doctrine()\n", - "\n", - "print(f\"Dataset shape: {castle.shape}\")\n", - "print(f\"Years: {castle['year'].min()} to {castle['year'].max()}\")\n", - "print(f\"States: {castle['state'].nunique()}\")\n", - "castle.head()" - ] + "source": "# Load the Castle Doctrine dataset\ncastle = load_castle_doctrine()\n\nprint(f\"Dataset shape: {castle.shape}\")\nprint(f\"Provenance: {castle.attrs['source']}\")\nprint(f\" (cheng_hoekstra_castle_data = canonical; synthetic_fallback = generated, not a replication)\")\nprint(f\"Years: {castle['year'].min()} to {castle['year'].max()}\")\nprint(f\"States: {castle['state'].nunique()}\")\ncastle.head()" }, { "cell_type": "code", diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 2e286f1e1..ffe9aac7b 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -495,6 +495,40 @@ def test_protocol_level_http_errors_warn_and_use_marked_fallback(self, tmp_path, assert result.attrs["source"] == "synthetic_fallback" assert result.shape == _construct_mpdta_data().shape + def test_verified_cache_recovers_canonical_data_on_download_failure( + self, tmp_path, monkeypatch + ): + """A dead network with a checksum-valid cache returns canonical data, silently. + + This is the one failure path that must NOT warn or fall back: the cached bytes + already passed the pinned SHA-256, so they are the canonical data. It holds even + under ``force_download=True``, which bypasses the cache on the way in but still + falls back to it when the download fails. + """ + import hashlib + import warnings + + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + + source_csv = _construct_mpdta_data().rename(columns={"first_treat": "first.treat"}) + payload = source_csv[["year", "countyreal", "lpop", "lemp", "first.treat", "treat"]].to_csv( + index=False + ) + digest = hashlib.sha256(payload.encode()).hexdigest() + monkeypatch.setattr(datasets_mod, "_MPDTA_SOURCE_SHA256", digest) + (tmp_path / "mpdta.csv").write_text(payload) + + for force in (False, True): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with patch("diff_diff.datasets.urlopen", side_effect=TimeoutError("network down")): + result = load_mpdta(force_download=force) + + assert result.attrs["source"] == "callaway_santanna_mpdta", f"force={force}" + assert not [w for w in caught if "SYNTHETIC" in str(w.message)], f"force={force}" + def test_documented_card_workflow_runs_on_canonical_frame(self): """The docstring/API example must survive the canonical data's missing outcomes. From ee5a5688aa48ad286365aaa5ca42fcdbc94776d4 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 09:49:41 -0400 Subject: [PATCH 11/17] chore: map datasets.py to its REGISTRY section; correct stale castle-availability note doc-deps.yaml listed only the API page and tutorial for datasets.py, so /docs-impact would not flag the new 'Castle Doctrine treatment coding' registry section when the loader changes - the same drift the map exists to prevent. test_methodology_lwdid.py justified committing castle_lw_subset.csv on the grounds that 'the load_castle_doctrine upstream source is currently unavailable'. That is no longer true as of this branch. Reworded to say the subset is retained as the pinned artifact its goldens were captured against, and that consolidating onto the loader is a separate decision. --- docs/doc-deps.yaml | 3 +++ tests/test_methodology_lwdid.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index 448745dbf..f5f53335f 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -1123,6 +1123,9 @@ sources: type: api_reference - path: docs/tutorials/09_real_world_examples.ipynb type: tutorial + - path: docs/methodology/REGISTRY.md + section: "Castle Doctrine treatment coding" + type: methodology diff_diff/practitioner.py: drift_risk: low diff --git a/tests/test_methodology_lwdid.py b/tests/test_methodology_lwdid.py index 1a2fe9684..3a324ff24 100644 --- a/tests/test_methodology_lwdid.py +++ b/tests/test_methodology_lwdid.py @@ -42,8 +42,11 @@ - Castle doctrine: ``benchmarks/data/real/castle_lw_subset.csv``, the real Cheng & Hoekstra (2013) panel as packaged by Cunningham (2021) and distributed with PR #588 (extracted from commit 8c5cccea; columns state, - year, effyear, lhomicide, homicide, population). Committed here because - the ``load_castle_doctrine`` upstream source is currently unavailable. + year, effyear, lhomicide, homicide, population). Originally committed + because the ``load_castle_doctrine`` upstream source was dead; that source + is now pinned and verified, but this subset stays as the pinned artifact + these goldens were captured against. Consolidating onto the loader is a + separate decision, not a consequence of the source being reachable again. - Walmart event-study goldens: ``benchmarks/data/lwdid_walmart_eventstudy_golden.json`` (Tables A4/A5, provenance embedded in the file). From 3a3282c9efea5a5f5cbb90c03bcfdb29fd6b4c91 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 09:55:22 -0400 Subject: [PATCH 12/17] docs: align the remaining fallback-contract surfaces with cache recovery Round 5 corrected the categorical 'any download failure warns and returns synthetic data' claim in CHANGELOG.md and docs/api/datasets.rst but left it standing in the three loader docstrings and the tutorial intro - four of six surfaces still asserted a contract the code does not honour. All six now say the same thing: a download failure falls back to a checksum-valid cache entry when one exists and returns that canonical data silently; the SYNTHETIC warning and synthetic_fallback marker apply only when neither a verified cache entry nor a verified fresh download is available, or when the source fails to parse or validate. --- diff_diff/datasets.py | 27 ++++++++++++++------- docs/tutorials/09_real_world_examples.ipynb | 2 +- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 2a87c437c..3f8dec30b 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -751,9 +751,12 @@ def load_card_krueger(force_download: bool = False) -> pd.DataFrame: work offline and fail once the canonical source is reachable. The canonical data are checksum-verified and returned with - ``df.attrs["source"] == "card_krueger_public_data"``. Any download, parse, - or validation failure emits one ``UserWarning`` containing ``SYNTHETIC`` - and returns ``df.attrs["source"] == "synthetic_fallback"``. + ``df.attrs["source"] == "card_krueger_public_data"``. A download failure + falls back to a checksum-valid cache entry when one exists, returning that + canonical data silently. Only when neither a verified cache entry nor a + verified fresh download is available - or when the source fails to parse or + validate - does the loader emit one ``UserWarning`` containing ``SYNTHETIC`` + and return ``df.attrs["source"] == "synthetic_fallback"``. References ---------- @@ -898,9 +901,12 @@ def load_castle_doctrine(force_download: bool = False) -> pd.DataFrame: 2005 and 2009, creating a staggered treatment design. The canonical data are checksum-verified and returned with - ``df.attrs["source"] == "cheng_hoekstra_castle_data"``. Any download, - parse, or validation failure emits one ``UserWarning`` containing - ``SYNTHETIC`` and marks the returned frame as ``"synthetic_fallback"``. + ``df.attrs["source"] == "cheng_hoekstra_castle_data"``. A download failure + falls back to a checksum-valid cache entry when one exists, returning that + canonical data silently. Only when neither a verified cache entry nor a + verified fresh download is available - or when the source fails to parse or + validate - does the loader emit one ``UserWarning`` containing ``SYNTHETIC`` + and mark the returned frame as ``"synthetic_fallback"``. Replicating Cheng-Hoekstra (2013) requires the paper's regressor and outcome: regress ``log(homicide_rate)`` on ``treatment_exposure`` (their ``CDL_it``, @@ -1279,9 +1285,12 @@ def load_mpdta(force_download: bool = False) -> pd.DataFrame: in tutorials demonstrating the Callaway-Sant'Anna estimator. The canonical data are checksum-verified and returned with - ``df.attrs["source"] == "callaway_santanna_mpdta"``. Any download, parse, - or validation failure emits one ``UserWarning`` containing ``SYNTHETIC`` - and marks the returned frame as ``"synthetic_fallback"``. + ``df.attrs["source"] == "callaway_santanna_mpdta"``. A download failure + falls back to a checksum-valid cache entry when one exists, returning that + canonical data silently. Only when neither a verified cache entry nor a + verified fresh download is available - or when the source fails to parse or + validate - does the loader emit one ``UserWarning`` containing ``SYNTHETIC`` + and mark the returned frame as ``"synthetic_fallback"``. References ---------- diff --git a/docs/tutorials/09_real_world_examples.ipynb b/docs/tutorials/09_real_world_examples.ipynb index 8eb1ce4f7..3fe74f9ab 100644 --- a/docs/tutorials/09_real_world_examples.ipynb +++ b/docs/tutorials/09_real_world_examples.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "id": "cell-0", "metadata": {}, - "source": "# Real-World Data Examples\n\nThis notebook demonstrates `diff-diff` on three classic econometric study designs:\n\n1. **Card & Krueger (1994)** - Classic 2x2 DiD: Effect of minimum wage on employment\n2. **Castle Doctrine Laws** - Staggered adoption: Effect of self-defense laws on homicide rates\n3. **Unilateral Divorce Laws** - Staggered adoption on a long panel (**simulated data**)\n\nThe first two run on checksum-verified replication data *when their canonical sources are reachable*. If a download fails, the loader warns and falls back to a generated panel with the same schema, so the numbers below become illustrative rather than empirical. Each section prints `df.attrs[\"source\"]` after loading - check it before reading any estimate as a replication. Note also that these sections illustrate `diff-diff`'s estimators on the authors' data rather than reproducing each paper's exact specification; where the two differ, the section says so.\n\nThe third is different in kind: `diff-diff` has no verified public source wired up for the Stevenson-Wolfers divorce panel, so `load_divorce_laws()` generates data and warns on *every* call. Its numbers illustrate estimator mechanics on a long staggered panel - they are not estimates of any real policy effect." + "source": "# Real-World Data Examples\n\nThis notebook demonstrates `diff-diff` on three classic econometric study designs:\n\n1. **Card & Krueger (1994)** - Classic 2x2 DiD: Effect of minimum wage on employment\n2. **Castle Doctrine Laws** - Staggered adoption: Effect of self-defense laws on homicide rates\n3. **Unilateral Divorce Laws** - Staggered adoption on a long panel (**simulated data**)\n\nThe first two run on checksum-verified replication data *when their canonical sources are reachable*. A download failure falls back to a checksum-valid cache entry if one exists, which is still the canonical data. Only when neither is available does the loader warn and substitute a generated panel with the same schema, which makes the numbers below illustrative rather than empirical. Each section prints `df.attrs[\"source\"]` after loading - check it before reading any estimate as a replication. Note also that these sections illustrate `diff-diff`'s estimators on the authors' data rather than reproducing each paper's exact specification; where the two differ, the section says so.\n\nThe third is different in kind: `diff-diff` has no verified public source wired up for the Stevenson-Wolfers divorce panel, so `load_divorce_laws()` generates data and warns on *every* call. Its numbers illustrate estimator mechanics on a long staggered panel - they are not estimates of any real policy effect." }, { "cell_type": "code", From 49b85dc72d6788ba00cf9f5091655db90fd6d967 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 10:04:32 -0400 Subject: [PATCH 13/17] fix: clear_cache() could not remove interrupted atomic-write scratch files _write_cache_atomically creates ... beside the cache entry. The write-failure path unlinks it, but a hard kill between creation and os.replace strands one, and it matches neither *.csv nor *.dta - so it survived clear_cache(), the only remedy the docs offer, and accumulated across runs. Adds the two hidden patterns to the glob, guards on is_file(), and tests that an unrelated file in the cache directory is left alone. Also covers the missing-directory case, which is now reachable because the cache directory is created on write rather than on path construction. --- diff_diff/datasets.py | 14 +++++++++++--- tests/test_datasets.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 3f8dec30b..b74741607 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -698,11 +698,19 @@ def _validate_mpdta(df: pd.DataFrame) -> None: def clear_cache() -> None: - """Clear the local dataset cache.""" + """Clear the local dataset cache. + + Also removes any ``...`` scratch files left behind by an + atomic cache write that was interrupted between creating the temporary file + and replacing the cache entry (a hard kill, for instance). Those are hidden + and do not match the plain ``*.csv`` / ``*.dta`` patterns, so without this + they would accumulate and survive the documented remedy. + """ if _CACHE_DIR.exists(): - for pattern in ("*.csv", "*.dta"): + for pattern in ("*.csv", "*.dta", ".*.csv.*", ".*.dta.*"): for f in _CACHE_DIR.glob(pattern): - f.unlink() + if f.is_file(): + f.unlink() print(f"Cleared cache at {_CACHE_DIR}") diff --git a/tests/test_datasets.py b/tests/test_datasets.py index ffe9aac7b..153557992 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -495,6 +495,38 @@ def test_protocol_level_http_errors_warn_and_use_marked_fallback(self, tmp_path, assert result.attrs["source"] == "synthetic_fallback" assert result.shape == _construct_mpdta_data().shape + def test_clear_cache_removes_interrupted_atomic_write_scratch_files( + self, tmp_path, monkeypatch + ): + """``clear_cache()`` must clear the hidden scratch files atomic writes can strand. + + ``_write_cache_atomically`` creates ``...`` next to the entry. + A hard kill between creation and ``os.replace`` leaves one behind, and it matches + neither ``*.csv`` nor ``*.dta``, so it would otherwise survive the one remedy the + docs offer and accumulate across runs. + """ + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + (tmp_path / "mpdta.csv").write_text("cached") + (tmp_path / "prop99.dta").write_bytes(b"cached") + (tmp_path / ".mpdta.csv.ab12cd34").write_bytes(b"orphaned") + (tmp_path / ".prop99.dta.ef56gh78").write_bytes(b"orphaned") + keep = tmp_path / "notes.txt" + keep.write_text("unrelated file must survive") + + datasets_mod.clear_cache() + + assert sorted(p.name for p in tmp_path.iterdir()) == ["notes.txt"] + assert keep.read_text() == "unrelated file must survive" + + def test_clear_cache_tolerates_missing_cache_directory(self, tmp_path, monkeypatch): + """The cache directory is only created on write, so clearing must not require it.""" + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path / "never-created") + datasets_mod.clear_cache() # must not raise + def test_verified_cache_recovers_canonical_data_on_download_failure( self, tmp_path, monkeypatch ): From 8c8fa8336011dd065be0dc507bded945a44de372 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 10:23:59 -0400 Subject: [PATCH 14/17] fix: verified cache must survive every fresh-download verification failure A transport error fell back to a checksum-valid cache, but a size-limit breach or checksum mismatch raised straight through to the synthetic fallback. So with force_download=True and canonical bytes already on disk, a tampered or moved upstream silently replaced real data with generated data - the opposite of the fail-closed behaviour the pinned-mirror comment claimed, and a contradiction of the contract the previous two commits aligned six surfaces to. The cache is now read up front and retained under force_download, and all three failure modes recover from it. A checksum mismatch additionally warns that the upstream no longer matches the pin: an integrity event should not pass unnoticed, and the warning deliberately omits SYNTHETIC because no synthetic data is involved and callers key provenance checks on that word. The warning's stacklevel is computed rather than fixed. The text and binary loaders reach the download helper at different depths, so the constant that was correct for prop99/walmart pointed inside datasets.py for mpdta/card_krueger. --- CHANGELOG.md | 11 +++-- diff_diff/datasets.py | 93 ++++++++++++++++++++++++++++++++---------- docs/api/datasets.rst | 11 +++-- tests/test_datasets.py | 77 ++++++++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 001d733d6..d7d678f59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -401,9 +401,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 panel-invariant validation; successful frames expose a dataset-specific `df.attrs["source"]`. When neither a verified cache entry nor a verified fresh download is available, the loader emits exactly one `UserWarning` containing - `SYNTHETIC` and marks the frame `source="synthetic_fallback"`. A download that - fails while a checksum-valid cache entry exists returns that canonical data - silently, including under `force_download=True`. `load_divorce_laws()` now follows that explicit + `SYNTHETIC` and marks the frame `source="synthetic_fallback"`. Every way a fresh + download can fail verification - transport error, size limit, or checksum + mismatch - falls back to a checksum-valid cache entry when one exists, including + under `force_download=True`, so a tampered or moved upstream can never downgrade + a user holding verified canonical bytes to generated data. A checksum mismatch + additionally warns that the upstream no longer matches the pin (a distinct + warning that deliberately does not contain `SYNTHETIC`, since no synthetic data + is involved). `load_divorce_laws()` now follows that explicit fallback path because no verified source currently reproduces its composite public schema without additional analytical choices. Verified fresh downloads are returned even when best-effort cache persistence fails. diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index b74741607..5666b92e4 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -11,6 +11,7 @@ import hashlib import os +import sys import warnings from http.client import HTTPException from io import BytesIO, StringIO @@ -32,8 +33,10 @@ # ``njmin.zip`` from David Card's data archive, and ``mpdta.csv`` matches # ``did::mpdta`` (R package 2.5.1) to CSV round-trip precision (max absolute # difference 1.07e-14 on ``lemp``). Every cached and downloaded byte sequence is -# verified against the pinned SHA-256 below, so a later change at any mirror fails -# closed to the loud synthetic fallback rather than substituting different data. +# verified against the pinned SHA-256 below, so a later change at any mirror can +# never substitute different data: it falls back to the verified cache entry if one +# exists (with a warning naming the integrity failure), and to the loud synthetic +# fallback only when there is no verified copy to fall back to. _CARD_KRUEGER_SOURCE_URL = ( "https://raw.githubusercontent.com/rafiash/CardKrueger-stata-sample/" "07bc929f1d6552db117bd27a7cf0d881d16e9494/public.dat" @@ -68,6 +71,26 @@ class _DatasetSourceError(RuntimeError): """Expected failure while fetching, parsing, or validating canonical data.""" +def _caller_stacklevel() -> int: + """``stacklevel`` that attributes a warning to the first frame outside this module. + + The text and binary loaders reach the download helper at different depths (the text + path routes through ``_load_verified_dataset`` and a source adapter; the binary path + calls it almost directly), so no fixed ``stacklevel`` points at user code for both. + """ + level = 1 + try: + frame = sys._getframe(1) + except ValueError: # pragma: no cover - defensive + return 2 + while frame is not None: + if frame.f_globals.get("__name__") != __name__: + return level + frame = frame.f_back + level += 1 + return 2 # pragma: no cover - defensive + + def _get_cache_path(name: str) -> Path: """Get the cache path for a dataset.""" return _CACHE_DIR / f"{name}.csv" @@ -136,11 +159,24 @@ def _download_verified_bytes( cache_path: Path, force_download: bool = False, ) -> bytes: - """Return checksum-verified bytes from cache or a fresh download.""" - if not force_download: - cached = _read_verified_cache(cache_path, sha256) + """Return checksum-verified bytes from cache or a fresh download. + + The cache is read up front and retained even under ``force_download``, so that + EVERY way a fresh download can fail verification - transport, size limit, or + checksum - can still fall back to bytes that already passed the pinned hash. + Falling through to the synthetic frame while verified canonical bytes sit on + disk would be a downgrade, and on the checksum path it would let a tampered + or moved upstream quietly replace real data with generated data. + """ + cached = _read_verified_cache(cache_path, sha256) + if cached is not None and not force_download: + return cached + + def _recover(message: str, cause: Optional[BaseException] = None) -> bytes: + """Prefer verified cached bytes over failing into the synthetic fallback.""" if cached is not None: return cached + raise _DatasetSourceError(message) from cause try: with urlopen(url, timeout=30) as response: @@ -150,21 +186,33 @@ def _download_verified_bytes( # from ``OSError``, so catching the base class keeps the whole family inside the # documented warn-and-fall-back boundary rather than only the named siblings. except (HTTPError, HTTPException, OSError, TimeoutError, URLError) as e: - cached = _read_verified_cache(cache_path, sha256) - if cached is not None: - return cached - raise _DatasetSourceError( + return _recover( f"Failed to download dataset '{name}' from {url}: {e}\n" - "Check your internet connection or try again later." - ) from e + "Check your internet connection or try again later.", + e, + ) if len(content) > _MAX_DATASET_BYTES: - raise _DatasetSourceError( + return _recover( f"Dataset '{name}' downloaded from {url} exceeds the " f"{_MAX_DATASET_BYTES}-byte safety limit." ) if hashlib.sha256(content).hexdigest() != sha256: + if cached is not None: + # Canonical bytes are already on disk, so the user keeps real data - but a + # pin mismatch is an integrity event, not a transport hiccup, and must not + # pass unnoticed. Deliberately not a SYNTHETIC warning: nothing synthetic + # is involved, and callers key their provenance checks on that word. + warnings.warn( + f"Upstream copy of dataset '{name}' at {url} no longer matches the " + "pinned SHA-256. Returning the verified cached copy instead. Verify " + "the source revision before updating the pinned checksum; until then " + "treat the upstream file as untrusted.", + UserWarning, + stacklevel=_caller_stacklevel(), + ) + return cached raise _DatasetSourceError( f"Checksum mismatch for dataset '{name}' downloaded from {url}.\n" "The upstream file differs from the pinned SHA-256. Verify the " @@ -761,9 +809,10 @@ def load_card_krueger(force_download: bool = False) -> pd.DataFrame: The canonical data are checksum-verified and returned with ``df.attrs["source"] == "card_krueger_public_data"``. A download failure falls back to a checksum-valid cache entry when one exists, returning that - canonical data silently. Only when neither a verified cache entry nor a - verified fresh download is available - or when the source fails to parse or - validate - does the loader emit one ``UserWarning`` containing ``SYNTHETIC`` + canonical data; a pin mismatch additionally warns that the upstream file has + changed. Only when neither a verified cache entry nor a verified fresh + download is available - or when the source fails to parse or validate - does + the loader emit one ``UserWarning`` containing ``SYNTHETIC`` and return ``df.attrs["source"] == "synthetic_fallback"``. References @@ -911,9 +960,10 @@ def load_castle_doctrine(force_download: bool = False) -> pd.DataFrame: The canonical data are checksum-verified and returned with ``df.attrs["source"] == "cheng_hoekstra_castle_data"``. A download failure falls back to a checksum-valid cache entry when one exists, returning that - canonical data silently. Only when neither a verified cache entry nor a - verified fresh download is available - or when the source fails to parse or - validate - does the loader emit one ``UserWarning`` containing ``SYNTHETIC`` + canonical data; a pin mismatch additionally warns that the upstream file has + changed. Only when neither a verified cache entry nor a verified fresh + download is available - or when the source fails to parse or validate - does + the loader emit one ``UserWarning`` containing ``SYNTHETIC`` and mark the returned frame as ``"synthetic_fallback"``. Replicating Cheng-Hoekstra (2013) requires the paper's regressor and outcome: @@ -1295,9 +1345,10 @@ def load_mpdta(force_download: bool = False) -> pd.DataFrame: The canonical data are checksum-verified and returned with ``df.attrs["source"] == "callaway_santanna_mpdta"``. A download failure falls back to a checksum-valid cache entry when one exists, returning that - canonical data silently. Only when neither a verified cache entry nor a - verified fresh download is available - or when the source fails to parse or - validate - does the loader emit one ``UserWarning`` containing ``SYNTHETIC`` + canonical data; a pin mismatch additionally warns that the upstream file has + changed. Only when neither a verified cache entry nor a verified fresh + download is available - or when the source fails to parse or validate - does + the loader emit one ``UserWarning`` containing ``SYNTHETIC`` and mark the returned frame as ``"synthetic_fallback"``. References diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index 2b78fd760..ace237e07 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -11,9 +11,14 @@ immutable upstream commit, so the URL itself cannot change under them; ``load_prop99()`` and ``load_walmart()`` read a mutable SSC path over plain HTTP, where the checksum alone establishes integrity. -Every loader reports its provenance through ``df.attrs["source"]``. When a source cannot -be obtained, the loader emits one ``UserWarning`` containing ``SYNTHETIC`` and returns a -generated fallback frame marked ``df.attrs["source"] == "synthetic_fallback"``. For +Every loader reports its provenance through ``df.attrs["source"]``. A fresh download that +fails verification - transport error, size limit, or checksum mismatch - falls back to a +checksum-valid cache entry when one exists, so canonical data is never downgraded to +generated data while verified bytes are on disk. A checksum mismatch additionally emits a +``UserWarning`` naming the integrity failure, since it means the upstream file no longer +matches the pin. When no verified copy is available at all, the loader emits one +``UserWarning`` containing ``SYNTHETIC`` and returns a generated fallback frame marked +``df.attrs["source"] == "synthetic_fallback"``. For ``load_card_krueger()``, ``load_castle_doctrine()`` and ``load_mpdta()`` that fallback also covers parse and schema-validation failures; ``load_prop99()`` and ``load_walmart()`` run their validation outside the fallback boundary and raise instead. Numbers computed on diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 153557992..117808f54 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -495,6 +495,83 @@ def test_protocol_level_http_errors_warn_and_use_marked_fallback(self, tmp_path, assert result.attrs["source"] == "synthetic_fallback" assert result.shape == _construct_mpdta_data().shape + @pytest.mark.parametrize( + "failure", + ["transport", "checksum", "oversized"], + ) + def test_verified_cache_survives_every_fresh_download_failure( + self, failure, tmp_path, monkeypatch + ): + """Canonical cached bytes must never be downgraded to the synthetic fallback. + + The transport path already recovered from cache, but the size-limit and + checksum paths raised straight through, so a tampered or moved upstream + replaced verified real data with generated data for every user holding a + valid cache. A checksum mismatch additionally warns, since that is an + integrity event rather than a transport hiccup. + """ + import hashlib + import warnings + + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + source = _construct_mpdta_data().rename(columns={"first_treat": "first.treat"}) + payload = source[["year", "countyreal", "lpop", "lemp", "first.treat", "treat"]].to_csv( + index=False + ) + monkeypatch.setattr( + datasets_mod, "_MPDTA_SOURCE_SHA256", hashlib.sha256(payload.encode()).hexdigest() + ) + (tmp_path / "mpdta.csv").write_text(payload) + + response = MagicMock() + response.__enter__ = lambda self: self + response.__exit__ = lambda self, *a: False + if failure == "transport": + response.read.side_effect = TimeoutError("network down") + elif failure == "checksum": + response.read.return_value = b"upstream was tampered with" + else: + response.read.return_value = b"x" * (datasets_mod._MAX_DATASET_BYTES + 1) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with patch("diff_diff.datasets.urlopen", return_value=response): + result = load_mpdta(force_download=True) + + assert result.attrs["source"] == "callaway_santanna_mpdta" + assert not [w for w in caught if "SYNTHETIC" in str(w.message)] + + integrity = [w for w in caught if "no longer matches the pinned" in str(w.message)] + if failure == "checksum": + assert len(integrity) == 1, "a pin mismatch must not pass unnoticed" + assert "diff_diff" not in integrity[0].filename, "warning must blame the caller" + else: + assert not integrity + + def test_tampered_upstream_without_cache_still_falls_back_to_synthetic( + self, tmp_path, monkeypatch + ): + """With no cache to recover, a pin mismatch must still take the SYNTHETIC path.""" + import warnings + + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + response = MagicMock() + response.__enter__ = lambda self: self + response.__exit__ = lambda self, *a: False + response.read.return_value = b"upstream was tampered with" + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with patch("diff_diff.datasets.urlopen", return_value=response): + result = load_mpdta(force_download=True) + + assert result.attrs["source"] == "synthetic_fallback" + assert len([w for w in caught if "SYNTHETIC" in str(w.message)]) == 1 + def test_clear_cache_removes_interrupted_atomic_write_scratch_files( self, tmp_path, monkeypatch ): From d5c7e2a0fd6db54ba42e552a50d52b14d4bcbe72 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 10:36:32 -0400 Subject: [PATCH 15/17] docs: two internal contract statements still said a checksum mismatch raises The semantic sweep after the cache-recovery fix searched for the phrasings I had written, so it missed two pre-existing ones: _download_with_cache_binary's docstring ('a checksum mismatch on freshly downloaded bytes raises') and the module docstring ('if a source cannot be verified, the loader warns and returns a synthetic fallback'). Both now describe cache recovery as the first resort. --- diff_diff/datasets.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/diff_diff/datasets.py b/diff_diff/datasets.py index 5666b92e4..654c04130 100644 --- a/diff_diff/datasets.py +++ b/diff_diff/datasets.py @@ -5,8 +5,10 @@ commonly used for teaching and demonstrating DiD methods. Canonical data are downloaded from checksum-pinned public sources and cached -locally. If a source cannot be verified, the loader warns and returns an -explicitly provenance-marked synthetic fallback. +locally. A download that fails verification falls back to a checksum-valid +cache entry when one exists, so verified data on disk is never displaced by +generated data. Only when no verified copy is available does the loader warn +and return an explicitly provenance-marked synthetic fallback. """ import hashlib @@ -243,8 +245,10 @@ def _download_with_cache_binary( """Download a binary file (e.g. Stata .dta), verify its checksum, and cache it. Every byte-load (cache or fresh download) is verified against a pinned - SHA-256. A stale/corrupt cache triggers one re-download; a checksum - mismatch on freshly downloaded bytes raises. + SHA-256. A stale or corrupt cache triggers one re-download. A checksum + mismatch on freshly downloaded bytes falls back to a verified cache entry + when one exists (warning that the upstream no longer matches the pin), and + raises only when there is no verified copy to fall back to. """ return _download_verified_bytes( url, From d126b6de66ba84ee311d4ce643e2d3b7ee2dc2a0 Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 10:44:35 -0400 Subject: [PATCH 16/17] fix: test_clear_cache_creates_directory wiped the developer's real dataset cache The test called clear_cache() with _CACHE_DIR unpatched, so every suite run deleted ~/.cache/diff_diff/datasets. That was survivable while the loaders served synthetic data; now that the cache holds verified canonical downloads, it forces a re-fetch of all three pinned sources after every test run. Pinned to tmp_path, and an AST sweep confirms no other test touches a loader or clear_cache without either pinning the cache directory or mocking the download. Also refreshes the CI-canary backlog row, which pointed at 'the loader-fallback repair row above' - a row this branch removed as completed. --- TODO.md | 2 +- tests/test_datasets.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index 21e052604..a472e2401 100644 --- a/TODO.md +++ b/TODO.md @@ -61,7 +61,7 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m | MMM interop PR-B: calibration tutorial notebook (fit DiD/CS -> scope -> `to_pymc_marketing_lift_test` / `to_meridian_roi_prior`) + a `llms-practitioner.txt` Step 8 pointer to the exporters as the MMM hand-off. | `docs/tutorials/`, `diff_diff/guides/llms-practitioner.txt` | mmm-interop | Mid | Low | | Section-8 naming-completeness guard test: sweep the public surface (module-level functions, class `__init__`/`fit` params, results dataclass fields) for contract-rename violations and fail unless each hit is either a `docs/v4-deprecations.yaml` row or an allowlist entry carrying a stated reason (seed the allowlist with the documented domain-vocabulary exemptions: the staggered family's ATT(g,t) `group`/`groups`, `TripleDifference.fit[group]`, wrapper params dying with `M-070..M-077`, and params inherited from `DifferenceInDifferences.__init__`). Today the rules are normative prose and the ledger holds only what an audit remembered — two successive manual sweeps each found surfaces the prior one missed (`M-094`/`M-095`, then the 19-row `M-097..M-115` function sweep). Should land BEFORE Phase 2c so the rename PR works from a mechanically-verified list. Two further checks belong in the same guard, each having been missed by hand across successive review rounds: (a) **phase-table agreement** — every row's `phase` must appear in the matching `docs/v4-design.md` section 9 table entry and vice versa (the section 9 checklist asserts this agreement but nothing verifies it, and three separate rounds caught a stale table); (b) **consumer coverage** (section 8 rule 11) — for each rename row, grep `diff_diff/` and `docs/methodology/` for the old name and require every hit to be either in the row's `code_refs` or allowlisted, since `getattr(obj, "old_name", default)` degrades silently rather than raising after removal. | `tests/`, `docs/v4-design.md` | gating-completeness amendment | Mid | Medium | | Tracking-file contract guard test: reject NEW active deferred-work pointers at `TODO.md` (deferred rows live in `DEFERRED.md`; allowlist for historical/past-tense prose and actionable-row pointers) and assert rows cross-linking a `docs/v4-deprecations.yaml` `M-xxx` id don't restate ledger status. Origin: tracking-split local review R2. | `tests/`, `TODO.md`, `DEFERRED.md` | tracking-split | Quick | Low | -| Real-data CI canary for dataset-backed replication tests: `test_methodology_lwdid.py`'s Prop 99 / Walmart goldens skip (visibly) when loaders fall back to synthetic; add a lane or canary asserting `df.attrs["source"] == "lwdid_ssc_ancillary"` in CI so network regressions cannot silently de-gate the replication tests. Pairs with the loader-fallback repair row above. | `tests/test_methodology_lwdid.py`, `.github/workflows/` | LWDiD validation suite | Quick | Low | +| Real-data CI canary for dataset-backed replication tests: `test_methodology_lwdid.py`'s Prop 99 / Walmart goldens skip (visibly) when loaders fall back to synthetic; add a lane or canary asserting `df.attrs["source"] == "lwdid_ssc_ancillary"` in CI so network regressions cannot silently de-gate the replication tests. Follow-on from the loader-fallback repair (#723), which made provenance explicit but deliberately did not add a network-dependent CI lane. | `tests/test_methodology_lwdid.py`, `.github/workflows/` | LWDiD validation suite | Quick | Low | | `worktree-rm` safety via a tested argv helper: the prose rewrite (ask-before-remove confirmation gate, detached-HEAD reachability/rescue, tip-identity force-delete guard) was reverted to the main version because editing the prose repeatedly reintroduced shell-injection (last: sourcing a state file built from a git-derived branch name). Restore those guards in a `worktree_rm.py` that takes the name via file ingress, invokes git through argv arrays, and has a metacharacter-branch/path injection regression test — the pattern that worked for `pr_prepare.py`/`premerge_scan.py`. | `.claude/commands/worktree-rm.md`, `.claude/scripts/` | skill-audit | Heavy | Medium | | `premerge_scan.py` should scan the staged blob (`git show :path`) for staged methodology files, not the working-tree copy — a stage-then-revert-working-copy edit currently reads the safe working version and misses the staged violation. Union staged-index findings with unstaged/untracked filesystem findings. | `.claude/scripts/premerge_scan.py` | skill-audit | Mid | Low | | Re-add committed-range methodology scanning to `/push-pr-update` §3b (clean tree, commits ahead) using `premerge_scan.py --range`, with the comparison ref passed as **data** (resolved into a quoted variable in one Bash call, never a raw ``). It was removed to avoid ref interpolation; the helper already implements and tests `--range`. | `.claude/commands/push-pr-update.md` | skill-audit | Quick | Low | diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 117808f54..b839ac016 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -1236,9 +1236,16 @@ def test_failed_atomic_replace_returns_verified_download(self, tmp_path, monkeyp class TestClearCache: """Tests for cache management.""" - def test_clear_cache_creates_directory(self): - """clear_cache should handle non-existent cache gracefully.""" - # This should not raise even if cache doesn't exist + def test_clear_cache_creates_directory(self, tmp_path, monkeypatch): + """clear_cache should handle non-existent cache gracefully. + + Pinned to a temporary directory: unpatched, this ran against the real + ``~/.cache/diff_diff/datasets`` and deleted the developer's canonical + downloads every time the suite ran. + """ + import diff_diff.datasets as datasets_mod + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path / "absent") try: clear_cache() except Exception as e: From 03c5433b4ab1072708f53ea87683dd19e841b5fd Mon Sep 17 00:00:00 2001 From: igerber Date: Mon, 27 Jul 2026 13:36:49 -0400 Subject: [PATCH 17/17] test: exercise the pinned production mpdta bytes end-to-end offline Every other canonical-path test substitutes an already-normalized fabricated frame, so the real parse - column naming, first.treat renaming, dtypes - was only ever covered by a live download and never by CI. The canonical bytes are already committed at benchmarks/data/real/mpdta.csv and hash to exactly the pinned digest, so the full loader path runs offline for free. Also guards the pin: re-pinning to a revision that does not match the committed fixture now fails here rather than silently at runtime. Verified the test has teeth by perturbing the digest and confirming it fails. Raised by the CI reviewer on the #723 mirror (#731). --- tests/test_datasets.py | 48 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index b839ac016..ab4bacdee 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -495,6 +495,54 @@ def test_protocol_level_http_errors_warn_and_use_marked_fallback(self, tmp_path, assert result.attrs["source"] == "synthetic_fallback" assert result.shape == _construct_mpdta_data().shape + def test_production_mpdta_bytes_load_end_to_end_offline(self, tmp_path, monkeypatch): + """Drive the real pinned bytes through the whole loader, with no network. + + Every other canonical test substitutes an already-normalized fabricated frame, + so the actual parse of the production file - column naming, ``first.treat`` + renaming, dtype handling - is only ever exercised by a live download. The + canonical bytes are already committed for the benchmarks, and their digest is + the pin, so the full path can be covered offline for free. + + Doubles as a guard on the pin itself: re-pinning to a revision that does not + match the committed fixture fails here rather than silently at runtime. + """ + import hashlib + import warnings + + import diff_diff.datasets as datasets_mod + + fixture = Path(__file__).resolve().parent.parent / "benchmarks/data/real/mpdta.csv" + if not fixture.exists(): + pytest.skip(f"{fixture.name} not committed (partial checkout)") + + payload = fixture.read_bytes() + assert ( + hashlib.sha256(payload).hexdigest() == datasets_mod._MPDTA_SOURCE_SHA256 + ), "pinned mpdta digest no longer matches the committed canonical fixture" + + monkeypatch.setattr(datasets_mod, "_CACHE_DIR", tmp_path) + response = MagicMock() + response.__enter__ = lambda self: self + response.__exit__ = lambda self, *a: False + response.read.return_value = payload + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with patch("diff_diff.datasets.urlopen", return_value=response): + df = load_mpdta(force_download=True) + + assert not caught, "production bytes must load without warning" + assert df.attrs["source"] == "callaway_santanna_mpdta" + # Anchors from the R `did` package's documented panel. + assert df.shape == (2500, 7) + assert df["countyreal"].nunique() == 500 + assert set(df["year"]) == {2003, 2004, 2005, 2006, 2007} + assert set(df["first_treat"]) == {0, 2004, 2006, 2007} + assert (df["cohort"] == df["first_treat"]).all() + assert (df["treat"] == (df["first_treat"] > 0).astype(int)).all() + assert df.notna().all().all() + @pytest.mark.parametrize( "failure", ["transport", "checksum", "oversized"],