diff --git a/moderndive/data/early_january_2023_weather.parquet b/moderndive/data/early_january_2023_weather.parquet index 9f8be5b..5792834 100644 Binary files a/moderndive/data/early_january_2023_weather.parquet and b/moderndive/data/early_january_2023_weather.parquet differ diff --git a/tests/test_data_and_theory.py b/tests/test_data_and_theory.py index cef3dd8..ab91052 100644 --- a/tests/test_data_and_theory.py +++ b/tests/test_data_and_theory.py @@ -40,6 +40,18 @@ def test_unknown_dataset_raises(): md.load_dataset("does_not_exist") +def test_early_january_2023_weather_has_real_measurements(): + # Regression: the R dataset ships temp/dewp/humid/pressure as all-NA; this + # package derives the table from `weather`, so those columns must be numeric + # and fully populated (Newark, first 15 days of January 2023 = 360 hours). + df = md.load_early_january_2023_weather() + assert df.height == 360 + assert df["origin"].unique().to_list() == ["EWR"] + for col in ("temp", "dewp", "humid", "pressure"): + assert df.schema[col].is_numeric() + assert df[col].null_count() == 0 + + def test_t_confidence_interval_matches_manual(): x = np.array([2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]) out = theory.t_confidence_interval(x, level=0.95) diff --git a/tools/build_data.py b/tools/build_data.py index 5df92d2..c53c40a 100644 --- a/tools/build_data.py +++ b/tools/build_data.py @@ -44,7 +44,6 @@ def _write(df: pl.DataFrame, name: str) -> None: def main() -> None: from_r = [ "envoy_flights", - "early_january_2023_weather", "weather", "flights", "airlines", @@ -82,6 +81,15 @@ def main() -> None: for name in from_r: _write(_read_csv(name), name) + # early_january_2023_weather: the R `moderndive` dataset ships with + # temp/dewp/humid/pressure entirely NA, so derive it from `weather` (Newark, + # first 15 days of January 2023) to get the real, correctly-typed values. + weather = _read_csv("weather") + ejw = weather.filter( + (pl.col("origin") == "EWR") & (pl.col("month") == 1) & (pl.col("day") <= 15) + ).sort("time_hour") + _write(ejw, "early_january_2023_weather") + # Gapminder ships a CSV inside the `gapminder` PyPI package. We read that file # directly (the package's Python import is broken on 3.14: it uses the removed # pkg_resources). The data matches the canonical R gapminder dataset.