From f10be6deac9b2c12ef12e6595362e27225d4681e Mon Sep 17 00:00:00 2001 From: Chester Ismay Date: Thu, 18 Jun 2026 17:04:10 -0700 Subject: [PATCH] fix(data): re-derive early_january_2023_weather from weather The R `moderndive::early_january_2023_weather` ships with temp, dewp, humid, and pressure entirely NA (mean temp is NaN), which carried through to the bundled Parquet as all-null string columns. Derive the table from `weather` instead (Newark / EWR, first 15 days of January 2023) so those four columns are real, correctly-typed floats. Same 360 rows. build_data.py now produces it from the freshly built weather frame rather than reading a separate CSV, and a regression test asserts the measurement columns are numeric and non-null. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_017CTL1QSTg1DmDUpqYuPEog --- .../data/early_january_2023_weather.parquet | Bin 8102 -> 11255 bytes tests/test_data_and_theory.py | 12 ++++++++++++ tools/build_data.py | 10 +++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/moderndive/data/early_january_2023_weather.parquet b/moderndive/data/early_january_2023_weather.parquet index 9f8be5b1708468db9bfc49728fef734cd800f4ae..5792834ec5fed7295acf5503d4dfc637b529877f 100644 GIT binary patch delta 6905 zcmeHLXIPU>vwkQEMT(S!7AYc4dM{EEdJ7~WbX0n8(h($}6cvyXR8T+!RFoned~GO2 z5CIE{NKsl|l`0@u5YLkY(9iRp@4L>w^W*Gw@nmLa=HA)e*aE+?9;71xurso- zLIxSwQwR)zL19pY8X(CCkXs@K$xEdGBogE$o8*7EnlOv80s0n#g0*6PEE!r4(%c+8 z3(L|RM%0Ec&dIKyTa@XktFDzQkMGwM*f)E@Zjd*;1%40XmsmKUb!JNA%Q00RW#wJ^ z#(MK^Ppi252t^~lJn3H-EK8Ev!c-q5O3E{;Ww7FE4_$GWZIZaMe|L+7Ca$BG>&+=y zms0xONDLiq5@W@)*u0rKsFH!<7$;J=HU2MgefNA7MWx0PjRI0-x8PzODTd(y4)GSQ zdSI?|la15kYg~6JjJu6q#?cwyAvS25W*oaTWt;g$hLD`oe6=#;AciqC>ehLj04FU; z6HnW_DP%jn$314ZgJoDzuA}PDWK(DnBT^nZS3AWXeM?|oVm!Z|wd+Pi{Y8iNyf|Gh z-8G-aQbo1;4&BPb2CcfzE%?&-mO7=5+ZcrURo#?KjtuDI#BdZosdyl)J|gbsJKpji z#|%mb3<;Nh2nq#b(GNVIFUZ#v_HRx~0EC#e2+`9mYlgboq$S(nuWi3X<8iJ*zq2{-|I#)Hj> zcrdqr?85iE^M>>z9iBX*M+Y>RXibjORronWV9b03F#yWPOCRAC6bfU3KZU?avhY;~ z@m5yt|0xW>C;=m;`yV3)ummHf?+Lj{{T4AV*#KH*A(0aGJ$k%mwvj%A>qAHBMw(rQ zEzpGS!?!O6#xcM6ymE+!`ce$WZlQU8HWXwGy{fFEHRZc5jsr{DM2YKg&XN#0apU-@?-rQ%0 zf28nX<&gKG@a}U3A^B?AB#9INV##W&grBjfR?1OwZwq63ru~R0&OPP(zS0?Rxn8j- zv4Ci}^~0T{>;9Zusj0)8VAEaT@}x6&=J;6umD|1MB9fJxwb2EN(%)fYu~7DEd(THf zgwt{drH2C-z1!;x&$Pd*Jea*6 zr5>fR|3=MG5tpmkvHq2X23Akbx`p^hNn2Z*H7dN`?*&<)^_~98$HLIj;ROZ~HI6*J zqJPCv38Q!({8t?P5ggPwBD~@Fr_=*Df`KHIGK*~;*ZM7x_J{%45{2;j4TJ0heQpLf<^{wk|3a}S*R6~Th1H_xT4)K$T=(sUTKZQYM@;) zCLGwySbN-e98&U$O$8RPalnC@lG)z^S!|BmkMZOdri zfQY+fy3rtYjJpFXtiyw?F}zeyK!4a zM`e#n{5P9Pn;~0&I~)pSfBLYUls%Gi$SmzLGi__|3Klq5j4$RZh7QYb&0x(giK#z; zW5-QJVMld7qR*S2HalUKXnxQv$D$BTUQ)psD=zC`%Nlf>)enL-GmoXel?fWkY_Z#X z-BeQBc(1XQ)e}rG7O{l85o#MKn1bKH-7z-OFdQ&&H{vyMx3Dy|lpI|c;IMv(R(^pd zT6$QVwlcR3wW^{$yKS1DBavz@qZ=G{wk68b>&p07!G6P@xpxD&8 z%W+Ka_&sIHuJK-aN5k!3aE&2EH|x7f(}8%2C`;gAX0_XWh0)lTBnw_O^mG_WQRZx& z8{0~-`{#7eniGyLPKIH7qLAUA?g!VXFg{?kIv2E(pWAM2l9>@RWF|8Qx%jigZ9>|z zr^@;Eig|1)q5DwPk&nPpo`$httX1m8Vy@fXUJru4*VQdtScKS&@tok`VR38uYCgXI zg?MlrzX?sG!+qYxiR$ife9 zhVDauMiw6fy0(mn@qx|*(94tSdvXh-{f(uCo}ZHAArNR6q^>>ea#1e*d}LUkYl8Wq z$f8Rm=|)>B;)(iFUenv<)Q=8QHrE_)o~S~#+?V}Xpq4csB?|QoZZ&q6{xnCQVI<#O z)fja;^3}ucxqQa&+_OK`zjg8t_nnUTrj{vs>wKh8lT-c0g~UPD0Grqri3?e0o6@2y z6l@FSf(FN82tFr^f~ICBnC=zibylt16&RKq&M5ZxX=Lm?DX%mSzt%k@)9>g;SN$os z$Vs|1UEtHcH`YFq=@~;4j~+1vKt@%>Ta}0+e0&VlB$`1ihsv4Z9!r1gS?~@2Ac;Tgpl~Phb)=SyPoS&8D;iRSI$>w49uKIrmCP z?Byg%G3+ILa|!C))K^3%u~r;8e63cQfd-OnW`6l44TmqwrR&-BS0sn|=(x1KXC30A zR9_E^HZy$>0<8AEjeK=t@1dEG7O`?rR@H^->iaLVA2_Gg{E+LrZ}TdcQ|+LTX}BEx zIk5;XUdZ1N-0@dhi$;x4xZGV#F(22%;MJ19Z&`dkKSTJg++x-^71&bvbklf1Dfv8C zQ4MO(^m>=(J&2$eeyvrR_FK~n-BfgRWK8|BsO#s#wL;IMAMSDrjOm+t?^;q!AHDff zO;a7XxT$`OVRyqg+hdX)ROg=5Wb4V5NM(WOqUNb^?t`(TIuRXC3Xg!szKJxMsQR>Z zxmSkE`C~bRJ;v(m5yRT%Zb=S@r#B)+*UPHD(X>nn^@a!KUaZR&Wq7HMQXH8QJC)opu`?V1az8F zd|6AISPSLd*==SfnSH7Z{!1OLSq{!v%vtVEhp9 z3xUDJU}F3Ci@}IlOnV7yDxSIKW6J-((f=1VT7H)R1l;JyhD7UK2m)ANf1~}QETWV( zN6YoLZ)89sSO#s~$ci9<3hma(r|Ko^$r>PLC)N0Yay@cggl#R(0|1kXwkut=0*SOz zO3r&YYj^LKPHhu$<##i7J$;$G|bDx?^oSOcupr?Vns=? z|1@iA&TC`&V}Zcs006e^?F)!@782yQ7O06ASY_W@An!Avv|?nlp_z8D;RH$$2ldObWkKKXun_|@~i?uU2VYObBn z%gWA3Kb4f4l6JDNyo8vYnUaxTTwYOk>rQjy&6>vghNh;L*4y=e^^d%rcsDjh`Y<>B z_rmJejZGjM$fl*uhGat-nVDHw*%)ZDe`bC8{AqD?u!YdrTv<|frKYy-ue%*jo_6#+ zd--~F^5YM>jjZoKY1fvr7UtjdwwISAMEJYep)Cyc^pOVICU_6u(7^D}u(#TG8Cu(b{^CQY&_3A(N~sa%PBr1J9Bz&1ZdeID{1@anHHoN>wzXwLzbn1u*sH zsf`6gGz3-XC4%8hYh%beP06fJ0fs>?z9K58j`(H}w1sY3&L`f`Tpqx?c#W_Q|LAdt z-pytgT5Qhq_Ik*cmhxDFp~TNer|q8VfvJ_6cLh=_&WiccP-hzN&tD}+1PvBS@<)0H zi&lFCC5AJ*JjG;*^KzuDRJx6%w{@CXN4`PWmG}rRzM06JS`Qt8ojf0Qq8N?Jn7lW$ zQ7)ikR+}==_c6f=^8&W)&vnZ=Zol}K{B-x|YJ7Zm1c!S}8SCq=8dv01sZedbqWWi@ms}iL zJZ{$(5e(nBZ!R<4&zLwdCnDQEss>p)^x)f%WxvTn%j^CvD))!2v{w#=Uuvy(B;2}H z@!tKx+^#04rPW8Y!SgkRcNpee9__c#cvo`%F;DYcgu32z>H)0rg1F%Af#kII3&51@ zf!A>%_k#M~==9RP9}9mM@zw+X;fAXCKsj6RabwpULL4BlEHjaK>978|Sw4p`PKAh; z&v!@W`*v3y%zUwtDfU#&!aTcJxYsjeOeb~}Syn;LF@53MN)#yC&xM?a{ zd*w$}>%{u~xSzMrjK*SuH%6amTrG0b)%bE^_^O6@ZOnWPAO9Do4P*&)`yeuc2QfT6 zQX$N1r`E}56(7g_s&)Q2U(}WS4XB819K_r;$_?+tV*x%ajx2qzo5 zCFUh}39k(NxNFR9^eo2Gv6ghn&;Gp8{^y9Uymo-T=LtQnZ-r~KCBwV3U7Qsd2erK! zG2H2Shb^VEs4*RX7|ba0dIDt7u1y_&zTCZPW2jF4`&i*+1vtj+roZX#Pf ztjCFs&k4 zlO;bv$XR47YedQcc>Arau_K)>agsWzqPSnCn?d@i^LWH!qYUqfblE#_cRAM^8uAUV zSPdW!AYhVr1;;*_yFb`IN#Ca72-Smrsu-&hPcyk#XdN3fIK8nu#o`hx(x?bg?Bp^y zy)@WaCeEJq#w#)`W-!=}*j|S2zoxD2AC~ZV0!vK02A5aWL|A4Jv$=%f`Cioh_*c(~ zU0nG@VQzXm@&o|z!H#e1KLvO_`<8bGSX# zsn9?oDBMoWzuQuv!MwX|3kC~COcaJmuy3s{Sw?o8Xa_SSw6elh89>hfJ_I}u4l7~Y zq6>v!^_*Mus0yqYvPCaz!KSx$9vl9mORQke>9%ynaj<;WEt=#C>)!6q_kx||CHKQ< z`8IZ&(uakF5MgjbVP-K}>hl)>qmbkm3@S;j`ibdqJyx=0>u`j55y#-J>?J8QFm8Bm z0&G7ZzDkG48P#BdH-dd|_?-k|IuDBRF@^b-KzzZY$M_Cpc;T%CA|Ef3@P)#x5n#H2 zcoht>9iji5H#m59DjBdbk%kP?DEJHHKR>{@?mW-ID<~+-RFrRB-2t!oT}9MZ7>2zcN7lQPl2+FHsb#j#KPw(8L@*#7-^5M}VJ9BTw}s zRlJPdaq`cq;olbx@Dd74#RlNh2yTK?=Zi+;xfDtwX<%2?nq0CSyQy&8qO5cpk8Y)_qN9MyZ%JT9t(PWY=K~a=YMuudX+`^;O<>jM6FHmGxP$&;m^FR+}D@BhW zTQZAM`49FgpgsQteDP!VK#EX6JI*xTDp-F1-H{KFm;n3E#|{H4su%ihk{gJNzN4kF1@ zVvlO=2%77lP&g159NLH+s^niRHEy;8N}fDGG?pwfAv@r%Y#!>Q6c`EG>JUAJv=|qe K004u+DgGD1X`$@^ delta 3709 zcmeHJdr(tX8b3)0WEBE|+-nkpKtcinZh|PqXL5N6Dvu-rAryCcg{KiDJVfb6cYt9< zRJ=8URmSb2!&t{v0i`~=thLZqV9`=#Ex2eYx;{{>gO9Cr&$$U;JJ$N&{uD3jpBVDtqY;XaFuSCrspr^)lgXi`fbTdzq7R76AYQ zE)WN^=Pv}dPJEtCq27>>+2g%TJOMA04m(KO94toIn**5MIzqa>tp8mK(KB>z>kjhe z9>cPNtev&F{vqE!d{|VR)AiI=Q{Pqbq%8U`WK4JSMw=U-e!5`!7ah{a-~CCfJlgZT z<*cqqSaYb=^{i>7zUchNl`9ryULg8)`ichswUrMzP4}Xg_}}BCK5EN0evvo)kM_UE zwa?sEoaM^eJjAPSx$k%4gD&pg&%8UOIbhkP{-M#qJMJUnoOQDbPCvR>d@N_-?wz7{ zuMURP6wLg5hGWxU_s#D1&Fs?`ibO)PKHjw3096>H>8ybl!Aei z^CM{AJr;g*Q=@`srOFkPP+I;S4GhzHaLKTb2TWRWAYjnsfMM4_2X-`j zdNHurmMa<+ZF4H_soNUfw*dhJ_}=x%^8K0yYzAKfZ23If`~rQNDccH-@y0%4RZxBe=E^E3@q9|2Xik+)y>Qrt^Jf?nHfYk*-nHof5goUb(9) z(mlH=`BbFc(4G$GCRKi*XnVd(HTnD1U;eujoQwEpV%68xcT>d%MR&`@P|qJv7plg^ z!bK}G>LVVG4PM%jNqg9291^tt;9FN-5aqp?oxJVh+6~|LEXmVMwvSnyZOl%!0RPB5 zyn`XV?@tEs{pCrm9!ZaxfX4#KM}lNy=od)+Ba_2t6#W*&YA4}&Qrg_i4`G^yJ_?C1 zR|vG;Up?x|=uF6eaX`o2QJC1~vHD%B!>t)j98bUVFWF##RI(a=&VaF-*j?)y`$i*6%R*f?P zte<72D?O*_-g($C&C2hQPnq8L$2u5R_WnSuiic>7&i}zF!locEuMnF9NVZ5h-(Etu zTvsF@QixP22D)9m%CKApEF5yZNa2FCrVhN7e@2wd{v;0L;^r942S}eVgtY@fEc{9w zhU4uqWR94yyMyS77_wQcu$zd(c=$OL`9iETnd2e-LWL~{(jm6+E$&hHv3TgIO^ta1 z>4#3QSp4F>2)~Fd1}KYR&a?$8P5Ro44gmOdWV1zsqKwQIlJBwSSZ^ju2ficz2xZ-< zL0Rt>3xD!&|9#~Wf%Q70Ep1%KUVZ~1EAsQ|!J$zxeSw+8|?K7vAD zh(!f7Mw7i#x9Hb@V-!l@6G(eY9Mr}~`e^Vold|w5AzXCwdG>@x6Qf;hOv}<6Qh38G z85(j@;T5)2M`RMjdSWF(glC2j1hD|EgV8!VGngPkjD(titT2KILui|d)t4qjs zFD^|@e(j~Bts-UKZj7*08L9T>nMzavGTBDx2nthfkPcdP0*7brH*nW2pxPV`<#>nMCSO)WI<_=B7?kfkEjBu z7(_)5L9Qdx^HU9)jKqNWP3dq+gpqDOax!KvMhhbKbKT&(Vy>sjsCws(nN#u-R7h}M zy4F8G5xI({BBOLrpWvX<3tmCd@MV!xqQvm{0F)vcsFH9*Ojjx*2o2Q-#HJJ|S%hX? zDk>3<)#P;_H;)t*w3Sl~N`ZQXXBDR>So)pA5?V_Uea8z7~9NDi%3B2AO$xX6X96-;xNEgnC8@w=4yP_1?6ET~h zZjcd?mV~Ce2Scwx`f%1*yJ|fMp+P7C=nsmhb5NXSShqe!yMO>ZTUFk=lma84{F|?w MMg9f=6`}F}FS94I#sB~S diff --git a/tests/test_data_and_theory.py b/tests/test_data_and_theory.py index cef3dd8..ab91052 100644 --- a/tests/test_data_and_theory.py +++ b/tests/test_data_and_theory.py @@ -40,6 +40,18 @@ def test_unknown_dataset_raises(): md.load_dataset("does_not_exist") +def test_early_january_2023_weather_has_real_measurements(): + # Regression: the R dataset ships temp/dewp/humid/pressure as all-NA; this + # package derives the table from `weather`, so those columns must be numeric + # and fully populated (Newark, first 15 days of January 2023 = 360 hours). + df = md.load_early_january_2023_weather() + assert df.height == 360 + assert df["origin"].unique().to_list() == ["EWR"] + for col in ("temp", "dewp", "humid", "pressure"): + assert df.schema[col].is_numeric() + assert df[col].null_count() == 0 + + def test_t_confidence_interval_matches_manual(): x = np.array([2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]) out = theory.t_confidence_interval(x, level=0.95) diff --git a/tools/build_data.py b/tools/build_data.py index 5df92d2..c53c40a 100644 --- a/tools/build_data.py +++ b/tools/build_data.py @@ -44,7 +44,6 @@ def _write(df: pl.DataFrame, name: str) -> None: def main() -> None: from_r = [ "envoy_flights", - "early_january_2023_weather", "weather", "flights", "airlines", @@ -82,6 +81,15 @@ def main() -> None: for name in from_r: _write(_read_csv(name), name) + # early_january_2023_weather: the R `moderndive` dataset ships with + # temp/dewp/humid/pressure entirely NA, so derive it from `weather` (Newark, + # first 15 days of January 2023) to get the real, correctly-typed values. + weather = _read_csv("weather") + ejw = weather.filter( + (pl.col("origin") == "EWR") & (pl.col("month") == 1) & (pl.col("day") <= 15) + ).sort("time_hour") + _write(ejw, "early_january_2023_weather") + # Gapminder ships a CSV inside the `gapminder` PyPI package. We read that file # directly (the package's Python import is broken on 3.14: it uses the removed # pkg_resources). The data matches the canonical R gapminder dataset.