From 73a08dffdbbcfdccb69153d5fad2e463238c2f5f Mon Sep 17 00:00:00 2001 From: mohammedalfakih-dev Date: Wed, 10 Jun 2026 00:52:47 +0200 Subject: [PATCH] Complete week 6 Azure deployment pipeline --- AI_ASSIST.md | 19 +++- Dockerfile | 8 +- README.md | 52 +++++++---- docs/execution_history.png | Bin 0 -> 21240 bytes docs/execution_history.txt | 3 + requirements.txt | 2 + src/pipeline.py | 181 +++++++++++++++++++++++++++++-------- 7 files changed, 201 insertions(+), 64 deletions(-) create mode 100644 docs/execution_history.png create mode 100644 docs/execution_history.txt diff --git a/AI_ASSIST.md b/AI_ASSIST.md index 9b31b94..28a225f 100644 --- a/AI_ASSIST.md +++ b/AI_ASSIST.md @@ -8,18 +8,31 @@ -TODO: paste your prompt here. +help me complete the Azure deployment for my weather pipeline. I am already done doing the pipeline to upload raw JSON to Azure Blob Storage and insert weather rows into Azure Postgres and both the local Python run and Docker run were working after pushing my Docker image to Azure Container Registry I tried to create and verify an Azure Container App Job but Azure CLI kept failing with this error: + +ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None) + +I ## The code or suggestion it returned ```text -TODO: paste the AI output here. +ChatGPT explained that the error was not caused by my Python pipeline or Docker image. The pipeline had already worked locally, Docker had worked locally, Blob Storage upload had worked, Postgres insert had worked, and the image tag was visible in ACR. The problem was with Azure CLI on my local Windows/Git Bash environment while running az containerapp commands. + +It suggested checking the job list first: + +az containerapp job list \ + --resource-group rg-hyf-data \ + --output table + +When the same Azure CLI connection reset error continued, ChatGPT suggested using Azure Cloud Shell instead of my local terminal, because Cloud Shell runs inside Azure and avoids the local network/CLI connection problem. ``` ## What I changed after reviewing it -TODO: describe your review here. +accepted the suggestion to use Azure Cloud Shell because the problem was not in the pipeline code. Before this, I had already verified that the Python pipeline could upload the blob and write rows to Postgres locally, and I had verified that the Docker image ran correctly. +In Cloud Shell, I created the Container App Job using my own image diff --git a/Dockerfile b/Dockerfile index ee3c978..b43c7cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,10 @@ FROM python:3.11-slim WORKDIR /app # TODO Task 4: copy requirements.txt (must appear before any COPY src command) - +COPY requirements.txt . # TODO Task 4: install dependencies with pip - +RUN pip install --no-cache-dir -r requirements.txt # TODO Task 4: copy the src/ folder - +COPY src/ src/ # TODO Task 4: set the CMD to run the pipeline (python -m src.pipeline) -CMD ["python", "-c", "raise SystemExit('Dockerfile not finished: Task 4 still pending')"] +CMD ["python", "-m", "src.pipeline"] diff --git a/README.md b/README.md index fa94600..39cc229 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,14 @@ data-assignment-week-6/ ## Where to start -| Step | File | Task in the chapter | -|---|---|---| -| 1 | `requirements.txt` | Pin `azure-storage-blob` and `psycopg2-binary` | -| 2 | `src/pipeline.py` | Implement `get_config`, `upload_raw_to_blob`, `write_to_postgres` (Tasks 1-3) | -| 3 | `Dockerfile` | Finish the cache-friendly image (Task 4) | -| 4 | Azure CLI | Deploy as a Container App Job (Task 4-5) | -| 5 | `docs/execution_history.png` | Add the Execution-history portal screenshot (Task 5) | -| 6 | `AI_ASSIST.md` | Fill in your AI prompt + review (Task 7) | +| Step | File | Task in the chapter | +| ---- | ---------------------------- | ----------------------------------------------------------------------------- | +| 1 | `requirements.txt` | Pin `azure-storage-blob` and `psycopg2-binary` | +| 2 | `src/pipeline.py` | Implement `get_config`, `upload_raw_to_blob`, `write_to_postgres` (Tasks 1-3) | +| 3 | `Dockerfile` | Finish the cache-friendly image (Task 4) | +| 4 | Azure CLI | Deploy as a Container App Job (Task 4-5) | +| 5 | `docs/execution_history.png` | Add the Execution-history portal screenshot (Task 5) | +| 6 | `AI_ASSIST.md` | Fill in your AI prompt + review (Task 7) | ## Open in Codespaces @@ -82,17 +82,17 @@ The grader reports a score out of 100. The passing threshold is 60. ## Scoring ladder -| Points | What the grader checks | -|---|---| -| 10 | Required files exist (Dockerfile, requirements.txt, src/pipeline.py, AI_ASSIST.md, docs/) | -| 10 | requirements.txt pins `azure-storage-blob` and `psycopg2-binary` | -| 10 | Dockerfile copies requirements before src (cache-friendly layer order) | -| 15 | Pipeline reads both env vars, wraps the Postgres connection so it is closed cleanly, and silences the Azure SDK logger | -| 15 | Pipeline uses an idempotent upsert (`ON CONFLICT ... DO UPDATE`) | -| 10 | Connection string uses the Azure-required SSL flag and the blob SDK client class | -| 10 | AI_ASSIST.md has all three sections and is filled in (>=1800 chars, no `TODO:`) | -| 10 | README has a `## Verification` heading and references an image in `docs/` | -| 10 | `docs/execution_history.png` exists and is non-trivial (real screenshot) | +| Points | What the grader checks | +| ------ | ---------------------------------------------------------------------------------------------------------------------- | +| 10 | Required files exist (Dockerfile, requirements.txt, src/pipeline.py, AI_ASSIST.md, docs/) | +| 10 | requirements.txt pins `azure-storage-blob` and `psycopg2-binary` | +| 10 | Dockerfile copies requirements before src (cache-friendly layer order) | +| 15 | Pipeline reads both env vars, wraps the Postgres connection so it is closed cleanly, and silences the Azure SDK logger | +| 15 | Pipeline uses an idempotent upsert (`ON CONFLICT ... DO UPDATE`) | +| 10 | Connection string uses the Azure-required SSL flag and the blob SDK client class | +| 10 | AI_ASSIST.md has all three sections and is filled in (>=1800 chars, no `TODO:`) | +| 10 | README has a `## Verification` heading and references an image in `docs/` | +| 10 | `docs/execution_history.png` exists and is non-trivial (real screenshot) | ## Submitting @@ -108,3 +108,17 @@ repo (`Data Track/Week 6/week_6__8_assignment.md`). The auto-grader checks code shape, not live Azure deployment, because the GitHub Actions runner has no Azure credentials. To rebuild from a fresh scaffold, follow `.agents/workflows/build_assignment_repo.md` in the curriculum repo. + +## Verification + +The Container App Job ran successfully. + +Execution history screenshot: + +![Execution history](docs/execution_history.png) + +CLI output is saved in: + +```text +docs/execution_history.txt +``` diff --git a/docs/execution_history.png b/docs/execution_history.png new file mode 100644 index 0000000000000000000000000000000000000000..fdb529f8ad2669cb565869f753d8b47c3d6d2655 GIT binary patch literal 21240 zcmeFZcUV(d*FMbXpd$#3(xi_{5fB(!q{l|@y#z!Fy$YeX*bwPT?*ZvuN(dbl>5xbX zp$4geP$D%D2>gQc%)I0K{QrI5^;${AJnPUlidTabKJ@CzI0cR9C}kw8R_*hS!2^+_?hAGQ!Dkx3;AC6l4GCW zM2mgri*dPkL9eAulWAC;Ng_7EYQmHtq*J@rvxPhAK1olT;Ujj34~y;vQ;FS`mpqzC zLk%F2&a=l_FL`fL=KIr8?0Lsu;HNv3i@PH457_?K&AV1P|Bv0ftL?J3ZsGt+@RRu0 zEE?$bk1tcMa{$tGx2}oPpis_AEu~*q9ucnoYt%Qn2V0VqYrTTaxda-dh+zDiTi3oL zE5xMXBH{EURkAxD1x}wcvipktLRHKN-##=DUi;B^U9v*zZ#K zvls7nYu@D?kbk3$*~*Y|YJ zv=8+VDN1OSsK#H`_?lYu+ z$}d!|*OY|+c?-QLc3w8Zm-4%M?@E;H2vQ<~js8ZU{g7U9zMDCZZH0V5_?0@6-)7OD zXOF<02*vkz!WU7aPxXz>73nyMqZ>v0M=tJ-niGtuK1w6hfx<3! zivPT^9+#oi^PGs(#D}+&PcZRp9>O2W&+(_4%UqRu-g5Zr_xrzHe>JM2!g6($))agL z(h;!JKj?UCD*e)hZ5+nX)jevko-)F6#&rzUXJO88HvGpke|;DPxPLdf&&p_>iMBC_ zw?IQ}dgOybg7%%i_2k>LBr`?uCH|?xpYTxYT>)DbCw^B+QzNbq9IelL?JX}IsyrVC z)4MHIi7g!UI-j5!SuhnXLM`2bnTa(t#J$x)R@_Q{TGicmUoY>uBDRwvy$?>D`li0u z`acwsVA})MsN(xb7xM(Rdj|38=sQckITar%ZpF<)HeQJh*M{1#Ie#&hs%{%9QO=SZs>+Tvk-kGp zJiMeGzvyBFaf!Piu$*L8oAasY`QwLRMUhq9(dY^=hp~IA&M#QKhy+5XKB}pP>k4$K zSfY?CTntV=q4!@}i8y;#zNyj|;R=zHVjk`mFA3_(VgdHncD|Jf zT>F=uy!diU9)xV)k)Pa=E-hGlTi9`rE-^sF8hSu^20gI|OTvtQ*Lv7*7(R*_df(-$ zT67rf?CNS#1|*uFnZREM;`Q$DU&E+hL~8Aom>YOESu0RE&-en4JX(?a&UOG>C+vN5#O)il}n0K;(oHyzdVG-yTU_LZv z0_QcAmS;B6gu*OB;*&dQ5C==dPUh@%lNuL2uEBVwv542cT>bUD8(efwCvSm!56r!c zcMQSlsTL+J)y5&@mw3XC1Ju$Y3qO)o>GmtdUy?b-EN!%`N@(wg2?UOYepe>kcUc+x z?hmuzGPDSyFD_Fq*)OD6#@?Nm+@2yjoFl6{nU1C-M1R^mO}0RkIynXGh?A>S`D9 z0@Vk*+F+AOP~6==-G>ah;9*eyO0?w)de%d$CT5z#_~YK-$1qPU>ISR@^j zgF7AjThvr`w)=foKTX{C;Vpj!vg1 zSN6N#(h{C4RxYJqu&=@FhfS#$q`Pf(X@Fu0y!=w>z}YKpcYeSBrq#6))n_SW6?bjKS#0tFXvDbdgdIeaNoZKJ4>Z^iE_zcm$R%!CnSsZvQk}?uXR^*iwuY$ZT^Htwjn1jU?%$nt_9N6@pbED*Ab{LqO-fIiM_YO zYZbWlAy!w_kRdp9kMmXt2gNby@nl2Sbz;IGnCKX;i&F4RJzl2Km5=DwlUkpwOlIrR zh)Q4bA)8-biVkeph9rT=_r-Hk>_0~?gH*enzN)7QA;GqcMOY4k|MBY|NqwTBW0DLm z&a!)`v9<*of1K%hU9xHGNXz?lrAI3OjJilH*YM<>bo%0MSR0+Qm%d@QLD=tM3c%K4 zEi1LC_HId5%9>|jxA3{DxiD1Azq^!+aZe%t#pAyN+rxriiG@8z4<=|snmW}b{noBQeqvhW?t`ZU^Fa7V?Zc6b4yUhy)0zolOO zc`B;MBhB8AiWmr{M@SLp4cUCdowA7+CMEh-syOCL^bOfIQ_`(EuQ@2o`3DayX)F^s zR(VqWwDL4~aGS{Wb==!oYNH@6fgNA&n)A=%K3h{(?nDjm$ceXRk{mSS=o6o&Acyqs z?+B2VJoZaOFDy`oe)PdD)#|RIc$y5SKu6M7kV!t$L@tbN33SVpzcS$$s=ZgJKD;8d z!RqYC0Ip=npKgcg=t;@qh(lu$?d0z;x4%U|g(;zlf)(ss?3Us%b$$i=bfrR?e~{ua zxVDR+*&3D3zOH&!C(FAc2nZW(5X?zmA+4>VIRCpD)`eC3sIcw24COb`1RO@w*Lz^h zWFXWN?<+q&bI8S`zxYT@pD9bhTy2e<7;I>cyfN8WaIb>vrJS5}6?*#VSLY4b*FF{F z1d1sCIz}5KDRXg<5^ok#-}c#EJ$OVoj&|q5_f7{%RUYPX1s;5NYV|*fcW9*Z9TQL2 z(JTY(+m}(#$_VXzN;VK!MjS&(t~{L@_N{My2(PCM1hKQ@_r(Fla6nhD+s{Yc8BV& zEPkW5re^Q$eDCy*h1{e9#PGHK4R+Tp9T8L-5TI!!v&ri2#_R<^*zHJQ2xn1Y1T-ie$uFqqe1K19cx2OuTGL% z|MF)__`tMkADf#`1gbf746VNzs8CczucZMMSce|!vY|(nrRa8xrFTwU%F5D-0~h~V zGAgQPqkr`J3)ufVLK9gJu9Uf1YR;YSn|@^!&y;0yiX`{lR#Zs&YG%5elF^Xt;2_=3 zo>DbYRaqH(E^p_R^w`(Rj#bsX>v<1Oj(yL6dQ(VB-$}+es`2mrs!b#$Lg1_kLGon=Ej*1A~QEkGdj5buhgrDz+KCSb;);6d10 zZsAP}GsSmD9jE?gBO_6fB#Tk>NZ#0P;FkqYX#jxb*tPc<2m&jVqL+-7^I}8yq|4xW z54b7@OHc}Y%t}MK`{1?U@pFHR*0M-z=+_tgHU>97-90(>evPwPEvTW*cm_#p&Erpe zDmI^FmM*m*nPy6xrr*7z`e3y;nfAxbdORR{?-?8YBIu<;&j9d09d z)m~60b5`Z~Iu~6Uyh?)BGQ%VwD^ug6tyq+ZRl7-@2zC4#emGgoV>bVl3T+GbEXQxx z^P3WVlq$nf8nkQGn>$zS?YurpRqRd7A3*Ejd;`Oacd=`XKL^>jMrSeBbiSn8T)CAo zs|{kdaS|G%9nP#Rr!uV0&uo>AE;(;C>4gHP`_b3b>WeB?z1#pT@^Fsq&X>+3_ueIH zb-~%Q(W6864$0A2ij{p>9mpOoSq%^oRb`IrUTWk_jQ8v`?MTm(E4>Ge+MGB|ba?)` z4botBaxx%Y=;wX~?wz%+H&pKmpsdhS#VHEjWUf#p1z26waSD~LoF0jfd|f0xM!;mn zWsUJ+Bo&g2q!pWsmmDp(T3B_2*sRg>yJn@jJRmUBKi=}$n&pEg8uq`}=kK#~deS@C zKj!U)a@^_NO1IxYbmP)Xk}I{3AECx-6Q+JP} zZtfw6``M_!KK6!GD!z2pf4$)L^)=&veqF`P-}lPV7k)~a@2ed)ega+k2ORzVm*B}p34o5O;jN3`3bgEn5aU4kFQ_k$lpu*K3u&vg^PtW{B0mF_x`A16m`UEXuO1(m&O8<{sp-1o%!v_KmbxaY0bFPx_ z^lq~ZeO5yd+0d7MWma!j`7-u=QYz>1ms<+JvcvDC_w#~UH5W?|#QrWGaIx~9Z{{Pz z=Lr_(bS|O3kLJ*PF~mUD4@JKF{TD(3mOdFyAj(wq7@U`ad@IWTK^Ehd_`(CWTT@v5 zFcX}#_;}vfvdUrRPOuUsl$I61hh*UZF1h@+<$?3CF$=K^G7x=0zqFl?j{AZ6GcsupS>8N)WDc% z`SUEYvxdKXdBxAqFD2bv+__X7uZ|jV)>jb3ICm{X%JH1Xn4!MqkeV#e>m=Iq<4keQ zx+cB;J`O#k4_9Dx#*WS-AN^0Ve!W`FETYA&Pvj;rW;(lqzSRrl41IfHVsojxP5agr z^WlNs>pp-0nBI8y$Kn^cZz%VRu@R{z%R0`*i7K}%RJ*b_pTt~bt-Bf}IAz97jwvWsB>UeVOGb~l}_-YzX^@V+oAT*6G0^EA@464vb zO0cTC>Du&a(Mv525(}|dG0ft%n`7})&?PR9Gi?}I=K@Fym(*I=*mYe-Fq?H5I0v>H z9Zu*ur3ZFv-#W@QcxY3QzXJha96@j7vu;B8m$3NO2*9?INZkH#O!?-QW427@f*EeepINrl^xUr*RBXgRVMBIg`NH`F7^~^ zVqc|eX|saewk;EQN=t9kaAfxzrST4YRtpvHeQozL64tNX+G<_dE3Q%=Hjf)jO2>*B zhXRZeu?hBkJLz4iX15%>!p|2rH&6ZgT6G}5mi^WhiM%zYVmpXPeOSY=Y+K@#UsvjQ z--}DxoOc!$1hk(%;UlUVbzQ5Ka0;ZYyWV@bp_G-8|0+#-!oq@muvgdZ#>h^JEQXVb z!Jso)(8O15yFwq&W=;Za1USQz!yK`jY5Iy1D zoAFkA`C_Ej72J0%d#Q?75n)~09;Ay+&wC5jE5gB=_as}oio>%W2n-GweBrgsyV$8^ z>(mv^z{ImvtS4@Qd#BDADZ{SuWuR}m;ZnIVXSG8Wn~a1LrFyN!oBTAcIO$Pb!h@_s zRP87aVDfum38W`86C^IK3#+{S>!9OfUF>$)@Tjb<4l^+#4o)j?UH$0=WV#;{^6hG_9+*SvCYD@JQsh(Uv$ zTH#L4<99F8G~(s|(00uKcznZ?yZa{fa>^`EtWj`zT>A7_;>~u)?tM>-0H_QW5=%6F zEaiCris~${jlG?_R8V$3ceiIv{c+oErYRVLR#;rsnpevnlpA)Tr1)EkF605czAVyz8b<1d8LUj!==I4WD)we1m^h`+yLFOg9&sc=iQ~@zk*?ic zAt+>A%>QyEzX?hPFxK=kPhIwRpGvQh!rJc#x+}Uyi${dLYpEX{b?LGqYKq)$|6Ph1 z!(e|tEB5`Hb2siu>gCYk02KoIIluR+Kq(5D_Nd#@GmLMbZ&44slgXBDX;Bmt{TXOv zJW`}=sG@I{Yh-b3xmRCqdQ8#iM}5o{OE!%MjgiEt3ZtOtl1Omt;9;mm6=AYV+jn(5 znr$0dC3RkM;>DL?S`Nq6{`(--kQT{jCnRk}$NeAJKcCMOe3_T0-9z|KR zyYV-W!~#E{EKu@BLLuhJQZ=*N;E{ZRZ7YtbV;@&!_S^T$sD^Hwwy~aLx4ZIO6;!+VF76Q7(0T*cm=iS}fx!@Mnf;qRw1^(fls15c*DaK5R zc4;z|M~t+~6t6LCfZsC0H@eA~efF7=Bb&Q%phJisFe(L@26)5>`gXydL)fX*{s2 znwlk`O@5Psf!@cPCZ9EPcSTCT{89nKw--zO74nX#E%Vsf7$r3!_aI)u)6uZY>0wFJ zE0%?itFxMK{V{)yS*l(N%-`KT*!%bvD3RiqtJ6LMk+ulIdYk*SCAe5t^SA;}Vvinx zL~*KV>s&s^YDs3|5P@**oaQO-i?0V8ihw5kK|7@T@kMWhp4`xc@cT=k1q5C~?0aR; zxxAyqpw*H0f_b-GAM=s}V+ZUAS6XeB6?VMX05jS3rPyS6BrGFv(0@~Dn!j4YyzZHV zM+QIFQDD}|+fHIs%(zTfsoYo=^>1rT3f{O)U5^`B-_i&$myP84!yVc#@!@m_m?@fP z)hM*wFJtaK0rBdRcHpc-+F~51mhUGp}Opo3&Hbvx*r3{x~Q>~eU(UkTB(=^W7J7C%2Ta~20LbF$q@fC$HitSMcT zjY4iCSFzEHG^t}&CQEK-S_3%Knw|{ni=NDd0Vlt2^xWT{t{qdFWxVE>Whx}}2gpTH zA`5P zc>sTPI3b)BnScz{;`qBrhht_d~DGlh3cZXlI!1$%%mqNnY+8*`8J_=?`st z2dH)J+q(zChVxjfJZJPNDg-5d80nHe&35xt{J9BvUq}o4$~Ci{gM(2L(or%MN-yKi z=gNu5L&lUZo>Z4&^z|HXNZLE zBLC;(eHMSRmRwdI)?Y=iZiwRX-`dt;WbbdhkTrO`I9P}r@)l22k3p`Aki1zQ@g)!V zd(2HWEsr&RXD~)n{C%P&Q$6xluS?%-b}7OxX5Im_QnD1w3G#yG+pLidd|WxfEU|l6 zx_?Zd*08_C6pT6`{l8rd%m=3Zf@Ag+Zgc7I|CYKn$A8?=K>&2rs~z}kYPxa z1E#5bM1bR+0V$wE$s^7`(rW*5M&o)n#a=EiwIg5W4?uEV4sVci;mCWj3>#lQziYuU z2UYl_?A>n1)*C@L9xrg(=Ea=rAN9{9geUcGzTdfj^AKfhSR{MZvNub!cI@Oq=6!P) zMc%Y^QQd*GxpSy?_G&awhCK-I{aC*bcdyZ$a7~zRseTAB&*mmYiS>z)5KCT)cSs5W z{lQT!f<&#oLc9q&kLaA)Cv~U(fO@%!&DMW~a=}U$O|X{hlj&b0Ep?eSOS(q_aRA~; zbXwTE`Ofi~8R`?g2^9`NTM)(LtF$?Q{&%bSZC}Ej>l_Z%F%WMx97A9wsL>ww?YO#6 z|Mppv>m0DycD~83x#&aJUOK;x{p)FErxNdsX%9%!NwLIZFZjbvfa@*=ne(RQu50j9 zK;#@xDlf&U|DJ<=kFvbySoyY8ukO z8t(J7Y_S=vz+jhikxll*)?u#gt?gv23{sM>93(qsK!(4JfxL|YY$GZPM;32yqr^1& znPE5{>nv5yn-C6gw0i2peb19^7yGQer!fO;Y2vD{yD9j#^NUR-^Pdq92g(Psq#n3% zcB8-OKwaOvO?=$xCc!S++0TH>Um1<75LPnHBRaKQ?Yg4NhgA6-vi3gIctqu=yG>-Y z-Mr`Lhq!b(F15GO($lrNxzlyAKGC9FNb}63Z3G?Ke?a|F3(Hx%81XJ%EnX(6yz>Pi z^(lc06(R1BlU(ObhPFP*tb zVS#qBJ=Eg7AlFH(-}6iYCv40{f}8_?psKKA7*7+ow>szWXdn63+hoXpd3MnGQUfk| z`LBju=}KI=^W<;E-Lt7AhqFGNXaH+ZU}&KJ&~7{ESTR=iCVo|okh;>spDlz{+0$4Z z(a0J)$)a<1eClvJGmU2~)!~uohEUzbF_V+}%YzTSZ|(5ELE`UXLbI?vo4yk(#C{8k zZ3AL`)OEF;lxizPtk~CmP|5kZpG2jLPPwubH`72Bk`{yImSFI{5AW(-Qx`l>8~vWV zZlK7ZhFgHa(9+IoL^3IlcZp{$scctf=GlqbYw59C-ENvt6WM*R$Yc&`ima?-hAZmm z{wagkD zs$}8n3H?3Ep>^-A#b~PRL3)t4&H6Kv9OWV_b^eTP;G)O4Q$`cB=`&dS>4zqjH14Q+ zuJM4^%Iacp48VMOga>9{o@cC@l@^Lk1h@WBq~c(ZinO?V*@SmtS8Qo}nX;1lRwpBV z)xZ3kUwF)Pveyq4VlqVj2U$pmPa%q-Hu^&Yg4#;bM@QZ(MrQn1hCcT`@gaBuyPH1d zG)b`XP+G>;YxfbUrBffaW+*WpzO6Xb^*_`fqsqE@+{*hxT9eV+E?b%|rysAZsF9xv ztM|_dqAcSyyP1&22EDTzX=3V;Ef(jZdYe!(n-S1~VQVnDZk5RWIC02xZEZW0#=>k;#WPZ~#H zaYN}97b8u3n~`F<8tFWx!y_-_m)e7EyN!~iD_()`;U_(&Gu4kOXyQg3bEX7t(VtzIE>vx3hEV$C=_UJ4iS2}E zU4>rM&M!l_oVQxc<9q2!(pm3ISl~(J%UR`@6+OMXE1Vb)9@=cSuTy4BEnG zE7~p_AueS=d=SVPf2{w|t~)h@QZBg#RTP>pC(9w^0l8JaH7PSMIyg3mo z@$T^-cUtdiG%7n4gjh%+-EPnA&tem6kevqTdYLi#`uJumxVW%=jpQj@Lnn>=gh0Of z=S1O{-uG)Yp3xIBW0I@cQo?D6)ohZBnSQp{o2zUj9~`0v3nNc%eShTM=jN|r^C>#t z!(~%JAli2F!vqhSdo8vc%E_#Q?k<6!JY6?MH8!>wc?KuGeNTRiM4z-+wa)@_fPJ#8 zlQI`&uU@00K<3mVa=coq*%`p<9{k>u*^6BX&B7$~O|b}L=&1{nLe9Jcmo35Z76Nvb zXxxFasoVRD)*5~0RS*~+|J-nCU9kQ*kWjHb_JCb_qV>bofgme~3-(?feP2FPu)i}F z;Y$Kbx@n5mqj)-zvrC1$;Bx=7YDPh2_(n+cbth-ZtmjW8m1-B|>~DPf1>*&%?+l_5 zQ*|7ja$lmv5IT;((uwf(Rc?R)&ZGucd70o@Dp4tdkwYv^_HYF)75--#U&pvi{}(6i zt&qWF4=SU_@mB63BYE3F^=6BX`SiiyXb}Lx$HqPKP)1V7B#P9fqNwTWh4#&HbKn_q zwO<_SqTi!ivX9w494X_#%`l-8<>YOL1Mq3 zaU2!{&nM-{r9D`-C`&r)`TliaD$2}`r;C@3LI%=nfWNzcjDrSsr+D|kLZ2^|+IL?- zFL@xDujfmetTD@N<=5Ovr;A&vfDYC1AmLXA;dA{yffE32n0rtCL!SoRv%(Q(TBXzL zeE0WEs6teZ{8(g?6($D7K3G(GVTm9|SsLW7xwo_*?GsJ%lT)*HmD_vaipHEg zCRP=wDSEifW^fjPofsYi(PN|ch8Lc~%R2>80~UYbV|KuwjqlAJez7xcrf>J}Z8jTA zLu;wO=jDSdgcw~!{Nwxgq?9(0*D z`hEH)3O4)oYtFz@;u(B?P4{D=e3q6Z<;L5rz_CEm2K!}ymYTBai7itq=F(z1j@h$H z;Q~9tWb7Y$M-C0gqvK$HP|_c85{50I!b!^NVS-Sic^zrN8bQBj%O2-);uy}Y6l@`u zjUe4H?VkAXeqxpdB>gSTDjq$s5@WDnVeA#8npDx5hb)%Utd@W%x}KwQ~Lfc&~?{UZmT>itNNrsBmt zol=jO3BF6eqEjlYGBKZavJc@Oe}pAz=Rd9um7F9S{bCoaFVHj@70a0+-CT|yXkXZ2 zpE~7Zu4v@MtM4V{+xna62H=W}>$uY~UiQlApwV~i{>{I2tK|!ojrxa}{P0x~mD`7T zHQ<3k%0d~{ahpPBE#f)U$MgO!M_%}mugZDLgwTBe5Cf*l#3g?X{RO!;oov!={Yzr* z-(jT$)4)Og8Ff7Ah=NJOQ=9pGx^q;f6n3}*yD9Y0US95OxTV^XU$kCh>32ir(C5>P zdj0IcCiH7D!OTNtc(vxm*m83=gYh)w3j5G?=Z&vwx%}xDm@<_T$@c)W_V^LH>0|<@ zNswtJRZ}BjbWfkkZRuL=({gz$xkAI8RhVN)=pL8T@KEJ|xr6x*BZs+XU8y6fAOmzP zH=M#=@MoJ$g#&LEEL9gKOvcOMO(7(is7kr03vRpNWN?VIAzg`zbfxUo4{iVwn6a@B zhEOHR5BoLi2q}>H2x0pIf<7w(aDRM*Uj+M; z9~tMQq!i;&N!xC(uw2=Bt|M>P)^i%_a`FY8T3p896#DY5LJq~R1x9|n%-O)DED_^c zZn?6S=06jq-B$!4LyR_oH)YHfpWcdKydwB|u9B08BsX+vrb zxr{zfn5()kY))w)>=YGG20GED$qrnSfTy57n;%Hwq4|Gmiu8K`*3+xM>n^tm*LgE- z#^t8HxqxZ<17eBmo2`1x_fpez=5-8#n`N??Qgr^eX~`9usXXB4yDIfd_wL<47s# zb;fq~Q~ z7f^~2_3M!wU=x1Y>;>A+E>DXPXwQs5HUuJKR}tDH0`XDuAjZD-BDbA2OK#U*7u>O2 zk-Iq9fV!uhrK%x!u_DPG6~tHll-DEey&xFmY9&7|b8*5kFIl4Icg?B~;t@&el6v11 zYfUCI31MI2*$9KcBKjqrCgoW%bY*y;0HoW{6?vH;_(99CN;XaLF(bd!WOBP!QuQzo zrHwhJM(H^jC^58@kgnB^zLu;+Jx##Q^3|Dd1iF4{tMCSmX;BIr^9MN^Ob6xTZ`|}F zFz1jz;+?-??|-t2i1iIXtJ~q)=3w?m_3p!g5UW3Wdm4FYT-Tmrsmdi=7AL^uzcY)UTQ0AR(qHcaY1n^|_9JyDH0hJ|p0^azw_(U&{AYF-N6g$b zqpgAN*E4!CO5w@bE&$}vqj`&L7-rtfNZq>j*vbmuLoc3gdo(gquj-pc zd;880LYF-sWpEby*x9npzP+hQjwad*k5%FzQle-ufM9Rq`(^*ikEiDnjD~G zXE!%v7tOrjY)payVj@<{P9-B@zKlY8wo0-H8XZB$vxf<>jqTdhlA37BGs2XTczqaL z6oT3bbw3!-Dv6``7JmB~#E@0CQgIQoiNBGeZ0D*>UnRMD?mC?@#1${%u|sc{843_( zJ=N}cK-U)Q@()j=dnvJdxU9FQj%4*Rfe-HQ9kuWF{TsZI9M1wnKi^Sx$|4TDU>g zexbf;s8OMjdkBBh8v521iY+90g@!)bsU4wFE$A9k+GE%+6O*6Dv#%M5(BkeBWN$Uf zRjp3g5oOi%mGcHQ=*-N%nR#%0?Rg%%TN7W^Xk1ZIv5k;wcAv?<99L3=dpZZ2GHn#8 zujKj*sR>WxKIfZs#DYGuGQu!h;c>lO=rY z9_qy1fNzg~bWc;7wUg=;>&GZvk~RI_`Z~JkmAk4~Wz$U=CpR6a!`sBmF6Cub*ebfo zXKwmz7)PBS7(pj#oz>ie(l@b$tgc1aN>r|F*O-{5)M79i*5D&@Xg8%QwR&%38Y*{W z7C-9!khV-<^qu)_?y=q7_wUPJ;(uCD#;H-Qck-#C~09}YzOz2c)KKUpH%2paQ%Yar$F z3+yH1*T6QC_QfB)>PHqQ5Y=hdi4txLU9I-lB@J@2N{>1y${FFp`|ZVcoVZK36P#Qf zp%R*mMoszf-ltZTsAHk%%u*Ia@v5-BSB}#L>N4)P!7$z%_D@>AA7aPq$D96BK&i-o zF%yqq;ANAhOYKea!Fszy7dv_=%30;LvodCIh-a-u2U{4IuE!a1WY~gSGl4Yuqo|k1 zOa0$uyIkw$w28gUC`L;C7;7MpL`P67ONE6oaU$00Vvxa z%(c0=5@%t+$+x^&q3b$(oLOEmU(>Zzn6u?dE5}0QOdflSr%*C7QzwuPVYcMcsv}9i zHjsarnbUFmHax&6B)XSUHuVm*c|_?eu|;>p3xSUP7`Kd zg@Mv~AmE$XDRR#D@JMoPA*=aQ=|-G(;UsEG^2o0g{Jpr)zQZD>dubDyKR;wQsB^h* zcDV8-3iIt&7y~-jVmQQA)>SY>jpVF8F-)41Yss7cgrvlWdEx^$iWY zA0B7{CueupEjycGSqiGC+1c4gzv?~Sow4~*Oy?BBiv6&9%;dUYR=Up2J>G0T?OTFu zroqzX6D4`iUb@85-vUeyojDwLO7fCx-<=m%tZThTRxOfh z=G+S=s0n3nPJQ~sh-4ErMm$+)4i5uRNJ91hO8tuO{>8`~{V?=)=z6;WBFfzzGiM<5 zQ17bgte3F+-znnRIj7JHenwlnTkv|xL;F;+sZ^QfXU3Xk(XG%6RNvCqOdjV9%8R@F z;csNsHrk#SWk~HI`!fj+^YVgvm;4nfDogD=nzRSo0+d~IVlDxctws6yT`WF6J|%Ej ztHVso(V^rJ($$+lkTko5++s@phc5O>pYi5@&x`6=`n|K=iWtkLg9-TwLAJ4n&^US} z-KAPLCg4Bm)7cYy3l|8KokmaLTw~w9r53W9GVAhljU+^>rbq|jpEX%b3I?AM(*HT0 z&VUyk{<-?kH$Kn*Kg8V&`%8+?H0WBlb9^<26_GdnA@wHAARV9z^bcGnO zZBQOPSJMVxQ?=OD!RGJ7g7l7j{fm|@gEup?1BT^&xZ;kV$A*XPk#lbF-Dfm;X7N3&;a%hPE z>|m2RVP%l#^nTHK{nnbUz@1RR7xNA&bb_Hu2TJ6@YLHJt{vBkfwSZF6uISG%d$-g5 zyABl~3Z&Gisfts|xl8oj=wVUP9HyJBSztK?2F5AvAV8L=66L9Ouc~uL?mAVcbd!OY zpxp!0P)zeRJr3XL#E`1aM+M)1-oN{pwG%a-sWe)CzT0obgy5aH(S@wQlPt<&LkYix z941|?J$|d^v!{&VZRYP^-f-8(w+i{p#ne5XgQ|_+`v(2V)z-FuF%{`?x ze8a)CY>0b`BnDNCif0P`T1$qt<=<(gt#Cl-y6F;zb0Mqbql2;aaw{In4x>W9)psx9 zr>Z|GDb=$dRJ_lUmFGqwq%ms3)g`AR6v8DOa(^3RaqVz+V*ivhBDOZQ(>~1MAY4)$cZT`B6Cz2y*H1irVC zVPzq1^XrCu%aLhQFG3js{O|x8ysi{7XTC4?lUd~iG6BJc&drzrM{E6qFAPkar`o=K zRge4)Pq!oNLiQvtb$h|rnx7L^t9-LYaK5eiVkh0jj{;_zUxh&B$iFC(o*^>fZ#m#c zTSvP^uw5Oo?wBU>P|$9~OYCZ~adUR7^+oPha5^Z%eNUEKV0JVN z>aC!n-bNnmUmYssvx;4Xa{EPRk|#`eNAe-78H=a2Q<;pz*r6m$Xu)>`v^e6F$v0+` zY#rB*cUrg*@+~iH^6uc_SDqE{%pH0DmI_$tksTS69o3YYwz~jx&L`dK+-b1qe`kbt?#FkPO(YZb6T7qVao96EgK zJzHBrGV^o(^{a3Sv;dL^H2BT~U31fquiJ@`-xc6#GH!NnRtHH?m-Zs|me)M+N2kXS zYM8h21N#jiF=v0HN1>@)vN<%L;Cr0S9WYO?*bt=IR6$V()O@;p^?PVQ?sxk`r8;AV z&|SccAHifMbgH?o6IXny(^hfET6blBzARmKix4TRih-Xs)Nkc4_@R9cT1gAOuzGt0 zj;r;sw%=vHWMn^%@U;WFe>B-THKTGP;DI|(J#c58I%LamQ=%*K$eWH!k^$dAT;jI= z%l|ZfVb3JrR5D0s7+K1YQHe7+!XIUQV*&l9L$jN zg1AsA`LQY?^Fa*w?+e{6)C9#^pq-R0&w!Zf4x zT^0Xoi?MD$9xWy5im-|XC+9;g(fyT8fiPdC`C!Wzn5q>wg4AeB#hvE3 z6a+L48`T^#@p<)*^=S-*64U>El;5g22kII5T*S2 zz`9wJr-!b=V!9r*wD!iD>=lPr_t9+KHd8l5KvBvkgf>rT(}cV< zg^Bq#3vf1DdIggF#Qf^}`kB%=U{JVkz)@aE^K0b#(94M{FYb(6gjje8jVFV$8$b^# zLynS>Tt{s$X1%p0f8#g^4LB0P<;@D2FW+rPtRGone#4kPYF+n#6<7`5X~hTT-)mY} zUiV}P@*!XOUyFXr^f`WN9xD@c54J`w37ty|UHYs{-n%HaGVRjnB=3vUVCM-9MEJ_9 z{@xaBK6~P-2AlHHIs4KD5?^DV4kUYIaEE+0ziO^^AE$ESSDF=9vGTrl42dbOkCWjp z;uAf!5QBv%Tc5g`-E5s5tn9#X(>Tq;MAx)&pZ%r_$u7~hASbAd@pRWszlNy%g!Zxf zg<=%XXbtsMX3g@+mrviV7Qk+#{^4`Hk+n&FFC2IXOWwsW6ddIXltL!~j z#<~VmblXVx$b;RIgcii^q$cuzcB}u;n-6)r^Ek{6xvi2{Oo3KFAErg2 z0CTZpJFWIW#az=zep;x515uypIseR-a*vRCRZ@z_2|#`|NdWquvHLWxEY-cg&EBV6 z_UZ-&RZtua`1npqc+_d3$MH@T*N(^64Tc7xAU<{L=F^LTn}$`Z$9HB;ZZFeHwLE#d%K|p`jBX-%e}kPg#Xo~j-R+v- zQ{1uLRQ#|dL}8xTVdm^-HvNP7cA?(bt?{LZW~(q9lpGVpw9>>Uukk$zgZ-MOd#o09 z9km0+zira@Av|>e(*s63o|MuWT zdU6S5(UWX(x@}pI2WZc`13j#--)tL_2Sc1V^c_RB$S3x83>PWoG_rtHJNvNXBkJAa z`>Qur)&BgJ=C=_-oJ;Yx{}ggr)~yvBezJJLgYO5rqEB9C1ke5qT`SYCM!~99Vra2W zf*z;CvRB>PW(b}vt*sSNw4G5((CtzP?+{1bLYoaW!~-8JowvLhiol*GS+SEa0L8_g<4j%@D@#Z;s$d6x)C?KGqhwf8x@GnLrFZx%Z8=b*lR=a$4%O4(D5-)gGsHCS|8ZB8#Ml zJj^=Wo>f}5PJs9dYfmAlq-mKPo=pESOP7hk0Ymg@S+NE~5{|PXlYRqLc@AD+^v@7P zuY@8SVH^Y>>6G!{uHuRJ6Cz&FZI+ly&mF9FRDWmX ztE-gs4Cehw#0QgQerYA8lCLEv-dhl(C_@J>H*_+>`iaCligIFuDR2(aGNMtNMjAb-w;U1+UBA8As?MO~iIAJ* zO2Dfh`O;-IYVwRHXCyW})wfCn~+RcAD$#Vw$?9unX+`79K6>PZcZsS`n-7-2ILsXshr%N?cws zYHK22%4u0en^e&v=>)-p`*=ZhjD2ORb>yU0NYnMH?buKf5jgtUC04!7jr6@1`YlyY zfF}Cjxa%_o%+KLfYeMa$O08uCrmKy4yp^y_^kuEN_vq;8CQrUQ5wsMcA1IqbFAY|8 zjVOOLkYGd4KWz8iL|M179ByvgMZv(V&=Q`LCFPKMC%e8LST~AkHEFtC+x!!vjUo zo)T{1YRkSRE{_AgQ0XTtVCzGnqU-ccP~J>`YTM-gCsezT1G(0BeRRlSBSx%sZW4jl zh55u$-tIVF=hv16l8Ga+d8xQ0SyW)p3b(NKrQd-bpDT}W(QHx+>`%^`^!py|SDg;O zdDJcJu?in(JmN8(3ETMj1&o|1{XVf*p-X`+3l9j!s?9F(ysOCq`zuNg)Qd~~axm_K z@klPhur{*0{WxYvzc1W<l(HRQfb1F z-I{8|@CcFaJDFKQ);6&my2|+_%G+TAYF#!Y{(~?zs7P~BtC9#w%6KnML1izjzdfte zo}=58oQsxv8n(wn5>xstc)-@P*jxsM-o5PAwQ0MDx2&P?lCEr>w1c#QXPaY0VpS`0 zLEt}e4>YDueEvL|UE!i}A*fIKNXX8bUD_Z%1GyE4M7+&#y*LDdr*Pe;tf6WpBt`hc zd-4bHeG=r}@iHlm6Y1@S?KYtkKBmWU(O|*hZms#HExz{c5r!gGh)Nhj%K2-T+&B5{ zKXL+ZC{3+5Rie2C2N7jQHWuMm@QPPM7%TjyK3v5ei+46An3q-bmT85M#Dm|r{?;lN z*ejlbI|i~FtV4I#WcDE$mb}1l?`{pkQ`#7{b1mQ7ei$5^~ z9O7T5pP;B2d1_!rjR^+bD+6T4^R?uv1pa8p0ab;|_9_JlwY1GQDu&i58jpJ)7@Yi| zBHr*903t?qxQ2#YoeT1Q2gt_DRJlAD-v9#7O;1V9MaoCO%9@3&=4~wq@VaXkk%=8x zG4V^Mm$S4LuZLghLNSXj4eY&Cw}&Zv`hv*JK_c?qX16nvBN)bl@;rAU>0lx6@ffis zLeCtMQ7fs$Uktd^VZJqSV&3v79>cVytMmG2)%;48eIy&q^^`Y~*^c@dKH~go%FG+( zaMeAi4PjSa98H6^+wXXL2vIvWW_~N5LMP- zlD~PH8mNQMR8xAZ#+Umnrr)_g`E+T>g6u(pb1)XtT&Z3JWII@zwXG?l6wFx8%=b|8 zQy9}8f5p=-YOi2>*AvLLSdTowKjhobfs_csfzfwyBrS)|h?$w3S&#;8`ySo91s$#q zs;oJLnxx*Sq*8hTD|bN0bw-10M=&pLZ|@L<(XP?$&4sY>Q**8!7ZSl}V+PUN(XSO{ zm?@C!O56oZf3@rhcy!64v0D#=4oQhPo#3`rL2w)KZ}lZrjbymFAV%){V#~s&i_YeJ z=a;$eJ=zO*Sq3?}sostEvkd2BtjFY03zAMVGSE!ElaAwAbLLTlP;XI5^uQ#b=NBWx zva9@leW9=I=_~eXQn|*%yXO0-?KfwlAiwgNRF#wNWKiSimgg8s{h-}1f6pSLZV*hJ zg?Q*q@!P#yp>8-hoJ(MDa)G4E{FwSAP6Rn85nQ1!zG_Z~cpC!uTsYtmk#kV6eiE{* zIzE9}lO2s7omD}@F1RoB%3Ssz)zbgc&<_?h(ao41?2Cc1vC4o+X!zsUgj64O`CkoC Z$|ts;yY9`l{gXFGjZ6GNw)&^Te*@ERGByAJ literal 0 HcmV?d00001 diff --git a/docs/execution_history.txt b/docs/execution_history.txt new file mode 100644 index 0000000..9fc8fef --- /dev/null +++ b/docs/execution_history.txt @@ -0,0 +1,3 @@ +Name StartTime Status +--------------------------------------- ------------------------- --------- +mohammedalfakih-dev-weather-job-lq9lmww 2026-06-09T22:20:30+00:00 Succeeded \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b5d18d7..3c2c3a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,5 @@ # TODO: pin psycopg2-binary (uncomment and add a version) # psycopg2-binary== +azure-storage-blob==12.24.0 +psycopg2-binary==2.9.10 diff --git a/src/pipeline.py b/src/pipeline.py index ef1fc8a..7e3fcae 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -13,78 +13,183 @@ - Container Job: Data Track/Week 6/week_6__5_container_apps_jobs.md """ +import json import logging import os -from datetime import date +from contextlib import closing +from datetime import date, datetime, timezone +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg2 +from azure.storage.blob import BlobServiceClient +from psycopg2.extras import execute_values + + +logging.basicConfig( + level=os.getenv("LOG_LEVEL", "INFO"), + format="%(levelname)s %(message)s", +) + +# Required by the assignment: silence noisy Azure SDK logs. +logging.getLogger("azure").setLevel(logging.WARNING) -logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") logger = logging.getLogger(__name__) +SCHEMA_NAME = "dev_mohammedalfakih" +TABLE_NAME = "weather_readings" # TASK 3 hint: quiet the Azure SDK so its DEBUG output does not drown your own # pipeline logs. The right call lives in Chapter 5 (Viewing logs). +def ensure_sslmode_require(postgres_url: str) -> str: + """Ensure Azure Postgres connection string uses sslmode=require.""" + parts = urlsplit(postgres_url) + query = dict(parse_qsl(parts.query)) + + if "sslmode" not in query: + query["sslmode"] = "require" + + return urlunsplit( + ( + parts.scheme, + parts.netloc, + parts.path, + urlencode(query), + parts.fragment, + ) + ) + def get_config() -> dict: """Return configuration read from environment variables. Required: - - POSTGRES_URL: full Azure Postgres connection string. - - AZURE_STORAGE_CONNECTION_STRING: blob storage account connection string. + POSTGRES_URL + AZURE_STORAGE_CONNECTION_STRING Optional: - - SOURCE_NAME: logical source label, default "weather". - - LOG_LEVEL: not parsed here; the orchestrator sets it via env var. - - Raise RuntimeError with a clear message if a required variable is missing. + SOURCE_NAME, default "weather" """ - raise NotImplementedError( - "Task 3: read POSTGRES_URL and AZURE_STORAGE_CONNECTION_STRING from os.environ" - ) + postgres_url = os.getenv("POSTGRES_URL") + blob_conn_str = os.getenv("AZURE_STORAGE_CONNECTION_STRING") + missing = [] + if not postgres_url: + missing.append("POSTGRES_URL") + if not blob_conn_str: + missing.append("AZURE_STORAGE_CONNECTION_STRING") -def fetch_records() -> list[dict]: - """Return a small batch of records to ingest. + if missing: + raise RuntimeError( + f"Missing required environment variables: {', '.join(missing)}" + ) - In a real pipeline you would call an API here. Return a list of at least - one dict with a stable key set (for example: station, timestamp, - temperature_c, humidity_pct). - """ - raise NotImplementedError("Task 3: return a list of at least one record") + return { + "postgres_url": ensure_sslmode_require(postgres_url), + "azure_storage_connection_string": blob_conn_str, + "source_name": os.getenv("SOURCE_NAME", "weather"), + } + + +def fetch_records() -> list[dict]: + """Return a small batch of weather records to ingest.""" + now = datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + return [ + { + "station": "Open-Meteo Copenhagen", + "timestamp": now, + "temperature_c": 12.5, + "humidity_pct": 80, + }, + { + "station": "Open-Meteo Amsterdam", + "timestamp": now, + "temperature_c": 10.8, + "humidity_pct": 75, + }, + ] def upload_raw_to_blob(records: list[dict], blob_conn_str: str, source: str) -> str: - """Upload the raw records as a single JSON blob and return its name. + """Upload raw records as JSON to Azure Blob Storage.""" + blob_name = f"raw/{source}/{date.today().isoformat()}.json" - The blob name must follow the assignment convention: - raw//.json + service = BlobServiceClient.from_connection_string(blob_conn_str) + blob_client = service.get_blob_client( + container="raw", + blob=blob_name, + ) - Use the azure-storage-blob SDK. The target container is "raw" (your - teacher has pre-created it). Overwrite if the blob already exists so - same-day reruns succeed. - """ - raise NotImplementedError("Task 1 + Task 3: upload records to blob storage") + payload = json.dumps(records, indent=2) + blob_client.upload_blob( + payload, + overwrite=True, + content_type="application/json", + ) -def write_to_postgres(records: list[dict], postgres_url: str) -> int: - """Insert (or upsert) records into Azure Postgres. Return the row count. + return blob_name - Steps: - 1. Open a psycopg2 connection wrapped so it is closed cleanly when the - function returns, even on error. - 2. Ensure the target table exists (create it if missing). - 3. Insert each record. The pipeline must be safe to rerun on the same - day: a re-run must update rather than fail. - 4. Commit and return the number of rows written. - See Chapter 4 for the connection-and-cursor pattern this is based on. - """ - raise NotImplementedError("Task 2 + Task 3: insert rows into Azure Postgres") + +def write_to_postgres(records: list[dict], postgres_url: str) -> int: + """Upsert weather readings into Azure Postgres and return row count.""" + with closing(psycopg2.connect(postgres_url)) as conn: + with conn: + with conn.cursor() as cur: + cur.execute(f"CREATE SCHEMA IF NOT EXISTS {SCHEMA_NAME}") + cur.execute(f"SET search_path TO {SCHEMA_NAME}") + + cur.execute( + f""" + CREATE TABLE IF NOT EXISTS {TABLE_NAME} ( + station TEXT NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + temperature_c DOUBLE PRECISION NOT NULL, + humidity_pct INTEGER NOT NULL, + ingested_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (station, timestamp) + ) + """ + ) + + rows = [ + ( + record["station"], + record["timestamp"], + record["temperature_c"], + record["humidity_pct"], + ) + for record in records + ] + + execute_values( + cur, + f""" + INSERT INTO {TABLE_NAME} ( + station, + timestamp, + temperature_c, + humidity_pct + ) + VALUES %s + ON CONFLICT (station, timestamp) + DO UPDATE SET + temperature_c = EXCLUDED.temperature_c, + humidity_pct = EXCLUDED.humidity_pct, + ingested_at = NOW() + """, + rows, + ) + + return len(records) def run() -> None: config = get_config() logger.info("starting pipeline") records = fetch_records() + logger.info("fetched %d records", len(records)) blob_name = upload_raw_to_blob( records,