feat: scaffold Week 4 assignment — MessyCorp Pandas pipeline

.gitignore

@@ -1,3 +1,13 @@
+data/
+output/
+__pycache__/
+*.pyc
+*.pyo
+.venv/
+venv/
+.pytest_cache/
+*.egg-info/
+
 # System files
 .DS_Store
 Thumbs.db

.hyf/test.sh

@@ -1,13 +1,108 @@
 #!/usr/bin/env bash
+# Week 4 autograder — MessyCorp Pandas pipeline
+# Runs from the .hyf/ directory; the project root is one level up.
 set -euo pipefail
 
-# Run your test scripts here.
-# Auto grade tool will execute this file within the .hyf working directory.
-# The result should be stored in score.json file with the format shown below.
+ROOT="$(cd .. && pwd)"
+SCORE=0
+PASS=false
+PASSING_SCORE=60
+
+add() { SCORE=$((SCORE + $1)); }
+
+# Helper: grep code lines only (skip blank lines, comment-only lines, TODO/FIXME lines)
+code_grep() {
+  local pattern="$1"; shift
+  grep -v '^\s*#' "$@" | grep -v 'TODO\|FIXME\|raise NotImplementedError' | grep -q "$pattern"
+}
+
+# ── Level 1 (10 pts): data exploration ───────────────────────────────────────
+CLEAN="$ROOT/src/clean.py"
+L1=0
+code_grep "\.info()"        "$CLEAN" && L1=$((L1+1)) || true
+code_grep "\.describe()"    "$CLEAN" && L1=$((L1+1)) || true
+code_grep "\.isna()\.sum()" "$CLEAN" && L1=$((L1+1)) || true
+code_grep "\.head("         "$CLEAN" && L1=$((L1+1)) || true
+[ "$L1" -eq 4 ] && add 10
+
+# ── Level 2 (20 pts): vectorized cleaning ────────────────────────────────────
+code_grep "str\.strip"                      "$CLEAN" && add 2 || true
+code_grep "str\.title"                      "$CLEAN" && add 1 || true
+code_grep "str\.lower"                      "$CLEAN" && add 1 || true
+code_grep "pd\.to_numeric"                  "$CLEAN" && add 3 || true
+code_grep "pd\.to_datetime"                 "$CLEAN" && add 3 || true
+# Targeted boolean row-drops (price/quantity filters, not bare dropna)
+code_grep "\[.*price\b"                     "$CLEAN" && add 2 || true
+code_grep "\[.*quantity\b"                  "$CLEAN" && add 2 || true
+# Deduplication on transaction_id with keep="first"
+code_grep "drop_duplicates"                 "$CLEAN" && \
+  code_grep "transaction_id"               "$CLEAN" && \
+  code_grep "keep.*=.*['\"]first['\"]"     "$CLEAN" && add 6 || true
+
+# ── Level 3 (15 pts): customer join ──────────────────────────────────────────
+TRANSFORM="$ROOT/src/transform.py"
+code_grep "str\.lower"                      "$TRANSFORM" && add 2 || true
+code_grep "str\.strip"                      "$TRANSFORM" && add 2 || true
+code_grep "how.*=.*['\"]inner['\"]"         "$TRANSFORM" && add 5 || true
+# Vectorized is_high_value — boolean expression, no row-level loop
+code_grep "is_high_value"                   "$TRANSFORM" && \
+  ! grep -q "iterrows\|for.*row\b"          "$TRANSFORM" && add 6 || true
+
+# ── Level 4 (20 pts): named aggregations ─────────────────────────────────────
+REPORT="$ROOT/src/report.py"
+# Named agg: keyword=("col", "func") style
+code_grep "total_revenue[[:space:]]*="      "$REPORT" && add 5 || true
+code_grep "order_count[[:space:]]*="        "$REPORT" && add 5 || true
+code_grep "isocalendar"                     "$REPORT" && \
+  code_grep "\.week"                        "$REPORT" && add 5 || true
+# ("customer_name", "first") pattern
+code_grep "customer_name.*first\|\"first\"" "$REPORT" && add 5 || true
+
+# ── Level 5 (10 pts): file outputs ───────────────────────────────────────────
+code_grep "weekly_revenue\.csv"             "$REPORT" && add 2 || true
+code_grep "customer_summary\.parquet"       "$REPORT" && add 3 || true
+code_grep "category_performance\.csv"       "$REPORT" && add 2 || true
+# index=False on writes
+code_grep "index=False"                     "$REPORT" && add 1 || true
+code_grep "savefig"                         "$REPORT" && add 2 || true
+
+# ── Level 6 (15 pts): Azure round-trip ───────────────────────────────────────
+INGEST="$ROOT/src/ingest.py"
+code_grep "DefaultAzureCredential"          "$INGEST" && add 3 || true
+code_grep "BlobServiceClient"               "$INGEST" && add 2 || true
+# data/ must be in .gitignore (exact path entry)
+grep -q "^data/" "$ROOT/.gitignore"                   && add 5 || true
+# Read-back assertion with row count comparison
+code_grep "assert"                          "$INGEST" && \
+  code_grep "len("                          "$INGEST" && add 5 || true
+
+# ── Level 7 (10 pts): code quality ───────────────────────────────────────────
+# pathlib.Path constructor used in src/ (not just type hints)
+grep -rq "Path(" "$ROOT/src/"                         && add 3 || true
+# logging.X() calls present, no bare print() calls
+grep -rq "logging\.\(info\|warning\|error\|debug\)" "$ROOT/src/" && add 3 || true
+! grep -rq "^[[:space:]]*print(" "$ROOT/src/"         && add 0 || true  # advisory only
+# All five required function names present
+grep -q "def download_inputs"  "$INGEST"  && \
+  grep -q "def upload_outputs" "$INGEST"  && \
+  grep -q "def clean_sales"    "$CLEAN"   && \
+  grep -q "def join_customers" "$TRANSFORM" && \
+  grep -q "def build_reports"  "$REPORT"  && add 4 || true
+
+# ── Level 8: AI_ASSIST.md (qualitative) ──────────────────────────────────────
+AI_ASSIST_EXISTS=false
+if [ -f "$ROOT/AI_ASSIST.md" ] && [ "$(wc -l < "$ROOT/AI_ASSIST.md")" -gt 5 ]; then
+  AI_ASSIST_EXISTS=true
+fi
+
+# ── Result ────────────────────────────────────────────────────────────────────
+[ "$SCORE" -ge "$PASSING_SCORE" ] && PASS=true || true
+
 cat << EOF > score.json
 {
-  "score": 0,
-  "pass": true,
-  "passingScore": 0
+  "score": $SCORE,
+  "pass": $PASS,
+  "passingScore": $PASSING_SCORE,
+  "ai_assist_present": $AI_ASSIST_EXISTS
 }
 EOF

AI_ASSIST.md

@@ -0,0 +1,19 @@
+# AI Assist Report
+
+## The prompt I gave
+
+<!-- Paste the exact prompt you gave the LLM here. -->
+
+## The code it suggested
+
+```python
+# Paste the relevant code the LLM suggested here.
+```
+
+## What I changed and why
+
+<!-- Describe what you kept, what you modified, and what you threw away. -->
+
+## Did it work?
+
+<!-- Yes / partially / no — and what you learned from the interaction. -->

README.md

@@ -1,17 +1,42 @@
-# [Track] week X assignment
-HackYourFuture <Track> week X assignment
-The Week X assignment for the HackYourFuture <TRACK> can be found at the following link: [TODO: Assignment url in the learning platform]
+# Week 4 Assignment: MessyCorp Pandas
 
+Read the full assignment on the HYF Data Track: [Assignment: MessyCorp Pandas](https://hub.hackyourfuture.nl/)
 
-## Implementation Instructions
+## Where to start
 
-Provide clear instructions on how trainees should implement the tasks.
+| File | Task |
+|---|---|
+| `src/ingest.py` | Task 1 (download inputs) and Task 7 (upload results) |
+| `src/clean.py` | Task 2 (explore) and Task 3 (clean sales) |
+| `src/transform.py` | Task 4 (join customers, add `is_high_value`) |
+| `src/report.py` | Task 5 (build report tables) and Task 6 (write outputs) |
+| `main.py` | Pipeline runner — set `GITHUB_USERNAME` before running Task 7 |
+| `AI_ASSIST.md` | Task 8 — fill in before submitting |
 
-### Task 1
-Instructions for Task 1
+## Setup
 
-### Task 2
-Instructions for Task 2
+```bash
+pip install pandas azure-identity azure-storage-blob matplotlib pyarrow
+```
 
-...
+Log in to Azure (reuses your Week 2 session):
 
+```bash
+az login
+```
+
+## Running the pipeline
+
+```bash
+python main.py
+```
+
+This downloads inputs from Azure, cleans and transforms them, writes reports to `output/`, and uploads results back to Azure.
+
+`data/` and `output/` are excluded from git — they are generated at runtime.
+
+## Submitting
+
+1. Create a branch `week4/your-name`.
+2. Commit your work.
+3. Push and open a Pull Request.

main.py

@@ -0,0 +1,36 @@
+"""MessyCorp Pandas pipeline — Task 1 through 7 runner."""
+import logging
+from pathlib import Path
+
+from src.ingest import download_inputs, upload_outputs
+from src.clean import load_and_explore, clean_sales
+from src.transform import join_customers
+from src.report import build_reports, write_outputs
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+
+DATA_DIR = Path("data")
+OUTPUT_DIR = Path("output")
+
+# TODO (Task 7): replace with your GitHub username before running the pipeline.
+GITHUB_USERNAME = "<your-github-username>"
+
+
+def run() -> None:
+    download_inputs(DATA_DIR)
+
+    sales_raw, customers_raw = load_and_explore(DATA_DIR)
+
+    sales_clean = clean_sales(sales_raw)
+    enriched = join_customers(sales_clean, customers_raw)
+
+    reports = build_reports(enriched)
+    write_outputs(reports, OUTPUT_DIR)
+
+    upload_outputs(OUTPUT_DIR, GITHUB_USERNAME)
+
+    logging.info("Pipeline complete.")
+
+
+if __name__ == "__main__":
+    run()

src/clean.py

@@ -0,0 +1,28 @@
+"""Tasks 2 and 3: Explore and clean the raw DataFrames."""
+import logging
+from pathlib import Path
+
+import pandas as pd
+
+
+def load_and_explore(data_dir: Path) -> tuple[pd.DataFrame, pd.DataFrame]:
+    """Task 2: Load both CSV files and explore their contents before cleaning."""
+    # TODO: Read messy_sales.csv and messy_customers.csv with pd.read_csv().
+    # TODO: For each DataFrame call .info(), .describe(), .head(20), and .isna().sum().
+    # TODO: Log what you discover (e.g. which columns have nulls, any suspicious values).
+    raise NotImplementedError("Task 2: implement load_and_explore")
+
+
+def clean_sales(sales: pd.DataFrame) -> pd.DataFrame:
+    """Task 3: Clean the sales DataFrame using vectorized Pandas operations."""
+    # TODO: Normalize product_name with .str.strip().str.title().
+    # TODO: Normalize customer_email with .str.lower().str.strip().
+    # TODO: Convert price to numeric with pd.to_numeric(errors="coerce").
+    # TODO: Parse date with pd.to_datetime(errors="coerce").
+    # TODO: Drop rows where product_name is missing.
+    # TODO: Drop rows where price is negative.
+    # TODO: Drop rows where quantity is zero.
+    # TODO: Drop rows where date is NaT (invalid after parsing).
+    # TODO: Remove duplicate transactions: .drop_duplicates(subset="transaction_id", keep="first").
+    # TODO: Decide what to do with outlier prices (clip, flag, or leave) and add a comment explaining why.
+    raise NotImplementedError("Task 3: implement clean_sales")

src/ingest.py

@@ -0,0 +1,33 @@
+"""Task 1: Download inputs from Azure. Task 7: Upload outputs back to Azure."""
+import io
+import logging
+from pathlib import Path
+
+import pandas as pd
+from azure.identity import DefaultAzureCredential
+from azure.storage.blob import BlobServiceClient
+
+ACCOUNT_URL = "https://sthyfstudentsdemo.blob.core.windows.net"
+SOURCE_CONTAINER = "week4-inputs"
+FILES = ["messy_sales.csv", "messy_customers.csv"]
+
+
+def download_inputs(data_dir: Path) -> None:
+    """Task 1: Download input CSV files from Azure Blob Storage."""
+    # TODO: Create a BlobServiceClient using DefaultAzureCredential and ACCOUNT_URL.
+    # TODO: Get a container client for SOURCE_CONTAINER.
+    # TODO: For each filename in FILES, download the blob and write it to data_dir/<filename>.
+    # TODO: Log a message for each downloaded file.
+    raise NotImplementedError("Task 1: implement download_inputs")
+
+
+def upload_outputs(output_dir: Path, github_username: str) -> None:
+    """Task 7: Upload Parquet outputs to a personal Azure container and verify the round-trip."""
+    container_name = f"week4-{github_username}"
+
+    # TODO: Create a BlobServiceClient using DefaultAzureCredential and ACCOUNT_URL.
+    # TODO: Get (or create) the container named container_name.
+    # TODO: Upload every .parquet file in output_dir to the container.
+    # TODO: Download customer_summary.parquet back and assert its row count matches the local file.
+    # TODO: Log the container name and number of files uploaded.
+    raise NotImplementedError("Task 7: implement upload_outputs")

src/report.py

@@ -0,0 +1,31 @@
+"""Tasks 5 and 6: Build report tables and write outputs."""
+import logging
+from pathlib import Path
+
+import pandas as pd
+
+
+def build_reports(enriched: pd.DataFrame) -> dict[str, pd.DataFrame]:
+    """Task 5: Build four summary tables using groupby and named aggregations."""
+    # TODO: Add a week column using .dt.isocalendar().week.
+    # TODO: Build weekly_revenue: group by week and region, columns week/region/total_revenue/order_count.
+    # TODO: Build customer_summary: group by customer_email, columns customer_email/customer_name/
+    #       region/loyalty_tier/total_spent/avg_order/order_count.
+    #       Use ("customer_name", "first") to pick the constant-per-group string columns.
+    # TODO: Build category_performance: group by category, columns category/total_revenue/order_count.
+    # TODO: Build loyalty_analysis: group by loyalty_tier, columns loyalty_tier/avg_spent/customer_count.
+    raise NotImplementedError("Task 5: implement build_reports")
+
+
+def write_outputs(reports: dict[str, pd.DataFrame], output_dir: Path) -> None:
+    """Task 6: Write report tables to CSV/Parquet and save a bar chart."""
+    output_dir.mkdir(exist_ok=True)
+
+    # TODO: Write reports["weekly_revenue"] to weekly_revenue.csv with index=False.
+    # TODO: Write reports["customer_summary"] to customer_summary.parquet with index=False.
+    # TODO: Write reports["category_performance"] to category_performance.csv with index=False.
+    # TODO: Sort category_performance by total_revenue descending.
+    # TODO: Plot a bar chart (x="category", y="total_revenue") and save to category_revenue.png
+    #       using plt.savefig(output_dir / "category_revenue.png", bbox_inches="tight").
+    #       Use matplotlib.use("Agg") before importing pyplot for headless environments.
+    raise NotImplementedError("Task 6: implement write_outputs")

src/transform.py

@@ -0,0 +1,13 @@
+"""Task 4: Join customer data and add derived columns."""
+import logging
+
+import pandas as pd
+
+
+def join_customers(sales: pd.DataFrame, customers: pd.DataFrame) -> pd.DataFrame:
+    """Task 4: Normalize join keys, merge, and add a derived boolean flag."""
+    # TODO: Normalize customer_email in both DataFrames with .str.lower().str.strip().
+    # TODO: Merge sales with customers on customer_email using an inner join.
+    # TODO: Add a vectorized boolean column is_high_value: True where price * quantity >= 150.
+    # TODO: (Optional hands-on) Try a left join instead and inspect rows where customer_name is NaN.
+    raise NotImplementedError("Task 4: implement join_customers")