diff --git a/.gitignore b/.gitignore index 2b76d7c..11d7f18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,13 @@ +data/ +output/ +__pycache__/ +*.pyc +*.pyo +.venv/ +venv/ +.pytest_cache/ +*.egg-info/ + # System files .DS_Store Thumbs.db diff --git a/.hyf/test.sh b/.hyf/test.sh index ee037fc..1665e23 100644 --- a/.hyf/test.sh +++ b/.hyf/test.sh @@ -1,13 +1,108 @@ #!/usr/bin/env bash +# Week 4 autograder — MessyCorp Pandas pipeline +# Runs from the .hyf/ directory; the project root is one level up. set -euo pipefail -# Run your test scripts here. -# Auto grade tool will execute this file within the .hyf working directory. -# The result should be stored in score.json file with the format shown below. +ROOT="$(cd .. && pwd)" +SCORE=0 +PASS=false +PASSING_SCORE=60 + +add() { SCORE=$((SCORE + $1)); } + +# Helper: grep code lines only (skip blank lines, comment-only lines, TODO/FIXME lines) +code_grep() { + local pattern="$1"; shift + grep -v '^\s*#' "$@" | grep -v 'TODO\|FIXME\|raise NotImplementedError' | grep -q "$pattern" +} + +# ── Level 1 (10 pts): data exploration ─────────────────────────────────────── +CLEAN="$ROOT/src/clean.py" +L1=0 +code_grep "\.info()" "$CLEAN" && L1=$((L1+1)) || true +code_grep "\.describe()" "$CLEAN" && L1=$((L1+1)) || true +code_grep "\.isna()\.sum()" "$CLEAN" && L1=$((L1+1)) || true +code_grep "\.head(" "$CLEAN" && L1=$((L1+1)) || true +[ "$L1" -eq 4 ] && add 10 + +# ── Level 2 (20 pts): vectorized cleaning ──────────────────────────────────── +code_grep "str\.strip" "$CLEAN" && add 2 || true +code_grep "str\.title" "$CLEAN" && add 1 || true +code_grep "str\.lower" "$CLEAN" && add 1 || true +code_grep "pd\.to_numeric" "$CLEAN" && add 3 || true +code_grep "pd\.to_datetime" "$CLEAN" && add 3 || true +# Targeted boolean row-drops (price/quantity filters, not bare dropna) +code_grep "\[.*price\b" "$CLEAN" && add 2 || true +code_grep "\[.*quantity\b" "$CLEAN" && add 2 || true +# Deduplication on transaction_id with keep="first" +code_grep "drop_duplicates" "$CLEAN" && \ + code_grep "transaction_id" "$CLEAN" && \ + code_grep "keep.*=.*['\"]first['\"]" "$CLEAN" && add 6 || true + +# ── Level 3 (15 pts): customer join ────────────────────────────────────────── +TRANSFORM="$ROOT/src/transform.py" +code_grep "str\.lower" "$TRANSFORM" && add 2 || true +code_grep "str\.strip" "$TRANSFORM" && add 2 || true +code_grep "how.*=.*['\"]inner['\"]" "$TRANSFORM" && add 5 || true +# Vectorized is_high_value — boolean expression, no row-level loop +code_grep "is_high_value" "$TRANSFORM" && \ + ! grep -q "iterrows\|for.*row\b" "$TRANSFORM" && add 6 || true + +# ── Level 4 (20 pts): named aggregations ───────────────────────────────────── +REPORT="$ROOT/src/report.py" +# Named agg: keyword=("col", "func") style +code_grep "total_revenue[[:space:]]*=" "$REPORT" && add 5 || true +code_grep "order_count[[:space:]]*=" "$REPORT" && add 5 || true +code_grep "isocalendar" "$REPORT" && \ + code_grep "\.week" "$REPORT" && add 5 || true +# ("customer_name", "first") pattern +code_grep "customer_name.*first\|\"first\"" "$REPORT" && add 5 || true + +# ── Level 5 (10 pts): file outputs ─────────────────────────────────────────── +code_grep "weekly_revenue\.csv" "$REPORT" && add 2 || true +code_grep "customer_summary\.parquet" "$REPORT" && add 3 || true +code_grep "category_performance\.csv" "$REPORT" && add 2 || true +# index=False on writes +code_grep "index=False" "$REPORT" && add 1 || true +code_grep "savefig" "$REPORT" && add 2 || true + +# ── Level 6 (15 pts): Azure round-trip ─────────────────────────────────────── +INGEST="$ROOT/src/ingest.py" +code_grep "DefaultAzureCredential" "$INGEST" && add 3 || true +code_grep "BlobServiceClient" "$INGEST" && add 2 || true +# data/ must be in .gitignore (exact path entry) +grep -q "^data/" "$ROOT/.gitignore" && add 5 || true +# Read-back assertion with row count comparison +code_grep "assert" "$INGEST" && \ + code_grep "len(" "$INGEST" && add 5 || true + +# ── Level 7 (10 pts): code quality ─────────────────────────────────────────── +# pathlib.Path constructor used in src/ (not just type hints) +grep -rq "Path(" "$ROOT/src/" && add 3 || true +# logging.X() calls present, no bare print() calls +grep -rq "logging\.\(info\|warning\|error\|debug\)" "$ROOT/src/" && add 3 || true +! grep -rq "^[[:space:]]*print(" "$ROOT/src/" && add 0 || true # advisory only +# All five required function names present +grep -q "def download_inputs" "$INGEST" && \ + grep -q "def upload_outputs" "$INGEST" && \ + grep -q "def clean_sales" "$CLEAN" && \ + grep -q "def join_customers" "$TRANSFORM" && \ + grep -q "def build_reports" "$REPORT" && add 4 || true + +# ── Level 8: AI_ASSIST.md (qualitative) ────────────────────────────────────── +AI_ASSIST_EXISTS=false +if [ -f "$ROOT/AI_ASSIST.md" ] && [ "$(wc -l < "$ROOT/AI_ASSIST.md")" -gt 5 ]; then + AI_ASSIST_EXISTS=true +fi + +# ── Result ──────────────────────────────────────────────────────────────────── +[ "$SCORE" -ge "$PASSING_SCORE" ] && PASS=true || true + cat << EOF > score.json { - "score": 0, - "pass": true, - "passingScore": 0 + "score": $SCORE, + "pass": $PASS, + "passingScore": $PASSING_SCORE, + "ai_assist_present": $AI_ASSIST_EXISTS } EOF diff --git a/AI_ASSIST.md b/AI_ASSIST.md new file mode 100644 index 0000000..a50dbac --- /dev/null +++ b/AI_ASSIST.md @@ -0,0 +1,19 @@ +# AI Assist Report + +## The prompt I gave + + + +## The code it suggested + +```python +# Paste the relevant code the LLM suggested here. +``` + +## What I changed and why + + + +## Did it work? + + diff --git a/README.md b/README.md index 96ce7bc..92c8951 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,42 @@ -# [Track] week X assignment -HackYourFuture week X assignment -The Week X assignment for the HackYourFuture can be found at the following link: [TODO: Assignment url in the learning platform] +# Week 4 Assignment: MessyCorp Pandas +Read the full assignment on the HYF Data Track: [Assignment: MessyCorp Pandas](https://hub.hackyourfuture.nl/) -## Implementation Instructions +## Where to start -Provide clear instructions on how trainees should implement the tasks. +| File | Task | +|---|---| +| `src/ingest.py` | Task 1 (download inputs) and Task 7 (upload results) | +| `src/clean.py` | Task 2 (explore) and Task 3 (clean sales) | +| `src/transform.py` | Task 4 (join customers, add `is_high_value`) | +| `src/report.py` | Task 5 (build report tables) and Task 6 (write outputs) | +| `main.py` | Pipeline runner — set `GITHUB_USERNAME` before running Task 7 | +| `AI_ASSIST.md` | Task 8 — fill in before submitting | -### Task 1 -Instructions for Task 1 +## Setup -### Task 2 -Instructions for Task 2 +```bash +pip install pandas azure-identity azure-storage-blob matplotlib pyarrow +``` -... +Log in to Azure (reuses your Week 2 session): +```bash +az login +``` + +## Running the pipeline + +```bash +python main.py +``` + +This downloads inputs from Azure, cleans and transforms them, writes reports to `output/`, and uploads results back to Azure. + +`data/` and `output/` are excluded from git — they are generated at runtime. + +## Submitting + +1. Create a branch `week4/your-name`. +2. Commit your work. +3. Push and open a Pull Request. diff --git a/main.py b/main.py new file mode 100644 index 0000000..9f17efc --- /dev/null +++ b/main.py @@ -0,0 +1,36 @@ +"""MessyCorp Pandas pipeline — Task 1 through 7 runner.""" +import logging +from pathlib import Path + +from src.ingest import download_inputs, upload_outputs +from src.clean import load_and_explore, clean_sales +from src.transform import join_customers +from src.report import build_reports, write_outputs + +logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + +DATA_DIR = Path("data") +OUTPUT_DIR = Path("output") + +# TODO (Task 7): replace with your GitHub username before running the pipeline. +GITHUB_USERNAME = "" + + +def run() -> None: + download_inputs(DATA_DIR) + + sales_raw, customers_raw = load_and_explore(DATA_DIR) + + sales_clean = clean_sales(sales_raw) + enriched = join_customers(sales_clean, customers_raw) + + reports = build_reports(enriched) + write_outputs(reports, OUTPUT_DIR) + + upload_outputs(OUTPUT_DIR, GITHUB_USERNAME) + + logging.info("Pipeline complete.") + + +if __name__ == "__main__": + run() diff --git a/src/clean.py b/src/clean.py new file mode 100644 index 0000000..b4036fd --- /dev/null +++ b/src/clean.py @@ -0,0 +1,28 @@ +"""Tasks 2 and 3: Explore and clean the raw DataFrames.""" +import logging +from pathlib import Path + +import pandas as pd + + +def load_and_explore(data_dir: Path) -> tuple[pd.DataFrame, pd.DataFrame]: + """Task 2: Load both CSV files and explore their contents before cleaning.""" + # TODO: Read messy_sales.csv and messy_customers.csv with pd.read_csv(). + # TODO: For each DataFrame call .info(), .describe(), .head(20), and .isna().sum(). + # TODO: Log what you discover (e.g. which columns have nulls, any suspicious values). + raise NotImplementedError("Task 2: implement load_and_explore") + + +def clean_sales(sales: pd.DataFrame) -> pd.DataFrame: + """Task 3: Clean the sales DataFrame using vectorized Pandas operations.""" + # TODO: Normalize product_name with .str.strip().str.title(). + # TODO: Normalize customer_email with .str.lower().str.strip(). + # TODO: Convert price to numeric with pd.to_numeric(errors="coerce"). + # TODO: Parse date with pd.to_datetime(errors="coerce"). + # TODO: Drop rows where product_name is missing. + # TODO: Drop rows where price is negative. + # TODO: Drop rows where quantity is zero. + # TODO: Drop rows where date is NaT (invalid after parsing). + # TODO: Remove duplicate transactions: .drop_duplicates(subset="transaction_id", keep="first"). + # TODO: Decide what to do with outlier prices (clip, flag, or leave) and add a comment explaining why. + raise NotImplementedError("Task 3: implement clean_sales") diff --git a/src/ingest.py b/src/ingest.py new file mode 100644 index 0000000..2501bb9 --- /dev/null +++ b/src/ingest.py @@ -0,0 +1,33 @@ +"""Task 1: Download inputs from Azure. Task 7: Upload outputs back to Azure.""" +import io +import logging +from pathlib import Path + +import pandas as pd +from azure.identity import DefaultAzureCredential +from azure.storage.blob import BlobServiceClient + +ACCOUNT_URL = "https://sthyfstudentsdemo.blob.core.windows.net" +SOURCE_CONTAINER = "week4-inputs" +FILES = ["messy_sales.csv", "messy_customers.csv"] + + +def download_inputs(data_dir: Path) -> None: + """Task 1: Download input CSV files from Azure Blob Storage.""" + # TODO: Create a BlobServiceClient using DefaultAzureCredential and ACCOUNT_URL. + # TODO: Get a container client for SOURCE_CONTAINER. + # TODO: For each filename in FILES, download the blob and write it to data_dir/. + # TODO: Log a message for each downloaded file. + raise NotImplementedError("Task 1: implement download_inputs") + + +def upload_outputs(output_dir: Path, github_username: str) -> None: + """Task 7: Upload Parquet outputs to a personal Azure container and verify the round-trip.""" + container_name = f"week4-{github_username}" + + # TODO: Create a BlobServiceClient using DefaultAzureCredential and ACCOUNT_URL. + # TODO: Get (or create) the container named container_name. + # TODO: Upload every .parquet file in output_dir to the container. + # TODO: Download customer_summary.parquet back and assert its row count matches the local file. + # TODO: Log the container name and number of files uploaded. + raise NotImplementedError("Task 7: implement upload_outputs") diff --git a/src/report.py b/src/report.py new file mode 100644 index 0000000..a002b7b --- /dev/null +++ b/src/report.py @@ -0,0 +1,31 @@ +"""Tasks 5 and 6: Build report tables and write outputs.""" +import logging +from pathlib import Path + +import pandas as pd + + +def build_reports(enriched: pd.DataFrame) -> dict[str, pd.DataFrame]: + """Task 5: Build four summary tables using groupby and named aggregations.""" + # TODO: Add a week column using .dt.isocalendar().week. + # TODO: Build weekly_revenue: group by week and region, columns week/region/total_revenue/order_count. + # TODO: Build customer_summary: group by customer_email, columns customer_email/customer_name/ + # region/loyalty_tier/total_spent/avg_order/order_count. + # Use ("customer_name", "first") to pick the constant-per-group string columns. + # TODO: Build category_performance: group by category, columns category/total_revenue/order_count. + # TODO: Build loyalty_analysis: group by loyalty_tier, columns loyalty_tier/avg_spent/customer_count. + raise NotImplementedError("Task 5: implement build_reports") + + +def write_outputs(reports: dict[str, pd.DataFrame], output_dir: Path) -> None: + """Task 6: Write report tables to CSV/Parquet and save a bar chart.""" + output_dir.mkdir(exist_ok=True) + + # TODO: Write reports["weekly_revenue"] to weekly_revenue.csv with index=False. + # TODO: Write reports["customer_summary"] to customer_summary.parquet with index=False. + # TODO: Write reports["category_performance"] to category_performance.csv with index=False. + # TODO: Sort category_performance by total_revenue descending. + # TODO: Plot a bar chart (x="category", y="total_revenue") and save to category_revenue.png + # using plt.savefig(output_dir / "category_revenue.png", bbox_inches="tight"). + # Use matplotlib.use("Agg") before importing pyplot for headless environments. + raise NotImplementedError("Task 6: implement write_outputs") diff --git a/src/transform.py b/src/transform.py new file mode 100644 index 0000000..18f82e5 --- /dev/null +++ b/src/transform.py @@ -0,0 +1,13 @@ +"""Task 4: Join customer data and add derived columns.""" +import logging + +import pandas as pd + + +def join_customers(sales: pd.DataFrame, customers: pd.DataFrame) -> pd.DataFrame: + """Task 4: Normalize join keys, merge, and add a derived boolean flag.""" + # TODO: Normalize customer_email in both DataFrames with .str.lower().str.strip(). + # TODO: Merge sales with customers on customer_email using an inner join. + # TODO: Add a vectorized boolean column is_high_value: True where price * quantity >= 150. + # TODO: (Optional hands-on) Try a left join instead and inspect rows where customer_name is NaN. + raise NotImplementedError("Task 4: implement join_customers")