From 4f8ac12ff132191429a5b55c47c0ef6c6ab19d30 Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Mon, 18 May 2026 15:57:31 +0200 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20scaffold=20Week=203=20assignment=20?= =?UTF-8?q?=E2=80=94=20validated=20ingestion=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds all template files for the Week 3 assignment: - README.md with task table, repo layout, scoring ladder, local grader instructions - task-1/: 6 skeleton Python modules with TODO comments (models, ingest_api, ingest_files, validate, database, pipeline), messy weather_stations.csv, requirements.txt, .env.example, and azure_compare.md template in output/ - task-2/: AI_DEBUG.md template with 4 required sections - .hyf/test.sh: full autograder (100 pts, passing=60) with incremental scoring ladder for the pipeline (0→10→20→40→50→70), Task 7 Azure output checks (15 pts), and Task 8 AI debug report checks (15 pts) - .devcontainer/devcontainer.json: Python 3.11 + Azure CLI Codespace - AZURE_LOGIN.md: device-code login instructions for Codespaces and local - .gitignore: Python patterns + generated pipeline outputs Co-Authored-By: Claude Sonnet 4.6 --- .devcontainer/devcontainer.json | 25 ++++ .gitignore | 23 ++++ .hyf/test.sh | 212 ++++++++++++++++++++++++++++++- AZURE_LOGIN.md | 55 ++++++++ README.md | 83 ++++++++++-- task-1/.env.example | 11 ++ task-1/data/weather_stations.csv | 11 ++ task-1/database.py | 48 +++++++ task-1/ingest_api.py | 39 ++++++ task-1/ingest_files.py | 17 +++ task-1/models.py | 14 ++ task-1/output/azure_compare.md | 20 +++ task-1/pipeline.py | 42 ++++++ task-1/requirements.txt | 2 + task-1/task 1 files | 0 task-1/validate.py | 18 +++ task-2/AI_DEBUG.md | 26 ++++ task-2/task 2 files | 0 18 files changed, 629 insertions(+), 17 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 AZURE_LOGIN.md create mode 100644 task-1/.env.example create mode 100644 task-1/data/weather_stations.csv create mode 100644 task-1/database.py create mode 100644 task-1/ingest_api.py create mode 100644 task-1/ingest_files.py create mode 100644 task-1/models.py create mode 100644 task-1/output/azure_compare.md create mode 100644 task-1/pipeline.py create mode 100644 task-1/requirements.txt delete mode 100644 task-1/task 1 files create mode 100644 task-1/validate.py create mode 100644 task-2/AI_DEBUG.md delete mode 100644 task-2/task 2 files diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..23f91dc --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,25 @@ +{ + "name": "Week 3 Assignment", + "image": "mcr.microsoft.com/devcontainers/python:3.11", + "features": { + "ghcr.io/devcontainers/features/azure-cli:1": {} + }, + "postCreateCommand": "python3 -m pip install -r task-1/requirements.txt", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.debugpy", + "charliermarsh.ruff" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true + } + } + } + } +} diff --git a/.gitignore b/.gitignore index 2b76d7c..5a0ee9d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,29 @@ Thumbs.db # hyf .hyf/score.json +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd +.Python +*.egg-info/ +dist/ +build/ +.venv/ +venv/ +env/ + +# dotenv +.env +.env.* +!.env.example + +# Generated pipeline output (committed templates stay; generated files do not) +task-1/output/error_report.json +task-1/output/azure_resource_groups.json +task-1/weather.db + # Editor and IDE settings .vscode/ .idea/ diff --git a/.hyf/test.sh b/.hyf/test.sh index ee037fc..0e45550 100644 --- a/.hyf/test.sh +++ b/.hyf/test.sh @@ -1,13 +1,211 @@ #!/usr/bin/env bash +# Auto-grade Week 3 assignment. Writes score.json next to this script. +# Total = 100, passing = 60. +# +# Run from repo root: bash .hyf/test.sh +# Or from .hyf/: bash test.sh set -euo pipefail -# Run your test scripts here. -# Auto grade tool will execute this file within the .hyf working directory. -# The result should be stored in score.json file with the format shown below. -cat << EOF > score.json +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$REPO_ROOT" + +PASSING=60 + +# --- Tasks 1-6: Ingestion Pipeline (70 points) --- +# +# Scoring ladder (each level depends on the previous): +# 0 nothing committed +# 10 required files all present +# 20 pipeline runs without crashing +# 40 output/error_report.json is valid + weather.db has rows +# 50 pipeline is idempotent (same row count on second run) +# 70 code uses required patterns (@field_validator, parameterized queries, +# ON CONFLICT, retry/backoff) +task16=0 +task16_msg="missing required files in task-1/" + +required_files=( + "task-1/models.py" + "task-1/ingest_api.py" + "task-1/ingest_files.py" + "task-1/validate.py" + "task-1/database.py" + "task-1/pipeline.py" + "task-1/.env.example" +) + +all_present=true +for f in "${required_files[@]}"; do + if [ ! -f "$f" ]; then + all_present=false + break + fi +done + +if [ "$all_present" = true ]; then + task16=10 + task16_msg="files exist but pipeline failed to run" + + if [ -f task-1/requirements.txt ]; then + python3 -m pip install -q -r task-1/requirements.txt || \ + echo "WARN: pip install failed; pipeline may crash with ModuleNotFoundError" >&2 + fi + + PIPELINE_ERR=$(mktemp) + if ( cd task-1 && python3 -m pipeline ) >/dev/null 2>"$PIPELINE_ERR"; then + task16=20 + task16_msg="pipeline ran but output checks failed" + + STRUCT_ERR=$(mktemp) + if python3 - <<'PY' 2>"$STRUCT_ERR" +import json, sqlite3 +from pathlib import Path + +# error_report.json must exist and be a non-empty list of error objects +rpt = Path("task-1/output/error_report.json") +assert rpt.exists(), "output/error_report.json was not created" +errors = json.loads(rpt.read_text()) +assert isinstance(errors, list), "error_report.json must be a JSON list" +assert len(errors) > 0, "error_report.json is empty — the CSV has intentional bad rows that should fail validation" + +required_fields = {"index", "source", "raw_record", "error_details"} +for i, e in enumerate(errors[:3]): + missing = required_fields - set(e.keys()) + assert not missing, f"error object {i} missing fields: {missing}" + +# weather.db must exist and have rows +db = Path("task-1/weather.db") +assert db.exists(), "weather.db was not created" +conn = sqlite3.connect(db) +count = conn.execute("SELECT COUNT(*) FROM weather_readings").fetchone()[0] +assert count > 0, "weather_readings table is empty after pipeline run" +PY + then + rm -f "$STRUCT_ERR" + task16=40 + task16_msg="output checks passed; testing idempotency" + + # Idempotency: run a second time, row count must stay the same + count_before=$(python3 -c " +import sqlite3 +conn = sqlite3.connect('task-1/weather.db') +print(conn.execute('SELECT COUNT(*) FROM weather_readings').fetchone()[0]) +") + if ( cd task-1 && python3 -m pipeline ) >/dev/null 2>&1; then + count_after=$(python3 -c " +import sqlite3 +conn = sqlite3.connect('task-1/weather.db') +print(conn.execute('SELECT COUNT(*) FROM weather_readings').fetchone()[0]) +") + if [ "$count_before" = "$count_after" ]; then + task16=50 + task16_msg="output + idempotency pass; checking code patterns" + else + task16_msg="second run changed row count ($count_before -> $count_after); upsert not working" + fi + fi + + # Code introspection for the final 20 points + has_field_validator=$(grep -cE "@field_validator" task-1/models.py || true) + has_classmethod=$(grep -cE "@classmethod" task-1/models.py || true) + has_param_queries=$(grep -cE "\?" task-1/database.py || true) + has_on_conflict=$(grep -ciE "ON CONFLICT" task-1/database.py || true) + has_retry=$(grep -cEi "retry|backoff|sleep" task-1/ingest_api.py || true) + + if [ "$has_field_validator" -gt 0 ] && \ + [ "$has_classmethod" -gt 0 ] && \ + [ "$has_param_queries" -gt 0 ] && \ + [ "$has_on_conflict" -gt 0 ] && \ + [ "$has_retry" -gt 0 ]; then + task16=70 + task16_msg="pipeline + idempotency + code patterns all pass" + else + missing_patterns=() + [ "$has_field_validator" -eq 0 ] && missing_patterns+=("@field_validator in models.py") + [ "$has_classmethod" -eq 0 ] && missing_patterns+=("@classmethod in models.py") + [ "$has_param_queries" -eq 0 ] && missing_patterns+=("parameterized ? queries in database.py") + [ "$has_on_conflict" -eq 0 ] && missing_patterns+=("ON CONFLICT in database.py") + [ "$has_retry" -eq 0 ] && missing_patterns+=("retry/backoff logic in ingest_api.py") + task16_msg="output passes but code missing: $(IFS=, ; echo "${missing_patterns[*]}")" + fi + else + err=$(tail -3 "$STRUCT_ERR" | tr '\n' ' ' | sed 's/ */ /g' | sed 's/^ //;s/ $//') + [ -n "$err" ] && task16_msg="output check failed: $err" + rm -f "$STRUCT_ERR" + fi + else + err=$(tail -3 "$PIPELINE_ERR" | tr '\n' ' ' | sed 's/ */ /g' | sed 's/^ //;s/ $//') + [ -n "$err" ] && task16_msg="pipeline failed to run: $err" + fi + rm -f "$PIPELINE_ERR" +fi + +# --- Task 7: Azure CLI + Portal (15 points) --- +# +# 5 pts azure_resource_groups.json exists and is valid JSON +# 10 pts azure_compare.md exists +# 15 pts azure_compare.md has all 3 sections and looks filled in (>600 chars) +task7=0 +task7_msg="missing task-1/output/azure_resource_groups.json" + +if [ -s "task-1/output/azure_resource_groups.json" ]; then + if python3 -c "import json; json.load(open('task-1/output/azure_resource_groups.json'))" 2>/dev/null; then + task7=5 + task7_msg="azure_resource_groups.json is valid JSON; azure_compare.md missing or not filled in" + + if [ -s "task-1/output/azure_compare.md" ]; then + task7=10 + task7_msg="azure_compare.md exists but looks too short or missing sections" + section_count=$(grep -cE "^## " task-1/output/azure_compare.md || true) + char_count=$(wc -c < task-1/output/azure_compare.md) + if [ "$section_count" -ge 3 ] && [ "$char_count" -gt 600 ]; then + task7=15 + task7_msg="azure_resource_groups.json and azure_compare.md both present and filled in" + fi + fi + else + task7_msg="task-1/output/azure_resource_groups.json is not valid JSON" + fi +fi + +# --- Task 8: AI Debug Report (15 points) --- +# +# 5 pts task-2/AI_DEBUG.md exists +# 10 pts all four sections present (## The Error, ## The Prompt, ## The Solution, ## Reflection) +# 15 pts file is meaningfully filled in (>1800 chars) +task8=0 +task8_msg="missing task-2/AI_DEBUG.md" + +if [ -s "task-2/AI_DEBUG.md" ]; then + task8=5 + task8_msg="AI_DEBUG.md exists but missing required sections" + if grep -q "^## The Error" task-2/AI_DEBUG.md && \ + grep -q "^## The Prompt" task-2/AI_DEBUG.md && \ + grep -q "^## The Solution" task-2/AI_DEBUG.md && \ + grep -q "^## Reflection" task-2/AI_DEBUG.md; then + task8=10 + task8_msg="all sections present but file looks too short to be filled in" + if [ "$(wc -c < task-2/AI_DEBUG.md)" -gt 1800 ]; then + task8=15 + task8_msg="AI_DEBUG.md is filled in" + fi + fi +fi + +score=$((task16 + task7 + task8)) +if [ "$score" -ge "$PASSING" ]; then pass=true; else pass=false; fi + +cat > "$SCRIPT_DIR/score.json" < week X assignment -The Week X assignment for the HackYourFuture can be found at the following link: [TODO: Assignment url in the learning platform] +# Data Track — Week 3 Assignment (Template) +The HackYourFuture Data Track Week 3 assignment: **Build a Validated Ingestion Pipeline**. -## Implementation Instructions +> 👩‍🎓 **Students:** you are in the wrong place. Do **not** fork or use this template. +> Go to your cohort's assignment repo under +> [`HackYourAssignment`](https://github.com/HackYourAssignment) (e.g. `c55-data-week3`, +> `c56-data-week3`, …). Your teacher posts the exact link in your cohort channel. +> Fork the cohort repo, branch, and open a PR back to it. Full instructions live in the +> Week 3 Assignment chapter in the learning platform. -Provide clear instructions on how trainees should implement the tasks. +## For instructors / track maintainers -### Task 1 -Instructions for Task 1 +This repo is the **upstream template** for the Week 3 assignment. At the start of each +cohort, generate a cohort-specific repo under the `HackYourAssignment` org from this +template (GitHub: **Use this template → Create a new repository**, owner = +`HackYourAssignment`, name = `c-data-week3`). Students then fork *that* cohort repo +and open PRs back to it; the auto-grader runs on every push. -### Task 2 -Instructions for Task 2 +Edits to the assignment, dataset, or grader belong here on the template, not on the +cohort copies. -... +## Tasks at a glance +| Task | Folder | Points | What you build | +|---|---|---|---| +| **Tasks 1-6** — Ingestion Pipeline | `task-1/` | 70 | A modular pipeline: `fetch_with_retry` with exponential backoff, Open-Meteo API ingestion, CSV file ingestion, Pydantic validation with `@field_validator`, SQLite upsert storage, and a `pipeline.py` orchestrator that produces an error report and pipeline summary. | +| **Task 7** — Azure CLI + Portal | `task-1/output/` | 15 | Run three `az` CLI commands, call the ARM API with a Bearer token, save `azure_resource_groups.json`, and fill in `azure_compare.md` with three comparison points. | +| **Task 8** — AI Debug Report | `task-2/` | 15 | Document one LLM-assisted debugging session. Fill in the four sections of `AI_DEBUG.md`. | + +Total: 100 · Passing: 60. + +## Repository layout + +```text +. +├── task-1/ +│ ├── data/ +│ │ └── weather_stations.csv # messy input dataset (committed; do not edit) +│ ├── output/ # pipeline writes here (gitignored except templates) +│ │ ├── error_report.json # generated by pipeline.py +│ │ ├── azure_resource_groups.json # Task 7: save ARM API response here +│ │ └── azure_compare.md # Task 7: fill in 3 comparison points +│ ├── models.py # Pydantic WeatherReading model — fill in TODOs +│ ├── ingest_api.py # fetch_with_retry + API ingestion — fill in TODOs +│ ├── ingest_files.py # CSV reader — fill in TODOs +│ ├── validate.py # batch validation — fill in TODOs +│ ├── database.py # SQLite create, upsert, query — fill in TODOs +│ ├── pipeline.py # orchestrator — fill in TODOs +│ ├── .env.example # no secrets needed; copy to .env if you extend it +│ └── requirements.txt +├── task-2/ +│ └── AI_DEBUG.md # Task 8: fill in the four sections +├── .hyf/ +│ └── test.sh # auto-grader (read it to see exactly what it checks) +└── .github/workflows/ + └── grade-assignment.yml # runs .hyf/test.sh on every PR +``` + +## Run the grader locally + +Before opening a PR, run the same checks the auto-grader runs: + +```bash +cd task-1 +python3 -m pip install -r requirements.txt +cd .. +bash .hyf/test.sh +cat .hyf/score.json +``` + +## Scoring ladder (Tasks 1-6) + +The grader awards points incrementally so partial credit is meaningful: + +- **10/70** — required files all exist (`models.py`, `ingest_api.py`, `ingest_files.py`, `validate.py`, `database.py`, `pipeline.py`, `.env.example`). +- **20/70** — `python3 -m pipeline` runs from `task-1/` without crashing. +- **40/70** — `output/error_report.json` exists, is a valid JSON list, and contains objects with `index`, `source`, `raw_record`, and `error_details` fields; `weather.db` has rows in `weather_readings`. +- **50/70** — pipeline is idempotent: running it twice leaves the same row count in `weather_readings` (upsert working correctly). +- **70/70** — code uses the required patterns: `@field_validator` + `@classmethod` in `models.py`, parameterized queries (`?` placeholders) in `database.py`, `ON CONFLICT ... DO UPDATE SET` in `database.py`, retry/backoff logic in `ingest_api.py`. diff --git a/task-1/.env.example b/task-1/.env.example new file mode 100644 index 0000000..b57c183 --- /dev/null +++ b/task-1/.env.example @@ -0,0 +1,11 @@ +# The weather pipeline uses only public APIs and local files. +# No secrets are required to run it. +# +# If you add your own data sources (e.g. a paid weather API), +# add the keys here and load them with python-dotenv: +# +# from dotenv import load_dotenv +# load_dotenv() +# API_KEY = os.getenv("MY_API_KEY") + +# MY_API_KEY=your_key_here diff --git a/task-1/data/weather_stations.csv b/task-1/data/weather_stations.csv new file mode 100644 index 0000000..cbbdffa --- /dev/null +++ b/task-1/data/weather_stations.csv @@ -0,0 +1,11 @@ +station,timestamp,temperature_c,humidity_pct +Copenhagen,2025-01-15T10:00,18.5,72 +,2025-01-15T11:00,20.1,65 +Aarhus,2025-01-15T12:00,N/A,58 +Odense,2025-01-15T13:00,15.2,150 +Copenhagen,2025-01-15T10:00,19.0,70 +Aalborg,,14.8,62 +Roskilde,2025-01-15T15:00,-95.0,45 +Esbjerg,2025-01-15T16:00,16.3, +Herning,2025-01-15T17:00,17.1,55 +Kolding,2025-01-15T18:00,14.9,68 diff --git a/task-1/database.py b/task-1/database.py new file mode 100644 index 0000000..0abecab --- /dev/null +++ b/task-1/database.py @@ -0,0 +1,48 @@ +import sqlite3 +from pathlib import Path + +from models import WeatherReading + +DB_PATH = Path("weather.db") + + +def get_connection() -> sqlite3.Connection: + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + return conn + + +def create_tables(conn: sqlite3.Connection) -> None: + """Create raw_weather and weather_readings tables if they do not exist. + + raw_weather columns: id, station, timestamp, temperature_c, humidity_pct, source, ingested_at + weather_readings columns: id, station, timestamp, temperature_c, humidity_pct + + UNIQUE(station, timestamp) constraint for upserts + """ + # TODO: use conn.execute() with CREATE TABLE IF NOT EXISTS statements + raise NotImplementedError + + +def insert_raw(conn: sqlite3.Connection, records: list[dict], source: str) -> None: + """Insert raw records (before validation) into raw_weather. + + Use parameterized queries (? placeholders). Do not use string formatting for values. + """ + # TODO: implement + raise NotImplementedError + + +def upsert_readings(conn: sqlite3.Connection, readings: list[WeatherReading]) -> None: + """Upsert valid WeatherReading objects into weather_readings. + + Use ON CONFLICT(station, timestamp) DO UPDATE SET ... to handle duplicates. + Use parameterized queries. + """ + # TODO: implement + raise NotImplementedError + + +def count_readings(conn: sqlite3.Connection) -> int: + """Return the total number of rows in weather_readings.""" + # TODO: implement + raise NotImplementedError diff --git a/task-1/ingest_api.py b/task-1/ingest_api.py new file mode 100644 index 0000000..f8e3ecf --- /dev/null +++ b/task-1/ingest_api.py @@ -0,0 +1,39 @@ +import logging +import time + +import requests + +logger = logging.getLogger(__name__) + +API_URL = "https://api.open-meteo.com/v1/forecast" + + +def fetch_with_retry(url: str, params: dict, max_retries: int = 3, timeout: int = 10) -> dict: + """Fetch url with exponential backoff on transient errors. + + Retry on: ConnectionError, Timeout, 5xx status codes. + Fail immediately on: 4xx status codes. + Log each retry attempt with the error and delay. + """ + # TODO: implement retry loop with exponential backoff + # Hint: use time.sleep(2 ** attempt) between retries + raise NotImplementedError + + +def fetch_api_records() -> list[dict]: + """Fetch hourly weather from Open-Meteo and return flat dicts. + + Returns a list of dicts with keys: station, timestamp, temperature_c, humidity_pct. + Returns [] if the API returns no data (do not raise an exception). + """ + params = { + "latitude": 55.67, + "longitude": 12.56, + "hourly": "temperature_2m,relative_humidity_2m", + "forecast_days": 7, + } + # TODO: + # - Call fetch_with_retry with API_URL and params + # - The API returns {"hourly": {"time": [...], "temperature_2m": [...], "relative_humidity_2m": [...]}} + # - Flatten to a list of dicts; set station="Open-Meteo Copenhagen" for all records + raise NotImplementedError diff --git a/task-1/ingest_files.py b/task-1/ingest_files.py new file mode 100644 index 0000000..9f01e15 --- /dev/null +++ b/task-1/ingest_files.py @@ -0,0 +1,17 @@ +import csv +from pathlib import Path + + +def read_csv_records(path: Path) -> list[dict]: + """Read weather_stations.csv and return normalized records. + + Returns a list of dicts with keys: station, timestamp, temperature_c, humidity_pct. + + Rules: + - Open with newline="" and encoding="utf-8". + - Use csv.DictReader. + - Convert temperature_c to float and humidity_pct to int where possible. + - Leave unconvertible values (e.g. "N/A", "") as-is so validation can catch them. + """ + # TODO: implement CSV reading and normalization + raise NotImplementedError diff --git a/task-1/models.py b/task-1/models.py new file mode 100644 index 0000000..7a4680f --- /dev/null +++ b/task-1/models.py @@ -0,0 +1,14 @@ +from pydantic import BaseModel, Field, field_validator + + +class WeatherReading(BaseModel): + station: str = Field(..., min_length=1) + timestamp: str = Field(..., min_length=1) + temperature_c: float = Field(..., ge=-90, le=60) + humidity_pct: int = Field(..., ge=0, le=100) + + @field_validator("station") + @classmethod + def clean_station(cls, v: str) -> str: + # TODO: strip whitespace and convert to title case + raise NotImplementedError diff --git a/task-1/output/azure_compare.md b/task-1/output/azure_compare.md new file mode 100644 index 0000000..3ba6f74 --- /dev/null +++ b/task-1/output/azure_compare.md @@ -0,0 +1,20 @@ +# Azure API Comparison + +Fill in each section below (2-3 sentences each) after completing the Task 7 steps. + +## Auth + + + +## Schema verbosity + + + +## api-version in the URL + + diff --git a/task-1/pipeline.py b/task-1/pipeline.py new file mode 100644 index 0000000..3cd1a85 --- /dev/null +++ b/task-1/pipeline.py @@ -0,0 +1,42 @@ +import json +import logging +from pathlib import Path + +from database import count_readings, create_tables, get_connection, insert_raw, upsert_readings +from ingest_api import fetch_api_records +from ingest_files import read_csv_records +from validate import validate_records + +logging.basicConfig(level=logging.INFO) + +OUTPUT_DIR = Path("output") +CSV_PATH = Path("data/weather_stations.csv") + + +def run_pipeline() -> None: + OUTPUT_DIR.mkdir(exist_ok=True) + + # TODO — implement each step in order: + # + # 1. Fetch records from Open-Meteo API using fetch_api_records() + # 2. Read records from CSV using read_csv_records(CSV_PATH) + # 3. Open a DB connection, create tables, insert all raw records (both sources) + # 4. Validate all records — collect valid WeatherReading objects and error dicts + # 5. Upsert valid records into weather_readings + # 6. Save error dicts as JSON to output/error_report.json + # 7. Print the pipeline summary exactly as shown below: + # + # === Pipeline Summary === + # API records fetched: 168 + # CSV records read: 10 + # Total raw records: 178 + # Valid records: 170 + # Invalid records: 8 + # Records in database: 170 + # Error report: output/error_report.json + + raise NotImplementedError + + +if __name__ == "__main__": + run_pipeline() diff --git a/task-1/requirements.txt b/task-1/requirements.txt new file mode 100644 index 0000000..82f9509 --- /dev/null +++ b/task-1/requirements.txt @@ -0,0 +1,2 @@ +pydantic>=2.0 +requests>=2.28 diff --git a/task-1/task 1 files b/task-1/task 1 files deleted file mode 100644 index e69de29..0000000 diff --git a/task-1/validate.py b/task-1/validate.py new file mode 100644 index 0000000..c2b694d --- /dev/null +++ b/task-1/validate.py @@ -0,0 +1,18 @@ +from pydantic import ValidationError + +from models import WeatherReading + + +def validate_records( + records: list[dict], source: str +) -> tuple[list[WeatherReading], list[dict]]: + """Validate records against WeatherReading and return (valid_list, error_list). + + Each error dict must contain: + index - position of the record in the input list + source - the source string passed in (e.g. "api" or "csv") + raw_record - the original dict + error_details - the Pydantic error list (ValidationError.errors()) + """ + # TODO: iterate over records, try WeatherReading(**record), accumulate results + raise NotImplementedError diff --git a/task-2/AI_DEBUG.md b/task-2/AI_DEBUG.md new file mode 100644 index 0000000..edf34a5 --- /dev/null +++ b/task-2/AI_DEBUG.md @@ -0,0 +1,26 @@ +# AI Debugging Report + +While building your pipeline you will run into at least one bug. +Document the debugging session below. If everything worked first try, introduce a bug intentionally and debug it. + +--- + +## The Error + + + +## The Prompt + + + +## The Solution + + + +## Reflection + + diff --git a/task-2/task 2 files b/task-2/task 2 files deleted file mode 100644 index e69de29..0000000 From 86989c44b2ac39707cf2e22b57172a40d4fa44b2 Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Mon, 18 May 2026 16:30:28 +0200 Subject: [PATCH 2/2] fix: address autograder and scaffold bugs from review - Nest code-pattern introspection inside idempotency gate so scores can't jump from 40 to 70 without passing the 50-point upsert check - Replace grep patterns that matched scaffold docstrings with patterns that only match actual code: execute(.*? for parameterised queries, time\.sleep for backoff (avoids the function name fetch_with_retry), ON CONFLICT check unchanged but removed from docstrings - Remove "ON CONFLICT" and "? placeholders" from database.py docstrings so they no longer satisfy the code-pattern greps on the unimplemented scaffold - Remove time.sleep hint comment from ingest_api.py for the same reason - Initialize score.json to 0/false at script start so a set -e crash before the final write does not leave a stale passing score - Raise azure_compare.md fill-in threshold from 600 to 1200 chars and shorten the committed template to 233 bytes so students must write actual prose to earn the 15-point tier - Fix pipeline.py example summary: invalid records 8 -> 6 (the duplicate Copenhagen row is valid and exercises upsert, not validation); add note that API count varies by time of day (up to 168 hourly records) - Move logging.basicConfig into the __main__ guard to avoid root-logger side effects on import - Add missing import os to .env.example code snippet --- .hyf/test.sh | 71 ++++++++++++++++++++-------------- task-1/.env.example | 1 + task-1/database.py | 4 +- task-1/ingest_api.py | 1 - task-1/output/azure_compare.md | 11 ++---- task-1/pipeline.py | 19 +++++---- 6 files changed, 61 insertions(+), 46 deletions(-) diff --git a/.hyf/test.sh b/.hyf/test.sh index 0e45550..130ee65 100644 --- a/.hyf/test.sh +++ b/.hyf/test.sh @@ -12,16 +12,22 @@ cd "$REPO_ROOT" PASSING=60 +# Initialise score.json to 0 immediately so a crash before the final write +# does not leave a stale result from a previous run. +cat > "$SCRIPT_DIR/score.json" <<'INIT' +{"score": 0, "pass": false, "passingScore": 60} +INIT + # --- Tasks 1-6: Ingestion Pipeline (70 points) --- # -# Scoring ladder (each level depends on the previous): +# Scoring ladder (each level requires all previous levels to pass): # 0 nothing committed # 10 required files all present # 20 pipeline runs without crashing # 40 output/error_report.json is valid + weather.db has rows # 50 pipeline is idempotent (same row count on second run) # 70 code uses required patterns (@field_validator, parameterized queries, -# ON CONFLICT, retry/backoff) +# ON CONFLICT upsert, time.sleep backoff) task16=0 task16_msg="missing required files in task-1/" @@ -101,34 +107,42 @@ print(conn.execute('SELECT COUNT(*) FROM weather_readings').fetchone()[0]) if [ "$count_before" = "$count_after" ]; then task16=50 task16_msg="output + idempotency pass; checking code patterns" + + # Code introspection for the final 20 points. + # These greps target actual code constructs, not docstrings: + # @field_validator / @classmethod — present in scaffold but only + # safe to match after pipeline passes (NotImplementedError in the + # validator would crash the pipeline before we get here) + # execute.*? — SQL parameterized placeholder in a call + # ON CONFLICT — upsert keyword in actual SQL string + # time\.sleep — stdlib sleep call (avoids matching the + # function name "fetch_with_retry" or docstring words) + has_field_validator=$(grep -cE "@field_validator" task-1/models.py || true) + has_classmethod=$(grep -cE "@classmethod" task-1/models.py || true) + has_param_queries=$(grep -cE "execute[a-z]*\(.*\?" task-1/database.py || true) + has_on_conflict=$(grep -ciE "ON CONFLICT" task-1/database.py || true) + has_sleep=$(grep -cE "time\.sleep" task-1/ingest_api.py || true) + + if [ "$has_field_validator" -gt 0 ] && \ + [ "$has_classmethod" -gt 0 ] && \ + [ "$has_param_queries" -gt 0 ] && \ + [ "$has_on_conflict" -gt 0 ] && \ + [ "$has_sleep" -gt 0 ]; then + task16=70 + task16_msg="pipeline + idempotency + code patterns all pass" + else + missing_patterns=() + [ "$has_field_validator" -eq 0 ] && missing_patterns+=("@field_validator in models.py") + [ "$has_classmethod" -eq 0 ] && missing_patterns+=("@classmethod in models.py") + [ "$has_param_queries" -eq 0 ] && missing_patterns+=("parameterized ? in execute() in database.py") + [ "$has_on_conflict" -eq 0 ] && missing_patterns+=("ON CONFLICT in database.py") + [ "$has_sleep" -eq 0 ] && missing_patterns+=("time.sleep backoff in ingest_api.py") + task16_msg="output passes but code missing: $(IFS=, ; echo "${missing_patterns[*]}")" + fi else task16_msg="second run changed row count ($count_before -> $count_after); upsert not working" fi fi - - # Code introspection for the final 20 points - has_field_validator=$(grep -cE "@field_validator" task-1/models.py || true) - has_classmethod=$(grep -cE "@classmethod" task-1/models.py || true) - has_param_queries=$(grep -cE "\?" task-1/database.py || true) - has_on_conflict=$(grep -ciE "ON CONFLICT" task-1/database.py || true) - has_retry=$(grep -cEi "retry|backoff|sleep" task-1/ingest_api.py || true) - - if [ "$has_field_validator" -gt 0 ] && \ - [ "$has_classmethod" -gt 0 ] && \ - [ "$has_param_queries" -gt 0 ] && \ - [ "$has_on_conflict" -gt 0 ] && \ - [ "$has_retry" -gt 0 ]; then - task16=70 - task16_msg="pipeline + idempotency + code patterns all pass" - else - missing_patterns=() - [ "$has_field_validator" -eq 0 ] && missing_patterns+=("@field_validator in models.py") - [ "$has_classmethod" -eq 0 ] && missing_patterns+=("@classmethod in models.py") - [ "$has_param_queries" -eq 0 ] && missing_patterns+=("parameterized ? queries in database.py") - [ "$has_on_conflict" -eq 0 ] && missing_patterns+=("ON CONFLICT in database.py") - [ "$has_retry" -eq 0 ] && missing_patterns+=("retry/backoff logic in ingest_api.py") - task16_msg="output passes but code missing: $(IFS=, ; echo "${missing_patterns[*]}")" - fi else err=$(tail -3 "$STRUCT_ERR" | tr '\n' ' ' | sed 's/ */ /g' | sed 's/^ //;s/ $//') [ -n "$err" ] && task16_msg="output check failed: $err" @@ -145,7 +159,8 @@ fi # # 5 pts azure_resource_groups.json exists and is valid JSON # 10 pts azure_compare.md exists -# 15 pts azure_compare.md has all 3 sections and looks filled in (>600 chars) +# 15 pts azure_compare.md has all 3 sections and is filled in (>1200 chars, +# which is above the committed template's ~310 chars of non-comment text) task7=0 task7_msg="missing task-1/output/azure_resource_groups.json" @@ -159,7 +174,7 @@ if [ -s "task-1/output/azure_resource_groups.json" ]; then task7_msg="azure_compare.md exists but looks too short or missing sections" section_count=$(grep -cE "^## " task-1/output/azure_compare.md || true) char_count=$(wc -c < task-1/output/azure_compare.md) - if [ "$section_count" -ge 3 ] && [ "$char_count" -gt 600 ]; then + if [ "$section_count" -ge 3 ] && [ "$char_count" -gt 1200 ]; then task7=15 task7_msg="azure_resource_groups.json and azure_compare.md both present and filled in" fi diff --git a/task-1/.env.example b/task-1/.env.example index b57c183..a8cce93 100644 --- a/task-1/.env.example +++ b/task-1/.env.example @@ -4,6 +4,7 @@ # If you add your own data sources (e.g. a paid weather API), # add the keys here and load them with python-dotenv: # +# import os # from dotenv import load_dotenv # load_dotenv() # API_KEY = os.getenv("MY_API_KEY") diff --git a/task-1/database.py b/task-1/database.py index 0abecab..f5e2345 100644 --- a/task-1/database.py +++ b/task-1/database.py @@ -26,7 +26,7 @@ def create_tables(conn: sqlite3.Connection) -> None: def insert_raw(conn: sqlite3.Connection, records: list[dict], source: str) -> None: """Insert raw records (before validation) into raw_weather. - Use parameterized queries (? placeholders). Do not use string formatting for values. + Use parameterized queries with placeholder syntax; do not build SQL via string formatting. """ # TODO: implement raise NotImplementedError @@ -35,7 +35,7 @@ def insert_raw(conn: sqlite3.Connection, records: list[dict], source: str) -> No def upsert_readings(conn: sqlite3.Connection, readings: list[WeatherReading]) -> None: """Upsert valid WeatherReading objects into weather_readings. - Use ON CONFLICT(station, timestamp) DO UPDATE SET ... to handle duplicates. + Use the upsert pattern to handle duplicate (station, timestamp) pairs. Use parameterized queries. """ # TODO: implement diff --git a/task-1/ingest_api.py b/task-1/ingest_api.py index f8e3ecf..a7acc55 100644 --- a/task-1/ingest_api.py +++ b/task-1/ingest_api.py @@ -16,7 +16,6 @@ def fetch_with_retry(url: str, params: dict, max_retries: int = 3, timeout: int Log each retry attempt with the error and delay. """ # TODO: implement retry loop with exponential backoff - # Hint: use time.sleep(2 ** attempt) between retries raise NotImplementedError diff --git a/task-1/output/azure_compare.md b/task-1/output/azure_compare.md index 3ba6f74..22395b3 100644 --- a/task-1/output/azure_compare.md +++ b/task-1/output/azure_compare.md @@ -4,17 +4,12 @@ Fill in each section below (2-3 sentences each) after completing the Task 7 step ## Auth - + ## Schema verbosity - + ## api-version in the URL - + diff --git a/task-1/pipeline.py b/task-1/pipeline.py index 3cd1a85..e19de9d 100644 --- a/task-1/pipeline.py +++ b/task-1/pipeline.py @@ -7,8 +7,6 @@ from ingest_files import read_csv_records from validate import validate_records -logging.basicConfig(level=logging.INFO) - OUTPUT_DIR = Path("output") CSV_PATH = Path("data/weather_stations.csv") @@ -24,19 +22,26 @@ def run_pipeline() -> None: # 4. Validate all records — collect valid WeatherReading objects and error dicts # 5. Upsert valid records into weather_readings # 6. Save error dicts as JSON to output/error_report.json - # 7. Print the pipeline summary exactly as shown below: + # 7. Print the pipeline summary in the format below. + # + # Note: the API count varies by time of day (Open-Meteo returns up to 168 hourly + # records for 7 forecast days; the exact number depends on the current UTC hour). + # The CSV contributes 6 invalid records and 4 valid ones; the duplicate Copenhagen + # row is valid and exercises the upsert path rather than the validation error path. + # Your actual output will look similar to this example: # # === Pipeline Summary === - # API records fetched: 168 + # API records fetched: 166 # CSV records read: 10 - # Total raw records: 178 + # Total raw records: 176 # Valid records: 170 - # Invalid records: 8 - # Records in database: 170 + # Invalid records: 6 + # Records in database: 169 # Error report: output/error_report.json raise NotImplementedError if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) run_pipeline()