feat: scaffold Week 3 assignment — validated ingestion pipeline

.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+{
+  "name": "Week 3 Assignment",
+  "image": "mcr.microsoft.com/devcontainers/python:3.11",
+  "features": {
+    "ghcr.io/devcontainers/features/azure-cli:1": {}
+  },
+  "postCreateCommand": "python3 -m pip install -r task-1/requirements.txt",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance",
+        "ms-python.debugpy",
+        "charliermarsh.ruff"
+      ],
+      "settings": {
+        "python.defaultInterpreterPath": "/usr/local/bin/python",
+        "[python]": {
+          "editor.defaultFormatter": "charliermarsh.ruff",
+          "editor.formatOnSave": true
+        }
+      }
+    }
+  }
+}

.gitignore

@@ -6,6 +6,29 @@ Thumbs.db
 # hyf
 .hyf/score.json
 
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+*.egg-info/
+dist/
+build/
+.venv/
+venv/
+env/
+
+# dotenv
+.env
+.env.*
+!.env.example
+
+# Generated pipeline output (committed templates stay; generated files do not)
+task-1/output/error_report.json
+task-1/output/azure_resource_groups.json
+task-1/weather.db
+
 # Editor and IDE settings
 .vscode/
 .idea/

.hyf/test.sh

@@ -1,13 +1,226 @@
 #!/usr/bin/env bash
+# Auto-grade Week 3 assignment. Writes score.json next to this script.
+# Total = 100, passing = 60.
+#
+# Run from repo root: bash .hyf/test.sh
+# Or from .hyf/: bash test.sh
 set -euo pipefail
 
-# Run your test scripts here.
-# Auto grade tool will execute this file within the .hyf working directory.
-# The result should be stored in score.json file with the format shown below.
-cat << EOF > score.json
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+cd "$REPO_ROOT"
+
+PASSING=60
+
+# Initialise score.json to 0 immediately so a crash before the final write
+# does not leave a stale result from a previous run.
+cat > "$SCRIPT_DIR/score.json" <<'INIT'
+{"score": 0, "pass": false, "passingScore": 60}
+INIT
+
+# --- Tasks 1-6: Ingestion Pipeline (70 points) ---
+#
+# Scoring ladder (each level requires all previous levels to pass):
+#   0   nothing committed
+#   10  required files all present
+#   20  pipeline runs without crashing
+#   40  output/error_report.json is valid + weather.db has rows
+#   50  pipeline is idempotent (same row count on second run)
+#   70  code uses required patterns (@field_validator, parameterized queries,
+#       ON CONFLICT upsert, time.sleep backoff)
+task16=0
+task16_msg="missing required files in task-1/"
+
+required_files=(
+    "task-1/models.py"
+    "task-1/ingest_api.py"
+    "task-1/ingest_files.py"
+    "task-1/validate.py"
+    "task-1/database.py"
+    "task-1/pipeline.py"
+    "task-1/.env.example"
+)
+
+all_present=true
+for f in "${required_files[@]}"; do
+    if [ ! -f "$f" ]; then
+        all_present=false
+        break
+    fi
+done
+
+if [ "$all_present" = true ]; then
+    task16=10
+    task16_msg="files exist but pipeline failed to run"
+
+    if [ -f task-1/requirements.txt ]; then
+        python3 -m pip install -q -r task-1/requirements.txt || \
+            echo "WARN: pip install failed; pipeline may crash with ModuleNotFoundError" >&2
+    fi
+
+    PIPELINE_ERR=$(mktemp)
+    if ( cd task-1 && python3 -m pipeline ) >/dev/null 2>"$PIPELINE_ERR"; then
+        task16=20
+        task16_msg="pipeline ran but output checks failed"
+
+        STRUCT_ERR=$(mktemp)
+        if python3 - <<'PY' 2>"$STRUCT_ERR"
+import json, sqlite3
+from pathlib import Path
+
+# error_report.json must exist and be a non-empty list of error objects
+rpt = Path("task-1/output/error_report.json")
+assert rpt.exists(), "output/error_report.json was not created"
+errors = json.loads(rpt.read_text())
+assert isinstance(errors, list), "error_report.json must be a JSON list"
+assert len(errors) > 0, "error_report.json is empty — the CSV has intentional bad rows that should fail validation"
+
+required_fields = {"index", "source", "raw_record", "error_details"}
+for i, e in enumerate(errors[:3]):
+    missing = required_fields - set(e.keys())
+    assert not missing, f"error object {i} missing fields: {missing}"
+
+# weather.db must exist and have rows
+db = Path("task-1/weather.db")
+assert db.exists(), "weather.db was not created"
+conn = sqlite3.connect(db)
+count = conn.execute("SELECT COUNT(*) FROM weather_readings").fetchone()[0]
+assert count > 0, "weather_readings table is empty after pipeline run"
+PY
+        then
+            rm -f "$STRUCT_ERR"
+            task16=40
+            task16_msg="output checks passed; testing idempotency"
+
+            # Idempotency: run a second time, row count must stay the same
+            count_before=$(python3 -c "
+import sqlite3
+conn = sqlite3.connect('task-1/weather.db')
+print(conn.execute('SELECT COUNT(*) FROM weather_readings').fetchone()[0])
+")
+            if ( cd task-1 && python3 -m pipeline ) >/dev/null 2>&1; then
+                count_after=$(python3 -c "
+import sqlite3
+conn = sqlite3.connect('task-1/weather.db')
+print(conn.execute('SELECT COUNT(*) FROM weather_readings').fetchone()[0])
+")
+                if [ "$count_before" = "$count_after" ]; then
+                    task16=50
+                    task16_msg="output + idempotency pass; checking code patterns"
+
+                    # Code introspection for the final 20 points.
+                    # These greps target actual code constructs, not docstrings:
+                    #   @field_validator / @classmethod  — present in scaffold but only
+                    #     safe to match after pipeline passes (NotImplementedError in the
+                    #     validator would crash the pipeline before we get here)
+                    #   execute.*?           — SQL parameterized placeholder in a call
+                    #   ON CONFLICT          — upsert keyword in actual SQL string
+                    #   time\.sleep          — stdlib sleep call (avoids matching the
+                    #     function name "fetch_with_retry" or docstring words)
+                    has_field_validator=$(grep -cE "@field_validator" task-1/models.py || true)
+                    has_classmethod=$(grep -cE "@classmethod" task-1/models.py || true)
+                    has_param_queries=$(grep -cE "execute[a-z]*\(.*\?" task-1/database.py || true)
+                    has_on_conflict=$(grep -ciE "ON CONFLICT" task-1/database.py || true)
+                    has_sleep=$(grep -cE "time\.sleep" task-1/ingest_api.py || true)
+
+                    if [ "$has_field_validator" -gt 0 ] && \
+                       [ "$has_classmethod" -gt 0 ] && \
+                       [ "$has_param_queries" -gt 0 ] && \
+                       [ "$has_on_conflict" -gt 0 ] && \
+                       [ "$has_sleep" -gt 0 ]; then
+                        task16=70
+                        task16_msg="pipeline + idempotency + code patterns all pass"
+                    else
+                        missing_patterns=()
+                        [ "$has_field_validator" -eq 0 ] && missing_patterns+=("@field_validator in models.py")
+                        [ "$has_classmethod" -eq 0 ] && missing_patterns+=("@classmethod in models.py")
+                        [ "$has_param_queries" -eq 0 ] && missing_patterns+=("parameterized ? in execute() in database.py")
+                        [ "$has_on_conflict" -eq 0 ] && missing_patterns+=("ON CONFLICT in database.py")
+                        [ "$has_sleep" -eq 0 ] && missing_patterns+=("time.sleep backoff in ingest_api.py")
+                        task16_msg="output passes but code missing: $(IFS=, ; echo "${missing_patterns[*]}")"
+                    fi
+                else
+                    task16_msg="second run changed row count ($count_before -> $count_after); upsert not working"
+                fi
+            fi
+        else
+            err=$(tail -3 "$STRUCT_ERR" | tr '\n' ' ' | sed 's/  */ /g' | sed 's/^ //;s/ $//')
+            [ -n "$err" ] && task16_msg="output check failed: $err"
+            rm -f "$STRUCT_ERR"
+        fi
+    else
+        err=$(tail -3 "$PIPELINE_ERR" | tr '\n' ' ' | sed 's/  */ /g' | sed 's/^ //;s/ $//')
+        [ -n "$err" ] && task16_msg="pipeline failed to run: $err"
+    fi
+    rm -f "$PIPELINE_ERR"
+fi
+
+# --- Task 7: Azure CLI + Portal (15 points) ---
+#
+#  5 pts  azure_resource_groups.json exists and is valid JSON
+# 10 pts  azure_compare.md exists
+# 15 pts  azure_compare.md has all 3 sections and is filled in (>1200 chars,
+#         which is above the committed template's ~310 chars of non-comment text)
+task7=0
+task7_msg="missing task-1/output/azure_resource_groups.json"
+
+if [ -s "task-1/output/azure_resource_groups.json" ]; then
+    if python3 -c "import json; json.load(open('task-1/output/azure_resource_groups.json'))" 2>/dev/null; then
+        task7=5
+        task7_msg="azure_resource_groups.json is valid JSON; azure_compare.md missing or not filled in"
+
+        if [ -s "task-1/output/azure_compare.md" ]; then
+            task7=10
+            task7_msg="azure_compare.md exists but looks too short or missing sections"
+            section_count=$(grep -cE "^## " task-1/output/azure_compare.md || true)
+            char_count=$(wc -c < task-1/output/azure_compare.md)
+            if [ "$section_count" -ge 3 ] && [ "$char_count" -gt 1200 ]; then
+                task7=15
+                task7_msg="azure_resource_groups.json and azure_compare.md both present and filled in"
+            fi
+        fi
+    else
+        task7_msg="task-1/output/azure_resource_groups.json is not valid JSON"
+    fi
+fi
+
+# --- Task 8: AI Debug Report (15 points) ---
+#
+#  5 pts  task-2/AI_DEBUG.md exists
+# 10 pts  all four sections present (## The Error, ## The Prompt, ## The Solution, ## Reflection)
+# 15 pts  file is meaningfully filled in (>1800 chars)
+task8=0
+task8_msg="missing task-2/AI_DEBUG.md"
+
+if [ -s "task-2/AI_DEBUG.md" ]; then
+    task8=5
+    task8_msg="AI_DEBUG.md exists but missing required sections"
+    if grep -q "^## The Error" task-2/AI_DEBUG.md && \
+       grep -q "^## The Prompt" task-2/AI_DEBUG.md && \
+       grep -q "^## The Solution" task-2/AI_DEBUG.md && \
+       grep -q "^## Reflection" task-2/AI_DEBUG.md; then
+        task8=10
+        task8_msg="all sections present but file looks too short to be filled in"
+        if [ "$(wc -c < task-2/AI_DEBUG.md)" -gt 1800 ]; then
+            task8=15
+            task8_msg="AI_DEBUG.md is filled in"
+        fi
+    fi
+fi
+
+score=$((task16 + task7 + task8))
+if [ "$score" -ge "$PASSING" ]; then pass=true; else pass=false; fi
+
+cat > "$SCRIPT_DIR/score.json" <<EOF
 {
-  "score": 0,
-  "pass": true,
-  "passingScore": 0
+  "score": $score,
+  "pass": $pass,
+  "passingScore": $PASSING
 }
 EOF
+
+echo "Tasks 1-6 (Ingestion Pipeline): $task16/70 — $task16_msg"
+echo "Task 7  (Azure CLI + Portal):   $task7/15  — $task7_msg"
+echo "Task 8  (AI Debug Report):      $task8/15  — $task8_msg"
+echo "----------------------------------------"
+echo "Total: $score/100 — pass=$pass (passing threshold: $PASSING)"

AZURE_LOGIN.md

@@ -0,0 +1,55 @@
+# Logging in to Azure
+
+Task 7 of the Week 3 assignment requires access to the HackYourFuture Azure tenant to call the Azure Resource Manager API.
+
+## In GitHub Codespaces
+
+The Azure CLI is pre-installed in this Codespace. You only need to authenticate.
+
+Run the following command in the Codespace terminal:
+
+```bash
+az login --use-device-code
+```
+
+You will see output like:
+
+```text
+To sign in, use a web browser to open the page https://microsoft.com/devicelogin
+and enter the code XXXXXXXX to authenticate.
+```
+
+1. Open [microsoft.com/devicelogin](https://microsoft.com/devicelogin) in your browser.
+2. Enter the code shown in the terminal.
+3. Sign in with the HackYourFuture credentials your teacher provided (format: `firstname.lastname@hackyourfuture.net` or similar).
+4. Return to the Codespace terminal. You should see your account details printed.
+
+Verify the login succeeded:
+
+```bash
+az account show --query "{user:user.name, subscription:name, tenant:homeTenantId}" --output table
+```
+
+The `user` field should show your HackYourFuture account email.
+
+## On your local machine
+
+If you are working locally and the Azure CLI is already installed from a previous week, run:
+
+```bash
+az login
+```
+
+This opens a browser window for you to sign in. After signing in, return to the terminal and verify with the command above.
+
+If `az` is not installed, follow the [official installation guide](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) for your OS, then run `az login`.
+
+## Troubleshooting
+
+**"The subscription is not in the HackYourFuture tenant"**: You are logged in with a personal or work account. Run `az logout`, then `az login --use-device-code` again and use the HYF credentials.
+
+**"Authorization failed"**: Your account may not have the Reader role on the subscription. Contact your teacher.
+
+**Code expired**: The device login code is valid for 15 minutes. If it expires, re-run `az login --use-device-code` to get a new one.
+
+**Token expired during the Python step**: Azure tokens are valid for about one hour. If you get a 401 error, re-run `az account get-access-token --query accessToken -o tsv` to get a fresh token.