diff --git a/.gitignore b/.gitignore index 9e117dc..21451a1 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,6 @@ env/ # Generated pipeline output (committed templates stay; generated files do not) output/error_report.json -output/azure_resource_groups.json weather.db # Editor and IDE settings diff --git a/AI_DEBUG.md b/AI_DEBUG.md index edf34a5..24dff07 100644 --- a/AI_DEBUG.md +++ b/AI_DEBUG.md @@ -7,20 +7,79 @@ Document the debugging session below. If everything worked first try, introduce ## The Error - + ## The Prompt - + ## The Solution - + + +```python +WeatherReading(record) +``` + +to + +```python +WeatherReading(**record) +``` + +The \*\* operator unpacks the dictionary into named keyword arguments that match the Pydantic model fields. + +After making the change, the validation step worked correctly. ## Reflection - + + +After the explanation, I understood that: + +```python + +WeatherReading(record) +``` + +passes the whole dictionary as one positional argument, while: + +```python +WeatherReading(**record) +``` + +expands the dictionary into separate keyword arguments. + +Next time I work with dictionaries and Pydantic models, I will remember to use \*\* unpacking when passing dictionary data into models or functions that expect keyword arguments. diff --git a/azure_fetch.py b/azure_fetch.py new file mode 100644 index 0000000..15e1e1e --- /dev/null +++ b/azure_fetch.py @@ -0,0 +1,28 @@ +import json +import requests +import os +from dotenv import load_dotenv + +load_dotenv() + +token = os.getenv("AZURE_TOKEN") +subscription_id = os.getenv("AZURE_SUBSCRIPTION_ID") + +url = f"https://management.azure.com/subscriptions/{subscription_id}/resourcegroups?api-version=2024-03-01" + +headers = { + "Authorization": f"Bearer {token}" +} + +response = requests.get(url, headers=headers, timeout=10) +response.raise_for_status() + + + +# Save full response to JSON file +with open("output/azure_resource_groups.json", "w", encoding="utf-8") as file: + json.dump(response.json(), file, indent=2) + +# Print resource groups +for rg in response.json()["value"]: + print(f"{rg['name']}: {rg['location']}") \ No newline at end of file diff --git a/database.py b/database.py index 8322d17..8469bc5 100644 --- a/database.py +++ b/database.py @@ -24,8 +24,31 @@ def create_tables(conn: sqlite3.Connection) -> None: weather_readings columns: id, station, timestamp, temperature_c, humidity_pct + UNIQUE(station, timestamp) constraint for upserts """ - # TODO: use conn.execute() with CREATE TABLE IF NOT EXISTS statements - raise NotImplementedError + + conn.execute(""" + CREATE TABLE IF NOT EXISTS raw_weather ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + station TEXT NOT NULL, + timestamp TEXT NOT NULL, + temperature_c REAL, + humidity_pct INTEGER, + source TEXT, + ingested_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS weather_readings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + station TEXT, + timestamp TEXT, + temperature_c REAL, + humidity_pct INTEGER, + UNIQUE(station, timestamp) + ) + """) + + conn.commit() def insert_raw(conn: sqlite3.Connection, records: list[dict], source: str) -> None: @@ -33,8 +56,23 @@ def insert_raw(conn: sqlite3.Connection, records: list[dict], source: str) -> No Use parameterized queries with placeholder syntax; do not build SQL via string formatting. """ - # TODO: implement - raise NotImplementedError + + with conn: + conn.executemany(""" + INSERT INTO raw_weather (station, timestamp, temperature_c, humidity_pct, source) + VALUES (?, ?, ?, ?, ?) + """, [ + ( + record.get("station"), + record.get("timestamp"), + record.get("temperature_c"), + record.get("humidity_pct"), + source + ) + for record in records + ]) + + def upsert_readings(conn: sqlite3.Connection, readings: list[WeatherReading]) -> None: @@ -43,11 +81,27 @@ def upsert_readings(conn: sqlite3.Connection, readings: list[WeatherReading]) -> Use the upsert pattern to handle duplicate (station, timestamp) pairs. Use parameterized queries. """ - # TODO: implement - raise NotImplementedError + + with conn: + conn.executemany(""" + INSERT INTO weather_readings (station, timestamp, temperature_c, humidity_pct) + VALUES (?, ?, ?, ?) + ON CONFLICT(station, timestamp) DO UPDATE SET + temperature_c=excluded.temperature_c, + humidity_pct=excluded.humidity_pct + """, [ + ( + reading.station, + reading.timestamp, + reading.temperature_c, + reading.humidity_pct + ) + for reading in readings + ]) def count_readings(conn: sqlite3.Connection) -> int: """Return the total number of rows in weather_readings.""" - # TODO: implement - raise NotImplementedError + + row = conn.execute("SELECT COUNT(*) AS count FROM weather_readings").fetchone() + return row["count"] if row else 0 diff --git a/ingest_api.py b/ingest_api.py index 66eb32c..53e6fa8 100644 --- a/ingest_api.py +++ b/ingest_api.py @@ -3,7 +3,6 @@ # fetch_api_records calls it and shapes the response into flat dicts (Task 2). import logging import time - import requests logger = logging.getLogger(__name__) @@ -18,8 +17,28 @@ def fetch_with_retry(url: str, params: dict, max_retries: int = 3, timeout: int Fail immediately on: 4xx status codes. Log each retry attempt with the error and delay. """ - # TODO: implement retry loop with exponential backoff - raise NotImplementedError + + for attempt in range(max_retries + 1): + try: + response = requests.get(url, params=params, timeout=timeout) + + if 400 <= response.status_code < 500: + response.raise_for_status() + + if response.status_code >= 500: + raise requests.HTTPError(f"Server error: {response.status_code}", response=response) + + return response.json() + except (requests.ConnectionError, requests.Timeout, requests.HTTPError) as e: + if isinstance(e, requests.HTTPError): + response = e.response + if response is not None and 400 <= response.status_code < 500: + raise # Do not retry on client errors + if attempt == max_retries: + raise + delay = 2 ** attempt + logger.warning("Retry %s/%s after error: %s. waiting %s seconds.", attempt + 1, max_retries, e, delay) + time.sleep(delay) def fetch_api_records() -> list[dict]: @@ -38,4 +57,20 @@ def fetch_api_records() -> list[dict]: # - Call fetch_with_retry with API_URL and params # - The API returns {"hourly": {"time": [...], "temperature_2m": [...], "relative_humidity_2m": [...]}} # - Flatten to a list of dicts; set station="Open-Meteo Copenhagen" for all records - raise NotImplementedError + data = fetch_with_retry(API_URL, params) + hourly = data.get("hourly", {}) + times = hourly.get("time", []) + temperatures = hourly.get("temperature_2m", []) + humidities = hourly.get("relative_humidity_2m", []) + + if not times: + return [] + records = [] + for timestamp, temp, humidity in zip(times, temperatures, humidities): + records.append({ + "station": "Open-Meteo Copenhagen", + "timestamp": timestamp, + "temperature_c": temp, + "humidity_pct": humidity + }) + return records diff --git a/ingest_files.py b/ingest_files.py index 8b010ff..8803ecd 100644 --- a/ingest_files.py +++ b/ingest_files.py @@ -5,6 +5,21 @@ from pathlib import Path +def convert_float(value: str) -> float: + """Convert value to float, or return original if conversion fails.""" + try: + return float(value) + except (ValueError, TypeError): + return value + + +def convert_int(value: str) -> int: + """Convert value to int, or return original if conversion fails.""" + try: + return int(value) + except (ValueError, TypeError): + return value + def read_csv_records(path: Path) -> list[dict]: """Read weather_stations.csv and return normalized records. @@ -16,5 +31,16 @@ def read_csv_records(path: Path) -> list[dict]: - Convert temperature_c to float and humidity_pct to int where possible. - Leave unconvertible values (e.g. "N/A", "") as-is so validation can catch them. """ - # TODO: implement CSV reading and normalization - raise NotImplementedError + records = [] + + with path.open(newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + records.append({ + "station": row.get("station", ""), + "timestamp": row.get("timestamp", ""), + "temperature_c": convert_float(row.get("temperature_c", "")), + "humidity_pct": convert_int(row.get("humidity_pct", "")), + }) + + return records diff --git a/models.py b/models.py index cae42fe..cdd34d4 100644 --- a/models.py +++ b/models.py @@ -13,5 +13,4 @@ class WeatherReading(BaseModel): @field_validator("station") @classmethod def clean_station(cls, v: str) -> str: - # TODO: strip whitespace and convert to title case - raise NotImplementedError + return v.strip().title() diff --git a/output/azure_compare.md b/output/azure_compare.md index 22395b3..d52d165 100644 --- a/output/azure_compare.md +++ b/output/azure_compare.md @@ -4,12 +4,12 @@ Fill in each section below (2-3 sentences each) after completing the Task 7 step ## Auth - + ## Schema verbosity - + ## api-version in the URL - + diff --git a/output/azure_resource_groups.json b/output/azure_resource_groups.json new file mode 100644 index 0000000..53f3055 --- /dev/null +++ b/output/azure_resource_groups.json @@ -0,0 +1,13 @@ +{ + "value": [ + { + "id": "/subscriptions/1120c89d-2a5f-4a15-a582-2ea34f0bb5c3/resourceGroups/rg-hyf-students-readonly", + "name": "rg-hyf-students-readonly", + "type": "Microsoft.Resources/resourceGroups", + "location": "westeurope", + "properties": { + "provisioningState": "Succeeded" + } + } + ] +} diff --git a/pipeline.py b/pipeline.py index d8dc710..fbc3f44 100644 --- a/pipeline.py +++ b/pipeline.py @@ -17,33 +17,40 @@ def run_pipeline() -> None: OUTPUT_DIR.mkdir(exist_ok=True) + api_records = fetch_api_records() + csv_records = read_csv_records(CSV_PATH) - # TODO — implement each step in order: - # - # 1. Fetch records from Open-Meteo API using fetch_api_records() - # 2. Read records from CSV using read_csv_records(CSV_PATH) - # 3. Open a DB connection, create tables, insert all raw records (both sources) - # 4. Validate all records — collect valid WeatherReading objects and error dicts - # 5. Upsert valid records into weather_readings - # 6. Save error dicts as JSON to output/error_report.json - # 7. Print the pipeline summary in the format below. - # - # Note: the API count varies by time of day (Open-Meteo returns up to 168 hourly - # records for 7 forecast days; the exact number depends on the current UTC hour). - # The CSV contributes 6 invalid records and 4 valid ones; the duplicate Copenhagen - # row is valid and exercises the upsert path rather than the validation error path. - # Your actual output will look similar to this example: - # - # === Pipeline Summary === - # API records fetched: 166 - # CSV records read: 10 - # Total raw records: 176 - # Valid records: 170 - # Invalid records: 6 - # Records in database: 169 - # Error report: output/error_report.json - - raise NotImplementedError + with get_connection() as conn: + create_tables(conn) + + + insert_raw(conn, api_records, "api") + insert_raw(conn, csv_records, "csv") + + valid_api, error_api = validate_records(api_records, "api") + valid_csv, error_csv = validate_records(csv_records, "csv") + + valid_records = valid_api + valid_csv + error_records = error_api + error_csv + + upsert_readings(conn, valid_records) + + db_count = count_readings(conn) + + error_report_path = OUTPUT_DIR / "error_report.json" + with error_report_path.open("w", encoding="utf-8") as f: + json.dump(error_records, f, indent=2) + print("=== Pipeline Summary ===") + print(f"API records fetched: {len(api_records)}") + print(f"CSV records read: {len(csv_records)}") + print(f"Total raw records: {len(api_records) + len(csv_records)}") + print(f"Valid records: {len(valid_records)}") + print(f"Invalid records: {len(error_records)}") + print(f"Records in database: {db_count}") + print(f"Error report: {error_report_path}") + + + if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 82f9509..6f097c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pydantic>=2.0 requests>=2.28 +python-dotenv>=1.0 diff --git a/validate.py b/validate.py index ada2716..7ce89df 100644 --- a/validate.py +++ b/validate.py @@ -18,5 +18,16 @@ def validate_records( raw_record - the original dict error_details - the Pydantic error list (ValidationError.errors()) """ - # TODO: iterate over records, try WeatherReading(**record), accumulate results - raise NotImplementedError + valid = [] + errors = [] + for index, record in enumerate(records): + try: + valid.append(WeatherReading(**record)) + except ValidationError as e: + errors.append({ + "index": index, + "source": source, + "raw_record": record, + "error_details": e.errors(), + }) + return valid, errors