From 4ee34abc65e20b282c2b6f9b533abf9e4232229c Mon Sep 17 00:00:00 2001 From: Muna Nasher Date: Wed, 3 Jun 2026 15:30:33 +0200 Subject: [PATCH 1/4] add CI workflow --- Dockerfile | 23 +++++------------------ src/pipeline.py | 43 ++++++++++++++++++++++++++++++------------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index d665b11..cd49d19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,11 @@ -# Task 4: Write a cache-friendly Dockerfile. -# -# Requirements (in order): -# 1. Use python:3.11-slim as the base image. -# 2. Copy requirements.txt BEFORE copying source code. -# 3. Install dependencies from requirements.txt. -# 4. Copy src/ into the image. -# 5. Set the CMD to run the pipeline: python -m src.pipeline -# -# Replace each TODO comment with the correct Dockerfile instruction. - -# TODO: set the base image -FROM TODO +FROM python:3.11-slim WORKDIR /app -# TODO: copy requirements.txt (before source — this keeps the install layer cached) +COPY requirements.txt . -# TODO: install dependencies +RUN pip install --no-cache-dir -r requirements.txt -# TODO: copy source code +COPY src/ src/ -# TODO: set the command that runs when the container starts -CMD ["TODO"] +CMD ["python", "-m", "src.pipeline"] \ No newline at end of file diff --git a/src/pipeline.py b/src/pipeline.py index 1cd17a4..3c90498 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -9,6 +9,7 @@ """ import logging +import os from pathlib import Path logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") @@ -21,40 +22,56 @@ def get_config() -> dict: Required variable: API_KEY Optional variable: OUTPUT_DIR (default "output") - - Raise RuntimeError with a clear message if a required variable is missing. """ - raise NotImplementedError("Task 5: read API_KEY and OUTPUT_DIR from the environment") + api_key = os.getenv("API_KEY") + + if not api_key: + raise RuntimeError("API_KEY is missing") + + output_dir = os.getenv("OUTPUT_DIR", "output") + + return { + "api_key": api_key, + "output_dir": output_dir, + } def fetch_data(api_key: str) -> list[dict]: """ Simulate fetching records from an external API. - - Return a list of at least one dict representing a record. - In a real pipeline you would call requests.get(...) here. """ - raise NotImplementedError("Task 1: return at least one sample record") + return [ + {"id": 1, "name": "Muna", "city": "Amsterdam"}, + {"id": 2, "name": "Ali", "city": "Rotterdam"}, + ] def save_results(records: list[dict], output_dir: Path) -> None: """ - Write each record as a line to output_dir/results.txt. - - Create output_dir if it does not exist. - Log the number of records written. + Write each record as a line to output_dir/results.txt """ - raise NotImplementedError("Task 1: write records to output_dir/results.txt") + output_dir.mkdir(parents=True, exist_ok=True) + + file_path = output_dir / "results.txt" + + with open(file_path, "w", encoding="utf-8") as f: + for record in records: + f.write(f"{record}\n") + + logger.info("%s records written", len(records)) def run() -> None: config = get_config() logger.info("starting pipeline") + records = fetch_data(config["api_key"]) output_dir = Path(config["output_dir"]) + save_results(records, output_dir) + logger.info("pipeline complete") if __name__ == "__main__": - run() + run() \ No newline at end of file From d6ab8c9725b1d575c504ae6cf136cdcc53ab3de5 Mon Sep 17 00:00:00 2001 From: Muna Nasher Date: Wed, 3 Jun 2026 15:48:30 +0200 Subject: [PATCH 2/4] fix dockerfile for CI build --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cd49d19..4d7cbef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,8 @@ FROM python:3.11-slim WORKDIR /app COPY requirements.txt . - RUN pip install --no-cache-dir -r requirements.txt -COPY src/ src/ +COPY . . CMD ["python", "-m", "src.pipeline"] \ No newline at end of file From 6a8f6efa03be2c5b571ca0c32db7598cf54c7789 Mon Sep 17 00:00:00 2001 From: Muna Nasher Date: Wed, 3 Jun 2026 15:59:20 +0200 Subject: [PATCH 3/4] fix CI docker build issue --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 42299cf..27ab112 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,6 @@ # ruff== # # Add your pinned dependencies below: + +pytest==8.4.0 +ruff==0.12.0 \ No newline at end of file From 2dcd2372daab7276f9a3593e6b961b5f033dafeb Mon Sep 17 00:00:00 2001 From: Muna Nasher Date: Wed, 3 Jun 2026 16:57:35 +0200 Subject: [PATCH 4/4] fix pipeline task 1 completed --- src/pipeline.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/src/pipeline.py b/src/pipeline.py index 3c90498..473b18e 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -1,13 +1,3 @@ -""" -Week 5 assignment: containerised data pipeline. - -Tasks: -- Task 1: confirm this script runs locally before touching the Dockerfile. -- Task 5: read all configuration from environment variables (no hardcoded values). - -Replace every `raise NotImplementedError` below with a real implementation. -""" - import logging import os from pathlib import Path @@ -17,14 +7,7 @@ def get_config() -> dict: - """ - Return configuration read from environment variables. - - Required variable: API_KEY - Optional variable: OUTPUT_DIR (default "output") - """ api_key = os.getenv("API_KEY") - if not api_key: raise RuntimeError("API_KEY is missing") @@ -37,9 +20,6 @@ def get_config() -> dict: def fetch_data(api_key: str) -> list[dict]: - """ - Simulate fetching records from an external API. - """ return [ {"id": 1, "name": "Muna", "city": "Amsterdam"}, {"id": 2, "name": "Ali", "city": "Rotterdam"}, @@ -47,9 +27,6 @@ def fetch_data(api_key: str) -> list[dict]: def save_results(records: list[dict], output_dir: Path) -> None: - """ - Write each record as a line to output_dir/results.txt - """ output_dir.mkdir(parents=True, exist_ok=True) file_path = output_dir / "results.txt" @@ -63,9 +40,11 @@ def save_results(records: list[dict], output_dir: Path) -> None: def run() -> None: config = get_config() + logger.info("starting pipeline") records = fetch_data(config["api_key"]) + output_dir = Path(config["output_dir"]) save_results(records, output_dir)