feat: add Dockerized ETL ingestion stack

.gitignore

@@ -3,4 +3,6 @@ __pycache__/
 .venv/
 .env
 .DS_Store
-Thumbs.db
+Thumbs.db
+etl.log
+.coverage

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libpq-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY src/ ./src
+COPY sql/ ./sql
+COPY config/ ./config
+COPY data_sources/ ./data_sources
+
+ENV PYTHONPATH=/app/src
+
+CMD ["python", "-m", "src.main"]

config/sources.yml

@@ -1,5 +1,5 @@
 defaults:
-  db_url: DATABASE_URL
+  db_url_env: DATABASE_URL
   batch_size: 500
   on_conflict: upsert
 
@@ -98,7 +98,7 @@ sources:
 
   - name: noaa_monthly_sample
     type: csv
-    path: sample_data/small_sample.csv
+    path: data_sources/small_sample.csv
     target_table: stg_noaa_monthly
     pk: [station, date]
 
@@ -120,7 +120,7 @@ sources:
   - name: open_meteo_sample
     type: json
     json_root: hourly
-    path: sample_data/open_meteo_sample.json
+    path: data_sources/open_meteo_sample.json
     target_table: stg_open_meteo
     pk: [latitude, longitude, time]
 

dev-requirements.txt

@@ -0,0 +1,16 @@
+-r requirements.txt
+
+pytest==9.0.3
+pytest-cov==7.1.0
+coverage==7.14.1
+
+black==26.5.1
+ruff==0.15.14
+
+pre_commit==4.6.0
+
+debugpy==1.8.21
+
+ipykernel==7.3.0
+jupyterlab==4.5.8
+notebook==7.5.7

docker-compose.yml

@@ -0,0 +1,76 @@
+services:
+
+  postgres:
+    image: postgres:16
+    container_name: weather_postgres
+
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+
+    ports:
+      - "5432:5432"
+
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+      - ./sql/init.sql:/docker-entrypoint-initdb.d/init.sql
+
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+
+  ingestion_app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+
+    container_name: weather_ingestion
+
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+    env_file:
+      - .env
+
+    environment:
+      RUNNING_IN_DOCKER: "true"
+      CONFIG_PATH: config/sources.yml
+
+    working_dir: /app
+
+    command: python -m src.main
+
+
+  pgadmin:
+    image: dpage/pgadmin4
+
+    container_name: weather_admin
+
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@example.com
+      PGADMIN_DEFAULT_PASSWORD: admin
+
+      PGADMIN_CONFIG_SERVER_MODE: "False"
+      PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: "False"
+
+      PGADMIN_CONFIG_ENHANCED_COOKIE_PROTECTION: "False"
+      PGADMIN_CONFIG_LOGIN_BANNER: '"Weather ingestion local"'
+
+    ports:
+      - "8080:80"
+
+    depends_on:
+      - postgres
+
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
+      - ./docker/pgadmin/servers.json:/pgadmin4/servers.json
+
+volumes:
+  postgres_data:
+  pgadmin_data:

docker/pgadmin/servers.json

@@ -0,0 +1,13 @@
+{
+  "Servers": {
+    "1": {
+      "Name": "Weather DB",
+      "Group": "Servers",
+      "Host": "postgres",
+      "Port": 5432,
+      "MaintenanceDB": "weather_ingestion",
+      "Username": "weather_user",
+      "SSLMode": "prefer"
+    }
+  }
+}

requirements.txt

@@ -1,40 +1,7 @@
-annotated-types==0.7.0
-black==26.5.1
-certifi==2026.5.20
-cfgv==3.5.0
-charset-normalizer==3.4.7
-click==8.4.1
-colorama==0.4.6
-coverage==7.14.1
-distlib==0.4.0
-filelock==3.29.0
-greenlet==3.5.1
-identify==2.6.19
-idna==3.16
-iniconfig==2.3.0
-mypy_extensions==1.1.0
-nodeenv==1.10.0
-numpy==2.4.4
-packaging==26.2
 pandas==3.0.2
-pathspec==1.1.1
-platformdirs==4.10.0
-pluggy==1.6.0
-pre_commit==4.6.0
+numpy==2.4.4
+requests==2.34.2
 psycopg==3.3.4
 psycopg-binary==3.3.4
-pydantic_core==2.46.4
-Pygments==2.20.0
-pytest==9.0.3
-pytest-cov==7.1.0
-python-dateutil==2.9.0.post0
-python-discovery==1.4.0
-python-dotenv==1.2.2
-pytokens==0.4.1
 PyYAML==6.0.3
-ruff==0.15.14
-six==1.17.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-tzdata==2026.2
-urllib3==2.7.0
+python-dotenv==1.2.2

src/cleaners/cleaners.py

@@ -13,7 +13,14 @@ def normalize_columns(df):
     df.columns = df.columns.str.strip()
     df.columns = df.columns.str.lower()
     df.columns = df.columns.str.replace(" ", "_")
-    logger.info(f"Normalized columns: {original_data} to: {df.columns.tolist()}")
+
+    logger.info(
+        "Normalized columns: %d → %d (added=%d removed=%d)",
+        len(original_data),
+        len(df.columns),
+        len(set(df.columns) - set(original_data)),
+        len(set(original_data) - set(df.columns)),
+    )
 
     return df
 

src/database/database.py

@@ -1,6 +1,8 @@
 import os
 import psycopg
 import logging
+from pathlib import Path
+from utils.utils import project_path
 
 logger = logging.getLogger(__name__)
 
@@ -12,7 +14,7 @@ def database_setup(defaults):
     Returns the connection
     """
 
-    db_url = os.getenv("DATABASE_URL")
+    db_url = os.getenv(defaults["db_url_env"])
 
     if not db_url:
         raise ValueError(f"DATABASE_URL env value not set or invalid: {db_url}")
@@ -40,10 +42,15 @@ def database_setup(defaults):
     return conn
 
 
-def init_sql(conn, path="../sql/init.sql"):
+def init_sql(conn, path=None):
     """
     Reads and initializes the sql database from the init.sql file
     """
+    if path is None:
+        path = project_path("sql", "init.sql")
+    else:
+        path = Path(path)
+
     try:
         with open(path, "r", encoding="utf-8") as file:
             sql = file.read()

src/loggers/logging_config.py

@@ -18,7 +18,7 @@ def log_source_start(source_name):
 def log_source_complete(source_name, rows_loaded, rejects):
     logger.info(
         "Completed source '%s' (rows_loaded=%s rejects=%s)",
-        source_name,
+        str(source_name),
         rows_loaded,
         rejects,
     )

src/main.py

@@ -27,6 +27,8 @@
     log_source_complete,
 )
 
+from utils.utils import project_path
+
 
 def run_source(connection, source, defaults):
     """
@@ -54,13 +56,13 @@ def run_source(connection, source, defaults):
         batch_size=defaults["batch_size"],
     )
     write_rejects(connection, rejects, defaults["batch_size"])
-    log_source_complete(str(source), len(df), len(rejects))
+    log_source_complete(str(source["name"]), len(df), len(rejects))
 
 
 def main():
     load_dotenv()
 
-    path: str = "../config/sources.yml"
+    path = project_path("config", "sources.yml")
     config = load_config(path)
 
     defaults = config["defaults"]

src/readers/readers.py

@@ -1,11 +1,11 @@
 import json
-import os
 import pandas as pd
 import requests
 
 from io import StringIO
 
 from loggers.logging_config import logger
+from utils.utils import project_path
 
 
 def read_input(source):
@@ -31,7 +31,7 @@ def read_csv(source):
     Reads a .csv file based on the path provided in the config
     Returns a pandas dataframe
     """
-    path = os.path.join("..", source["path"])
+    path = project_path(source["path"])
     logger.info(f"CSV file read from: {path}")
 
     try:
@@ -47,7 +47,7 @@ def read_json(source):
     Reads a .json file based on the path provided in the config
     Returns a pandas dataframe
     """
-    path = os.path.join("..", source["path"])
+    path = project_path(source["path"])
 
     # Load the entire json file
     try:

src/utils/utils.py

@@ -1,4 +1,15 @@
 import pandas as pd
+import os
+from pathlib import Path
+
+if os.getenv("RUNNING_IN_DOCKER") == "true":
+    BASE_DIR = Path("/app")
+else:
+    BASE_DIR = Path(__file__).resolve().parents[2]
+
+
+def project_path(*parts):
+    return BASE_DIR.joinpath(*parts)
 
 
 def emit_reject(source_name, reason, row):