diff --git a/.env.example b/.env.example index 42aa982..362c4b6 100644 --- a/.env.example +++ b/.env.example @@ -8,9 +8,9 @@ # echo "POSTGRES_URL=$(az keyvault secret show --vault-name kv-hyf-data --name postgres-url --query value -o tsv)" >> .env # echo "AZURE_STORAGE_CONNECTION_STRING=$(az keyvault secret show --vault-name kv-hyf-data --name storage-connection-string --query value -o tsv)" >> .env -POSTGRES_URL=postgresql://:@.postgres.database.azure.com:5432/?sslmode=require -AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net +export POSTGRES_URL=postgresql://:@.postgres.database.azure.com:5432/?sslmode=require +export AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net # Your personal schema on the shared Postgres server — replace with your GitHub handle. # All tables are created here so they don't collide with other students. -DB_SCHEMA=dev_ -LOG_LEVEL=INFO +export DB_SCHEMA=dev_ +export LOG_LEVEL=INFO diff --git a/README.md b/README.md index d86d31d..aef441f 100644 --- a/README.md +++ b/README.md @@ -42,15 +42,15 @@ uv run pytest tests/ -v ```bash # Build for linux/amd64 (required by Azure Container Apps) and push to ACR -docker build --platform linux/amd64 -t hyfregistry.azurecr.io/my-pipeline:1.0 . -docker push hyfregistry.azurecr.io/my-pipeline:1.0 +docker build --platform linux/amd64 -t hyfregistry.azurecr.io/my-pipeline:latest . +docker push hyfregistry.azurecr.io/my-pipeline:latest # Create Container App Job (runs daily at 06:00 UTC) az containerapp job create \ --name my-pipeline-job \ --resource-group rg-hyf-data \ --environment env-hyf-data \ - --image hyfregistry.azurecr.io/my-pipeline:1.0 \ + --image hyfregistry.azurecr.io/my-pipeline:latest \ --registry-server hyfregistry.azurecr.io \ --trigger-type Schedule \ --cron-expression "0 6 * * *" \ diff --git a/pyproject.toml b/pyproject.toml index cc04ace..287e092 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "azure-storage-blob==12.25.1", "requests==2.32.3", "pandas==2.2.3", + "python-dotenv>=1.0", ] [dependency-groups] diff --git a/src/pipeline.py b/src/pipeline.py index 635112d..0d19b1e 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -4,6 +4,10 @@ import os import sys +from dotenv import load_dotenv + +load_dotenv() + import pandas as pd from pydantic import ValidationError diff --git a/uv.lock b/uv.lock index 50ac0b7..fe43735 100644 --- a/uv.lock +++ b/uv.lock @@ -247,6 +247,7 @@ dependencies = [ { name = "pandas" }, { name = "psycopg2-binary" }, { name = "pydantic" }, + { name = "python-dotenv" }, { name = "requests" }, ] @@ -262,6 +263,7 @@ requires-dist = [ { name = "pandas", specifier = "==2.2.3" }, { name = "psycopg2-binary", specifier = "==2.9.10" }, { name = "pydantic", specifier = "==2.10.6" }, + { name = "python-dotenv", specifier = ">=1.0" }, { name = "requests", specifier = "==2.32.3" }, ] @@ -544,6 +546,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + [[package]] name = "pytz" version = "2026.2"