Migration

.dockerignore

@@ -0,0 +1,13 @@
+.git
+.github
+.Rproj.user
+.Rhistory
+.RData
+*.Rproj
+a4d-python/.pytest_cache
+a4d-python/.ruff_cache
+a4d-python/htmlcov
+a4d-python/.coverage
+a4d-python/profiling/*.prof
+data/
+secrets/

.github/workflows/python-ci.yml

@@ -0,0 +1,52 @@
+name: Python CI
+
+on:
+  push:
+    branches: [migration]
+    paths:
+      - 'a4d-python/**'
+      - '.github/workflows/python-ci.yml'
+  pull_request:
+    branches: [main, develop, migration]
+    paths:
+      - 'a4d-python/**'
+      - '.github/workflows/python-ci.yml'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: a4d-python
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v2
+      with:
+        enable-cache: true
+
+    - name: Set up Python
+      run: uv python install 3.14
+
+    - name: Install dependencies
+      run: uv sync --all-extras
+
+    - name: Run ruff linting
+      run: uv run ruff check .
+
+    - name: Run ruff formatting check
+      run: uv run ruff format --check .
+
+    - name: Run type checking with ty
+      run: uv run ty check src/
+
+    - name: Run tests
+      run: uv run pytest -m "not slow and not integration" --cov --cov-report=xml
+
+    - name: Upload coverage
+      uses: codecov/codecov-action@v3
+      with:
+        files: ./a4d-python/coverage.xml
+        flags: python

.gitignore

@@ -10,4 +10,10 @@
 rsconnect
 
 data/output
-data/mapping_table.csv
+data/mapping_table.csv
+
+# Serena (MCP server state)
+.serena/
+
+# Secrets (GCP service accounts, etc.)
+secrets/

.vscode/settings.json

@@ -0,0 +1,29 @@
+{
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false,
+    "python.testing.cwd": "${workspaceFolder}/a4d-python",
+    "python.testing.pytestArgs": [
+        "${workspaceFolder}/a4d-python/tests"
+    ],
+    "python.defaultInterpreterPath": "${workspaceFolder}/a4d-python/.venv/bin/python",
+    "workbench.colorCustomizations": {
+        "activityBar.activeBackground": "#ab307e",
+        "activityBar.background": "#ab307e",
+        "activityBar.foreground": "#e7e7e7",
+        "activityBar.inactiveForeground": "#e7e7e799",
+        "activityBarBadge.background": "#25320e",
+        "activityBarBadge.foreground": "#e7e7e7",
+        "commandCenter.border": "#e7e7e799",
+        "sash.hoverBorder": "#ab307e",
+        "statusBar.background": "#832561",
+        "statusBar.foreground": "#e7e7e7",
+        "statusBarItem.hoverBackground": "#ab307e",
+        "statusBarItem.remoteBackground": "#832561",
+        "statusBarItem.remoteForeground": "#e7e7e7",
+        "titleBar.activeBackground": "#832561",
+        "titleBar.activeForeground": "#e7e7e7",
+        "titleBar.inactiveBackground": "#83256199",
+        "titleBar.inactiveForeground": "#e7e7e799"
+    },
+    "peacock.color": "#832561"
+}

CLAUDE.md

@@ -0,0 +1,61 @@
+# CLAUDE.md
+
+This repository contains **two projects**:
+
+## 1. R Pipeline (Production - Legacy)
+
+**Location**: Root directory
+**Status**: Production (being phased out)
+
+The original R implementation of the A4D medical tracker data processing pipeline.
+
+**Key Files**:
+- `R/` - R package code
+- `scripts/R/` - Pipeline scripts
+- `reference_data/` - Shared YAML configurations
+
+**Commands**: See README.md for R-specific commands
+
+---
+
+## 2. Python Pipeline (Active Development)
+
+**Location**: `a4d-python/`
+**Status**: Active migration
+**Branch**: `migration`
+
+New Python implementation with better performance and incremental processing.
+
+**Documentation**: [a4d-python/docs/CLAUDE.md](a4d-python/docs/CLAUDE.md)
+
+**Quick Start**:
+```bash
+cd a4d-python
+uv sync
+uv run pytest
+```
+
+**Migration Guide**: [a4d-python/docs/migration/MIGRATION_GUIDE.md](a4d-python/docs/migration/MIGRATION_GUIDE.md)
+
+---
+
+## Working on This Repository
+
+**If working on R code**: Stay in root, use R commands
+
+**If working on Python migration**:
+```bash
+cd a4d-python
+# See a4d-python/docs/CLAUDE.md for Python-specific guidance
+```
+
+## Shared Resources
+
+Both projects use the same reference data:
+- `reference_data/synonyms/` - Column name mappings
+- `reference_data/data_cleaning.yaml` - Validation rules
+- `reference_data/provinces/` - Allowed provinces
+
+**Do not modify these** without testing both R and Python pipelines.
+- Always check your implementation against the original R pipeline and check if the logic is the same
+- Limit comments to explain why a desigin was made or give important context information for the migration but do not use comments for obvious code otherwise

R/script2_helper_patient_data_fix.R

@@ -176,6 +176,15 @@ parse_dates <- function(date) {
         return(lubridate::NA_Date_)
     }
 
+    # Handle Excel serial numbers (e.g., "45341.0", "39920.0")
+    # Excel stores dates as days since 1899-12-30
+    numeric_date <- suppressWarnings(as.numeric(date))
+    if (!is.na(numeric_date) && numeric_date > 1 && numeric_date < 100000) {
+        # This is likely an Excel serial number
+        excel_origin <- as.Date("1899-12-30")
+        return(excel_origin + as.integer(numeric_date))
+    }
+
     parsed_date <- suppressWarnings(lubridate::as_date(date))
 
     if (is.na(parsed_date)) {

a4d-python/.env.example

@@ -0,0 +1,25 @@
+# Environment Configuration
+A4D_ENVIRONMENT=development
+
+# GCP Configuration
+A4D_PROJECT_ID=a4dphase2
+A4D_DATASET=tracker
+A4D_DOWNLOAD_BUCKET=a4dphase2_upload
+A4D_UPLOAD_BUCKET=a4dphase2_output
+
+# GCP Authentication (optional - uses Application Default Credentials if not set)
+# For local development: run `gcloud auth application-default login`
+# For CI/CD or VM: set path to service account key file
+# GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account-key.json
+
+# Paths
+A4D_DATA_ROOT=/path/to/tracker/files
+A4D_OUTPUT_DIR=output
+
+# Processing Settings
+A4D_MAX_WORKERS=4
+
+# Error Values (matching R pipeline)
+A4D_ERROR_VAL_NUMERIC=999999
+A4D_ERROR_VAL_CHARACTER=Undefined
+A4D_ERROR_VAL_DATE=9999-12-31

a4d-python/.gitignore

@@ -0,0 +1,67 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# uv
+.uv/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Environment
+.env
+.env.local
+
+# Logs
+*.log
+logs/
+
+# Data (sensitive)
+data/
+output/
+*.parquet
+*.xlsx
+!reference_data/
+
+# OS
+.DS_Store
+Thumbs.db