diff --git a/Dockerfile.unified b/Dockerfile.unified index 57f102c..e2f9fd8 100644 --- a/Dockerfile.unified +++ b/Dockerfile.unified @@ -1,11 +1,12 @@ # Unified Dockerfile for BrainKB Services -# Deploys: APItokenmanager, query_service, ml_service, and oxigraph +# Deploys: APItokenmanager, query_service, ml_service, usermanagement_service, and oxigraph # Note: Oxigraph binary extraction skipped - oxigraph image is distroless # Oxigraph will be run as a separate service in docker-compose or use the fallback script # Main build stage -FROM python:3.10-slim +# Python 3.11+ required: synthscholar (ml_service) depends on it. +FROM python:3.11-slim # Set metadata LABEL project="BrainyPedia" \ @@ -46,12 +47,88 @@ COPY ml_service/ /app/ml_service/ WORKDIR /app/ml_service RUN pip install --use-deprecated=legacy-resolver "structsense==0.0.4" || \ pip install --use-deprecated=legacy-resolver --no-deps "structsense==0.0.4" && \ - pip install -r requirements.txt + pip install --use-deprecated=legacy-resolver -r requirements.txt + +# Copy usermanagement_service +COPY usermanagement_service/ /app/usermanagement_service/ +WORKDIR /app/usermanagement_service +RUN pip install -r requirements.txt # Create supervisor configuration RUN mkdir -p /etc/supervisor/conf.d -COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf +RUN cat > /etc/supervisor/conf.d/supervisord.conf << 'EOF' +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 + +[supervisord] +nodaemon=true +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[program:api_tokenmanager] +command=gunicorn -b 0.0.0.0:8000 APIAuthManager.wsgi:application +directory=/app/APItokenmanager +autostart=true +autorestart=true +startsecs=10 +stderr_logfile=/var/log/supervisor/api_tokenmanager.err.log +stdout_logfile=/var/log/supervisor/api_tokenmanager.out.log +environment=PATH="/usr/local/bin:/usr/bin:/bin" +priority=100 + +[program:query_service] +command=gunicorn core.main:app --bind 0.0.0.0:8010 --workers 6 --worker-class uvicorn.workers.UvicornWorker --threads 2 --timeout 300 --keep-alive 120 --max-requests 1000 --max-requests-jitter 50 +directory=/app/query_service +autostart=true +autorestart=true +startsecs=30 +stderr_logfile=/var/log/supervisor/query_service.err.log +stdout_logfile=/var/log/supervisor/query_service.out.log +environment=PATH="/usr/local/bin:/usr/bin:/bin",WEB_CONCURRENCY="6" +startretries=10 +stopwaitsecs=10 +priority=200 + +[program:ml_service] +command=gunicorn core.main:app --bind 0.0.0.0:8007 --workers 6 --worker-class uvicorn.workers.UvicornWorker --threads 2 --max-requests 1000 --max-requests-jitter 50 --timeout 220 --keep-alive 220 --log-level debug +directory=/app/ml_service +autostart=true +autorestart=true +startsecs=15 +stderr_logfile=/var/log/supervisor/ml_service.err.log +stdout_logfile=/var/log/supervisor/ml_service.out.log +environment=PATH="/usr/local/bin:/usr/bin:/bin",WEB_CONCURRENCY="6" + +[program:usermanagement_service] +command=gunicorn core.main:app --bind 0.0.0.0:8004 --workers 4 --worker-class uvicorn.workers.UvicornWorker --threads 4 --max-requests 1000 --max-requests-jitter 50 --timeout 220 --keep-alive 220 --log-level debug +directory=/app/usermanagement_service +autostart=true +autorestart=true +startsecs=15 +stderr_logfile=/var/log/supervisor/usermanagement_service.err.log +stdout_logfile=/var/log/supervisor/usermanagement_service.out.log +environment=PATH="/usr/local/bin:/usr/bin:/bin" +priority=300 + +[program:oxigraph] +command=/app/oxigraph/oxigraph_server --bind 0.0.0.0:7878 --storage /data +directory=/app/oxigraph +autostart=false +autorestart=false +stderr_logfile=/var/log/supervisor/oxigraph.err.log +stdout_logfile=/var/log/supervisor/oxigraph.out.log +environment=PATH="/usr/local/bin:/usr/bin:/bin",TMPDIR="/tmp" +startsecs=0 +startretries=0 +EOF # Create directories for logs and data # Ensure supervisor socket directory exists and is writable @@ -68,11 +145,85 @@ RUN echo '#!/bin/bash' > /app/oxigraph/oxigraph_server && \ chmod +x /app/oxigraph/oxigraph_server # Create startup script -COPY start.sh /app/start.sh +RUN cat > /app/start.sh << 'EOF' +#!/bin/bash +set -e + +# Note: Oxigraph runs as a separate service in docker-compose + +# Ensure supervisor socket directory exists and is writable +mkdir -p /var/run +chmod 755 /var/run + +# Wait for PostgreSQL to be ready +# Use JWT_POSTGRES_DATABASE_USER if available, otherwise fall back to DB_USER +# This ensures we use the same user that PostgreSQL was initialized with +PG_USER="${JWT_POSTGRES_DATABASE_USER:-${DB_USER:-postgres}}" +PG_PASSWORD="${JWT_POSTGRES_DATABASE_PASSWORD:-${DB_PASSWORD}}" +PG_HOST="${JWT_POSTGRES_DATABASE_HOST_URL:-${DB_HOST:-postgres}}" +PG_DB="${JWT_POSTGRES_DATABASE_NAME:-${DB_NAME:-brainkb}}" + +echo "Waiting for PostgreSQL to be ready..." +echo "Connecting as user: ${PG_USER} to database: ${PG_DB} on host: ${PG_HOST}" +until PGPASSWORD="$PG_PASSWORD" psql -h "$PG_HOST" -U "$PG_USER" -d "$PG_DB" -c '\q' 2>/dev/null; do + echo "PostgreSQL is unavailable - sleeping" + sleep 2 +done +echo "PostgreSQL is ready!" + +# Note: Oxigraph is optional - services will handle connection failures gracefully +# No need to wait for it here - services will retry when needed + +# Run Django migrations for APItokenmanager if needed +cd /app/APItokenmanager +if [ -f .env ] || [ -n "$DB_NAME" ]; then + echo "Running Django migrations..." + python manage.py makemigrations || true + python manage.py migrate || true + python manage.py collectstatic --noinput || true + + # Create superuser if credentials are provided and user doesn't exist + if [ -n "$DJANGO_SUPERUSER_USERNAME" ] && [ -n "$DJANGO_SUPERUSER_EMAIL" ] && [ -n "$DJANGO_SUPERUSER_PASSWORD" ]; then + echo "Creating Django superuser..." + export DJANGO_SUPERUSER_USERNAME DJANGO_SUPERUSER_EMAIL DJANGO_SUPERUSER_PASSWORD + python manage.py shell << 'PYTHON_SCRIPT' || true +import os +from django.contrib.auth import get_user_model +User = get_user_model() +username = os.environ.get('DJANGO_SUPERUSER_USERNAME') +email = os.environ.get('DJANGO_SUPERUSER_EMAIL') +password = os.environ.get('DJANGO_SUPERUSER_PASSWORD') +if username and email and password: + if not User.objects.filter(username=username).exists(): + User.objects.create_superuser(username, email, password) + print(f'Superuser {username} created successfully') + else: + print(f'Superuser {username} already exists') +else: + print('Error: Missing superuser credentials in environment') +PYTHON_SCRIPT + else + echo "Warning: DJANGO_SUPERUSER credentials not provided. Superuser not created." + echo "You can create one manually with: python manage.py createsuperuser" + fi + + echo "Django migrations completed" +fi + +# Ensure supervisor socket directory exists and is writable (in case /var/run is tmpfs) +mkdir -p /var/run +chmod 755 /var/run + +# Start supervisor +echo "Starting all services..." +# Use our config file that includes socket configuration +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf +EOF + RUN chmod +x /app/start.sh # Expose ports -EXPOSE 8000 8007 8010 +EXPOSE 8000 8004 8007 8010 # Set working directory WORKDIR /app diff --git a/README.unified-docker.md b/README.unified-docker.md new file mode 100644 index 0000000..fe9cf69 --- /dev/null +++ b/README.unified-docker.md @@ -0,0 +1,570 @@ +# Unified Docker Deployment + +This Dockerfile deploys the following BrainKB backend services: +- **APItokenmanager** (Django) - Port 8000 +- **query_service** (FastAPI) - Port 8010 +- **ml_service** (FastAPI) - Port 8007 +- **usermanagement_service** (FastAPI) - Port 8004 +- **oxigraph** (SPARQL Database) - Port 7878 + +**Services NOT included in this unified deployment (deploy separately):** +- **brainkb-ui** - The UI is not included. See [SETUP_UI.md](SETUP_UI.md) for UI deployment instructions. +- **chat_service** - Deploy separately using `chat_service/docker-compose-prod.yml` or `chat_service/docker-compose-dev.yml`. + +## Quick Start with Docker Compose (Recommended) + +The easiest way to deploy is using the provided `start_services.sh` wrapper script: + +### 1. Create Environment File + +Create a `.env` file in the project root with your configuration: + +```bash +# Copy the template and edit as needed +cp env.template .env +# Edit .env with your actual values +nano .env # or use your preferred editor +``` + +**Important:** Update at minimum these values in `.env`: +- `JWT_POSTGRES_DATABASE_PASSWORD` - Database password (primary PostgreSQL configuration) +- `DB_PASSWORD` - Should match JWT_POSTGRES_DATABASE_PASSWORD (for Django APItokenmanager) +- `DJANGO_SUPERUSER_PASSWORD` - Admin password for Django +- `BRAINYPEDIA_APITOKEN_MANAGER_SECRET_KEY` - Django secret key +- `*_SERVICE_JWT_SECRET_KEY` - Service-specific JWT signing secrets (one per service) +- `OXIGRAPH_PASSWORD` - Oxigraph authentication password +- `OLLAMA_MODEL` - Ollama model to use (default: nomic-embed-text) +- `NEXTAUTH_SECRET` - NextAuth.js secret for UI +- `NEXT_PUBLIC_*` - UI API endpoint URLs (adjust based on your deployment) + +### 2. Start All Services + +**Recommended: Use the wrapper script (includes Ollama setup and auto-configuration):** +```bash +./start_services.sh +``` + +This will automatically: +- ✅ Detect GPU availability and set up Ollama (CPU or GPU mode) +- ✅ Load the Ollama model specified in `OLLAMA_MODEL` from `.env` +- ✅ Auto-generate pgAdmin servers.json +- ✅ Start PostgreSQL database +- ✅ Build and start the unified BrainKB container +- ✅ Set up networking and volumes +- ✅ Configure all environment variables + +**Alternative: Manual docker-compose (without Ollama auto-setup):** +```bash +docker-compose -f docker-compose.unified.yml up -d +``` + +### 3. Managing Services + +The `start_services.sh` script provides flexible service management: + +#### Start/Stop All Services +```bash +# Start all services +./start_services.sh up -d + +# Stop all services +./start_services.sh down + +# Restart all services +./start_services.sh restart +``` + +#### Control Individual Containers +```bash +# Start/stop specific containers +./start_services.sh brainkb-unified up +./start_services.sh postgres down +./start_services.sh pgadmin restart +``` + +#### Control Individual Microservices (inside brainkb-unified) +```bash +# Restart only query service (doesn't affect other services) +./start_services.sh query-service restart + +# Restart only ML service +./start_services.sh ml-service restart + +# Restart only API token manager +./start_services.sh api-token-manager restart + +# Check status of a microservice +./start_services.sh query-service status + +# View logs of a microservice +./start_services.sh query-service logs +``` + +**Available microservices:** +- `query-service` - Query Service (port 8010) +- `ml-service` - ML Service (port 8007) +- `api-token-manager` - API Token Manager (port 8000) +- `usermanagement-service` - User Management Service (port 8004) + +#### Ollama Management +```bash +# Start Ollama (auto-detects GPU/CPU) +./start_services.sh ollama up + +# Stop Ollama +./start_services.sh ollama down + +# Restart Ollama +./start_services.sh ollama restart +``` + +### 4. View Logs + +```bash +# All services +./start_services.sh logs -f + +# Specific container +./start_services.sh logs -f brainkb-unified + +# Specific microservice (inside brainkb-unified) +./start_services.sh query-service logs +``` + +### 5. Stop Services + +```bash +# Stop all services +./start_services.sh down + +# Stop specific container +./start_services.sh brainkb-unified down +``` + +### 5. Access pgAdmin + +pgAdmin is included by default and will start automatically. + +Access pgAdmin at `http://localhost:5051` (default port, configurable via `PGADMIN_PORT` in `.env`) + +**Default credentials:** +- Email: `admin@brainkb.org` (set via `PGADMIN_DEFAULT_EMAIL` in `.env`) +- Password: `admin` (set via `PGADMIN_DEFAULT_PASSWORD` in `.env`) + +**Automatic PostgreSQL Server Registration:** +The PostgreSQL server is automatically configured when pgAdmin starts! After logging in, you should see **"BrainKB PostgreSQL"** (or your `PGADMIN_SERVER_NAME` value) in the left panel under "Servers". Simply click on it to connect. + +The connection uses: +- Host: `postgres` (Docker service name) +- Port: `5432` +- Database: Your `JWT_POSTGRES_DATABASE_NAME` value (default: `brainkb`) +- Username: Your `JWT_POSTGRES_DATABASE_USER` value (default: `postgres`) +- Password: Automatically configured from `JWT_POSTGRES_DATABASE_PASSWORD` in your `.env` file + +See [PGADMIN_SETUP.md](PGADMIN_SETUP.md) for more details. + +## Manual Docker Build and Run + +### Building the Image + +```bash +docker build -f Dockerfile.unified -t brainkb-unified:latest . +``` + +### Running the Container + +### Basic Usage + +```bash +docker run -d \ + --name brainkb-unified \ + -p 8000:8000 \ + -p 8004:8004 \ + -p 8007:8007 \ + -p 8010:8010 \ + -v /path/to/data:/data \ + -e DB_NAME=your_db_name \ + -e DB_USER=your_db_user \ + -e DB_PASSWORD=your_db_password \ + -e DB_HOST=your_db_host \ + -e DB_PORT=5432 \ + -e OXIGRAPH_USER=admin \ + -e OXIGRAPH_PASSWORD=admin \ + brainkb-unified:latest +``` + +### Environment Variables + +#### Database Configuration (for APItokenmanager) +- `DB_NAME` - PostgreSQL database name +- `DB_USER` - PostgreSQL username +- `DB_PASSWORD` - PostgreSQL password +- `DB_HOST` - PostgreSQL host (default: localhost) +- `DB_PORT` - PostgreSQL port (default: 5432) + +#### Oxigraph Authentication +- `OXIGRAPH_USER` - Username for oxigraph SPARQL endpoint +- `OXIGRAPH_PASSWORD` - Password for oxigraph SPARQL endpoint + +#### Django Superuser (Optional) +- `DJANGO_SUPERUSER_USERNAME` - Django admin username +- `DJANGO_SUPERUSER_EMAIL` - Django admin email +- `DJANGO_SUPERUSER_PASSWORD` - Django admin password + +### Service Endpoints + +Services are accessible directly on their respective ports: + +- **API Token Manager**: `http://localhost:8000/` +- **Query Service**: `http://localhost:8010/` +- **ML Service**: `http://localhost:8007/` +- **User Management Service**: `http://localhost:8004/` +- **Oxigraph SPARQL**: `http://localhost:7878/` (password protected) +- **pgAdmin**: `http://localhost:5051/` +- **Ollama**: `http://localhost:11434/` (if started separately) + +**Note:** +- UI should be deployed separately and configured to point to these backend endpoints +- All services are accessible directly on their configured ports +- pgAdmin starts automatically with the deployment +- Ollama is set up automatically by `start_services.sh` (GPU/CPU auto-detected) + +### Data Persistence + +Mount a volume for oxigraph data storage: +```bash +-v /path/to/oxigraph-data:/data +``` + +### Viewing Logs + +**Using start_services.sh (recommended):** +```bash +# All services +./start_services.sh logs -f + +# Specific microservice +./start_services.sh query-service logs +./start_services.sh ml-service logs +./start_services.sh api-token-manager logs +``` + +**Direct docker commands:** +```bash +# All services +docker logs brainkb-unified + +# Individual service logs (inside container) +docker exec brainkb-unified tail -f /var/log/supervisor/api_tokenmanager.out.log +docker exec brainkb-unified tail -f /var/log/supervisor/query_service.out.log +docker exec brainkb-unified tail -f /var/log/supervisor/ml_service.out.log +docker exec brainkb-unified tail -f /var/log/supervisor/usermanagement_service.out.log +docker exec brainkb-unified tail -f /var/log/supervisor/oxigraph.out.log +``` + +### Managing Services + +**Using start_services.sh (recommended):** +```bash +# Restart individual microservices (fast, no downtime for other services) +./start_services.sh query-service restart +./start_services.sh ml-service restart +./start_services.sh api-token-manager restart +./start_services.sh usermanagement-service restart + +# Check status +./start_services.sh query-service status + +# Start/stop individual microservices +./start_services.sh query-service start +./start_services.sh query-service stop +``` + +**Direct supervisor commands:** +```bash +# View all service statuses +docker exec -it brainkb-unified supervisorctl status + +# Restart individual services +docker exec -it brainkb-unified supervisorctl restart api_tokenmanager +docker exec -it brainkb-unified supervisorctl restart query_service +docker exec -it brainkb-unified supervisorctl restart ml_service +docker exec -it brainkb-unified supervisorctl restart usermanagement_service +docker exec -it brainkb-unified supervisorctl restart oxigraph +``` + +### Ollama Configuration + +Ollama is automatically set up by `start_services.sh` with the following features: + +- **Automatic GPU Detection**: Detects NVIDIA GPU and uses GPU acceleration if available +- **CPU Fallback**: Falls back to CPU mode if GPU is not available +- **Model Loading**: Automatically loads the model specified in `OLLAMA_MODEL` from `.env` +- **Persistent Storage**: Uses Docker volume for model storage + +**Configuration in `.env`:** +```bash +# Ollama model to use +OLLAMA_MODEL=nomic-embed-text + +# Ollama port (default: 11434) +OLLAMA_PORT=11434 + +# Ollama API endpoint (for services in Docker) +OLLAMA_API_ENDPOINT=http://host.docker.internal:11434 +``` + +**Manual Ollama Management:** +```bash +# Start Ollama +./start_services.sh ollama up + +# Stop Ollama +./start_services.sh ollama down + +# Check Ollama status +docker ps | grep ollama +docker exec ollama ollama list +``` + +**GPU Requirements:** +- For GPU support, install [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) +- The script automatically detects GPU availability and configures accordingly + +## Notes + +1. **Oxigraph**: The oxigraph binary is extracted from the official Docker image. If it's not available for your architecture, you may need to run oxigraph in a separate container. + +2. **Database**: Ensure PostgreSQL is running and accessible. The APItokenmanager service will run migrations automatically on startup. + +3. **Static Files**: Django static files are collected automatically on startup. + +4. **Resource Requirements**: This unified container runs multiple services. Ensure adequate CPU and memory resources: + - Minimum: 4 CPU cores, 8GB RAM + - Recommended: 8 CPU cores, 16GB RAM + +## Troubleshooting + +### Services not starting +Check supervisor logs: +```bash +docker exec brainkb-unified cat /var/log/supervisor/supervisord.log +``` + +### Database connection issues + +1. **Verify all passwords match in `.env`:** + - `JWT_POSTGRES_DATABASE_PASSWORD` must match: + - `DB_PASSWORD` (for Django APItokenmanager) + - All should be the same value! + +2. **Test database connection:** + ```bash + docker exec brainkb-unified psql -h postgres -U postgres -d brainkb + # Enter password when prompted + ``` + +3. **Check database logs:** + ```bash + docker logs brainkb-postgres + ``` + +4. **Verify services can reach postgres:** + ```bash + docker exec brainkb-unified ping -c 3 postgres + ``` + +### API Token Manager login issues + +1. **Verify SECRET_KEY is set:** + ```bash + docker exec brainkb-unified env | grep BRAINYPEDIA_APITOKEN_MANAGER_SECRET_KEY + ``` + If empty, add it to your `.env` file. Generate one with: + ```bash + python -c "from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())" + ``` + +2. **Check if superuser was created:** + ```bash + docker exec brainkb-unified python /app/APItokenmanager/manage.py shell -c "from django.contrib.auth import get_user_model; User = get_user_model(); print('Superusers:', list(User.objects.filter(is_superuser=True).values_list('username', flat=True)))" + ``` + +3. **Verify superuser credentials in .env:** + ```bash + docker exec brainkb-unified env | grep DJANGO_SUPERUSER + ``` + Make sure `DJANGO_SUPERUSER_PASSWORD` is set (not the placeholder value). + +4. **Create superuser manually if needed:** + ```bash + docker exec -it brainkb-unified python /app/APItokenmanager/manage.py createsuperuser + ``` + +5. **Check Django logs:** + ```bash + docker exec brainkb-unified tail -f /var/log/supervisor/api_tokenmanager.out.log + docker exec brainkb-unified tail -f /var/log/supervisor/api_tokenmanager.err.log + ``` + +6. **Test database connection:** + ```bash + docker exec brainkb-unified python /app/APItokenmanager/manage.py dbshell + ``` + +See [TROUBLESHOOTING_API_TOKEN_MANAGER.md](TROUBLESHOOTING_API_TOKEN_MANAGER.md) for detailed troubleshooting steps. + +### Query Service not working (port 8010) + +1. **Check if service is running:** + ```bash + docker exec brainkb-unified supervisorctl status query_service + ``` + +2. **Check query service logs:** + ```bash + docker exec brainkb-unified tail -f /var/log/supervisor/query_service.out.log + docker exec brainkb-unified tail -f /var/log/supervisor/query_service.err.log + ``` + +3. **Verify database connection:** + - Check that `DB_PASSWORD` matches `JWT_POSTGRES_DATABASE_PASSWORD` in `.env` + - Check that `JWT_POSTGRES_DATABASE_HOST_URL=postgres` (Docker service name) + +4. **Test query service directly:** + ```bash + curl http://localhost:8010/ + ``` + +### Testing Service Accessibility + +After starting services, test if they're accessible: + +```bash +# Check if services are running +docker-compose -f docker-compose.unified.yml ps + +# Test API Token Manager +curl http://localhost:8000/ + +# Test Query Service +curl http://localhost:8010/ + +# Test ML Service +curl http://localhost:8007/ + +# Test User Management Service +curl http://localhost:8004/ + + +# Test Oxigraph (password protected via HTTP Basic Auth) +# Use credentials from OXIGRAPH_USER and OXIGRAPH_PASSWORD in .env +curl -u admin:pwdoxigraph http://localhost:7878/ + +# Check service logs if not accessible +docker logs brainkb-unified +docker exec brainkb-unified supervisorctl status +``` + +### Services not accessible + +1. **Check if services are running:** + ```bash + docker exec brainkb-unified supervisorctl status + ``` + +2. **Check service logs:** + ```bash + docker exec brainkb-unified tail -f /var/log/supervisor/query_service.out.log + docker exec brainkb-unified tail -f /var/log/supervisor/api_tokenmanager.out.log + ``` + +3. **Verify ports are exposed:** + ```bash + docker-compose -f docker-compose.unified.yml ps + # Should show all ports mapped + ``` + +4. **Check if services are binding to 0.0.0.0:** + ```bash + docker exec brainkb-unified netstat -tlnp + # Should show services listening on 0.0.0.0:PORT + ``` + +### pgAdmin not accessible + +pgAdmin should start automatically. If it's not accessible: + +1. **Check if pgAdmin container is running:** + ```bash + docker-compose -f docker-compose.unified.yml ps + ``` + +2. **Check pgAdmin logs:** + ```bash + docker logs brainkb-pgadmin + ``` + +3. **Verify port is not in use:** + ```bash + # Check if port 5051 is already in use + lsof -i :5051 + ``` + +4. **Access at:** `http://localhost:5051` (or your `PGADMIN_PORT` value) + +## Docker Compose Configuration + +The `docker-compose.unified.yml` file includes: + +### Services + +1. **postgres**: PostgreSQL database for APItokenmanager and JWT services +2. **brainkb-unified**: Main container with all BrainKB services (API Token Manager, Query Service, ML Service, User Management Service) +3. **oxigraph**: SPARQL Database (internal, not directly exposed) +4. **oxigraph-nginx**: Nginx reverse proxy with HTTP Basic Authentication for Oxigraph (password protected) +5. **pgadmin**: PostgreSQL Admin Interface (starts automatically) + +### Volumes + +- `postgres_data`: Persistent storage for PostgreSQL +- `oxigraph_data`: Persistent storage for Oxigraph SPARQL database +- `pgadmin_data`: Persistent storage for pgAdmin (if enabled) + +### Networks + +- `brainkb-network`: Bridge network connecting all services + +### Environment Variables + +All environment variables are loaded from the `.env` file in the project root. The `env_file` directive in docker-compose automatically loads all variables from `.env` into the container. + +**Key Variables to Configure:** + +- **Database**: `JWT_POSTGRES_DATABASE_USER`, `JWT_POSTGRES_DATABASE_PASSWORD`, `JWT_POSTGRES_DATABASE_NAME` +- **Django**: `DJANGO_SUPERUSER_USERNAME`, `DJANGO_SUPERUSER_PASSWORD`, `BRAINYPEDIA_APITOKEN_MANAGER_SECRET_KEY` +- **JWT**: `*_SERVICE_JWT_SECRET_KEY` (service-specific keys), `JWT_POSTGRES_*` (all JWT-related variables) +- **Oxigraph**: `OXIGRAPH_USER`, `OXIGRAPH_PASSWORD` +- **Ports**: `API_TOKEN_PORT`, `QUERY_SERVICE_PORT`, `ML_SERVICE_PORT`, `USERMANAGEMENT_SERVICE_PORT`, `OXIGRAPH_PORT`, `PGADMIN_PORT` +- **User Management OAuth**: `USERMANAGEMENT_SERVICE_JWT_SECRET_KEY`, `USERMANAGEMENT_PUBLIC_BASE_URL`, `USERMANAGEMENT_FRONTEND_CALLBACK_URL`, `USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY`, `USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS`, `GITHUB_CLIENT_ID/SECRET`, `ORCID_CLIENT_ID/SECRET`, `GLOBUS_CLIENT_ID/SECRET` +- **Ollama**: `OLLAMA_MODEL`, `OLLAMA_PORT`, `OLLAMA_API_ENDPOINT` +- **ML Service**: `MONGO_DB_URL`, `WEAVIATE_*`, etc. +- **Query Service**: `GRAPHDATABASE_*`, `RAPID_RELEASE_FILE` + +See `env.template` for the complete list of all available environment variables with descriptions. + +### Health Checks + +- PostgreSQL has a health check to ensure it's ready before starting the unified container +- The unified container has a health check on the API Token Manager endpoint + +### Scaling + +To run multiple instances or scale specific services, modify the compose file or use: + +```bash +docker-compose -f docker-compose.unified.yml up -d --scale brainkb-unified=2 +``` + +Note: Only scale if you have a load balancer in front, as port conflicts will occur. + diff --git a/docker-compose.unified.yml b/docker-compose.unified.yml index ed8b250..e271967 100644 --- a/docker-compose.unified.yml +++ b/docker-compose.unified.yml @@ -33,7 +33,8 @@ services: - "${API_TOKEN_PORT:-8000}:8000" - "${QUERY_SERVICE_PORT:-8010}:8010" - "${ML_SERVICE_PORT:-8007}:8007" - + - "${USERMANAGEMENT_SERVICE_PORT:-8004}:8004" + volumes: - ./logs:/var/log/supervisor networks: diff --git a/docs/design_docs/PI-Grant-Skills-Research-Design.md b/docs/design_docs/PI-Grant-Skills-Research-Design.md new file mode 100644 index 0000000..15710d7 --- /dev/null +++ b/docs/design_docs/PI-Grant-Skills-Research-Design.md @@ -0,0 +1,157 @@ +# Construction of Enriched Research Knowledge Graphs and an Interactive User Interface for Project Grants and Research Findings + +## Overview +This document builds on ongoing discussions in Slack and consolidates ideas, requirements, and early explorations related to research knowledge graph construction and user interface design. Its purpose is to provide a unified and concrete design reference that connects these discussions with an evolving implementation. + +The screenshots included in this document reflect early explorations of knowledge graph structure, enrichment pipelines, and user interaction patterns. These artifacts ground the design in practical progress and serve as shared reference points for feedback and iteration. + +The figures (from @tekrajchhetri and @Sulstice) below illustrate components of the current proof-of-concept (PoC), including project- and PI-centered views, skill representations, summary statistics, and related explorations across systems. Together, they demonstrate early capabilities and help contextualize the design decisions discussed in this document. + +![](img/poc-projects-brainkb.png) +![](img/poc-projects-pi-brainkb.png) +![](img/poc-skills-brainkb.png) +![](img/poc-stats-brainkb.png) +![](img/bbqs-projects.png) + +By bringing together concepts discussed across channels, PoC artifacts, and related efforts, this document aims to align define scope, tasks and connect parallel work streams. + +## Use Cases + +The following use cases illustrate scenarios where this system provides value. + +1. **Understand Funding Landscape and Expertise** + *As a program manager,* I want to explore funding information across projects and investigators so that I can understand who has received funding, what skills and research areas are being supported, and how funded work connects to outcomes. + +2. **Discover People and Skills Behind Funded Research** + *As a researcher or PI,* I want to find people working in specific funded areas and understand their skills, projects, and publications so that I can identify potential collaborators and align my work with existing efforts. + +3. **Assess Impact and Identify Strategic Opportunities** + *As a research leader,* I want to connect funding, people, skills, and findings in one place so that I can evaluate research impact, identify gaps or overlaps, and make informed strategic decisions. + + +## Goals + +The goals of this work are to: + +1. **Construct enriched research knowledge graphs** capturing project funding, principal investigators (PIs), and projects, and automatically enrich them with related information such as publications, publication-derived findings, skills, and research areas. + +2. **Build and extend an interactive user interface** that supports user interactions for searching, browsing, and navigating the knowledge graphs to discover related and connected information. + +3. **Integrate and align with existing cross-project efforts**, including BICAN, BBQS, and Connects. The intent is not to duplicate existing work, but to reuse, connect, and extend shared data, infrastructure, and capabilities across these projects where appropriate. + +**Estimated deadline for completion: June, 30 2026.** + +## Requirements + + +The following requirements guide the design and implementation of the system. + + +1. The Knowledge Graphs (KGs) must be backed by a clearly defined, and machine-readable ontology or schema that formally specifies core entities, relationships, and constraints, ensuring interoperability, validation, and long-term maintainability. +2. Provenance should be treated as a first-class citizen and **W3C PROV-O (PROV ontology)** should be reused. +3. Grant data must be downloaded, transformed into KGs and stored locally, and all API queries and operations must use this locally stored enriched KG as the primary data source. +4. The system should provide capabilities to perform automated knowledge graph enrichment using AI agents, enabling extraction, linking, and augmentation of entities and relationships from external and unstructured sources, e.g., publications. +4. The KGs are stored in graph database, such as Oxigraph. +4. The API must support asynchronous execution for long-running and resource-intensive operations (e.g., KG ingestion and enrichment), enabling background processing with job tracking independent of client session state. +5. The implementation must check for existing data (e.g., project records) before retrieving new information, ensuring deduplication and avoiding unnecessary re-fetching or duplication of data, e.g., grant and publication informations. +6. The system must provide search features allowing one to search skills, projects PIs, and Co-PIs. +6. The user interface (UI) must support intuitive, CivicDB-style navigation, enabling users to start from any entity (e.g., a project) and seamlessly explore all directly linked and related information through interactive relationships. + +--- + +## Admin Data Flow + +```mermaid +--- +config: + theme: mc +--- +flowchart TB + subgraph UI["Admin UI"] + B{"Choose operation"} + A["Admin UI"] + C["Ingestion Form"] + C1["Enter Years (comma-separated)\n(e.g., 2022, 2023, 2025) or all data"] + C2["Select Data Types\n• Grants info\n• PIs/Authors\n• Orgs\n• Projects\n• Publications (optional)"] + C3["Select Sources / Connectors\n(e.g., internal grants DB, NIH RePORTER, Crossref)"] + C4["Set Options\n• Dry run\n• Overwrite vs upsert\n• Pagination/batch size"] + D["Submit"] + E["Enrichment Form"] + E1["Select scope\n• By Year(s)\n• By Project\n• By PI/Author\n• By Grant ID\n• By Changed-since date"] + E2["Select enrichment modules\n• Publication linking\n• Skill extraction\n• Research area tagging\n• Finding extraction\n• Entity resolution/dedup"] + E3["Configure models/rules\n• Model version\n• Thresholds\n• Provenance level"] + S["Scheduling UI"] + S1["Choose cadence\n• Daily / Weekly / Monthly / Quarterly"] + S2["Choose years rule\n• Current year\n• Last N years\n• Explicit list"] + S4["Set notifications\n• Email/Slack on success/failure"] + S5["Save schedule"] + X{"Import or Export?"} + X1["Export Form\n• Select dataset/scope\n• Choose RDF format\n(TTL / RDFXML / N-Triples)\n• Include provenance?"] + X2["Import Form\n• Upload RDF file(s)\n• Validate (LinkML)\n• Merge strategy\n(upsert / replace)\n• Preserve provenance?"] + UI1["Job created + Job ID\n(safe to close browser)"] + I["Return Job ID to UI"] + M1["Live status\nqueued/running/succeeded/failed"] + M["Admin Monitoring Dashboard"] + M2["Progress + logs + errors"] + M3["Retry / cancel (admin-only)"] + end + subgraph BE["Backend Services + Storage"] + F["Backend API Gateway"] + G["Job Orchestrator / Queue"] + H["Persist Job Record\n(status, progress, logs)"] + J["Scheduler Service"] + W1["Ingestion Workers"] + W2["Enrichment Workers"] + P1["Fetch raw data by year(s)\n(connectors)"] + P2["Process raw information and construct KG"] + KG[("RDF Triplestore / KG Store")] + Q1["Read target entities\n(from KG)"] + Q2["Extract/Link\n(project information, publications, skills, areas, findings)"] + Q3["Entity resolution + dedup\n(PIs/authors/orgs)"] + Q4["Update KGs with enriched knowledge"] + IDX["Search/Graph Indices\n(full-text + graph caches)"] + N["Notification Service"] + N1["Notify admin on completion/failure"] + end + A --> B + B -- Pull grants data --> C + C --> C1 & C2 & C3 & C4 + C1 --> D + C2 --> D + C3 --> D + C4 --> D + B -- Run enrichment --> E + E --> E1 & E2 & E3 + E1 --> D + E2 --> D + E3 --> D + B -- Schedule periodic pulls --> S + S --> S1 & S2 & S4 & S5 + B -- Import / Export data --> X + X -- Export --> X1 + X -- Import --> X2 + X1 --> D + X2 --> D + I --> UI1 + M --> M1 & M2 & M3 + D --> F + F --> G + G --> H & W1 & W2 + H --> I & M & N + S5 --> J + J --> G + W1 --> P1 + P1 --> P2 + W2 --> Q1 + Q1 --> Q2 + Q2 --> Q3 + Q3 --> Q4 + Q4 --> KG + KG --> IDX + N --> N1 + P2 --> KG +``` + +### References +[1] Xu, J., Yu, C., Xu, J. et al. PubMed knowledge graph 2.0: Connecting papers, patents, and clinical trials in biomedical science. Sci Data 12, 1018 (2025). https://doi.org/10.1038/s41597-025-05343-8 +[2] Xu, J., Kim, S., Song, M. et al. Building a PubMed knowledge graph. Sci Data 7, 205 (2020). https://doi.org/10.1038/s41597-020-0543-2 \ No newline at end of file diff --git a/docs/design_docs/PI-Grant-Skills-Research-microservice.md b/docs/design_docs/PI-Grant-Skills-Research-microservice.md new file mode 100644 index 0000000..ae8e384 --- /dev/null +++ b/docs/design_docs/PI-Grant-Skills-Research-microservice.md @@ -0,0 +1,763 @@ +# Design Document for Project Grants and Research Findings Microservices + +### Sequence diagram + +The sequence diagram below provides a high-level overview of the end-to-end data flow and interactions among the system components for KG construction. + +```mermaid +sequenceDiagram + participant User + participant Frontend + participant API + participant KG as Knowledge Graph + participant Oxigraph + participant NIH as NIH API + participant SS as Semantic Scholar + participant LLM as OpenRouter LLM + + User->>Frontend: Search/View/Edit + Frontend->>API: HTTP Request + API->>KG: Query/Update + KG->>Oxigraph: Read/Write RDF Triples + Oxigraph-->>KG: Return Results + KG-->>API: Processed Data + API-->>Frontend: JSON Response + Frontend-->>User: Display Results + + Note over API,LLM: Data Ingestion Flow + API->>NIH: Fetch Projects + NIH-->>API: Project Data + API->>KG: Add Projects + KG->>LLM: Extract Skills/Research Areas + LLM-->>KG: Extracted Entities + KG->>Oxigraph: Store Triples + + Note over API,SS: Enrichment Flow + API->>SS: Fetch Publications + SS-->>API: Publication Data + API->>KG: Add Publications + KG->>LLM: Extract from Abstracts + LLM-->>KG: Skills/Research Areas + KG->>Oxigraph: Store with Provenance +``` + + +## Implementation (API Endpoints) + +All the responses of API are in JSON format. The entities in API context denote things such as people and projects. + + +#### `GET /` +Health/root endpoint. + +**Response** + +- `200 OK` — JSON (empty schema) + +--- + + +### Unified Search + +#### `GET /api/search` +Unified search endpoint for **skills**, **projects**, and **people** with pagination. + +**Query Parameters** + +- `q` (string, optional): Search query. If omitted, returns recent/default data. +- `search_type` (string, optional, default `all`): `skills | projects | people | all` +- `use_semantic` (boolean, optional, default `true`): Enable semantic expansion. +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `20`, min `1`, max `100`) + +**Response** + +- `200 OK` — JSON (paginated) +- `422` — Validation error + +--- + + +#### `GET /api/search/skills` +Search for people by **skill**. + +**Query Parameters** + +- `skill` (string, required): Skill keyword + +**Response** + +- `200 OK` — `SearchResult[]` +- `422` — Validation error + +--- + +#### `GET /api/search/projects` +Search for projects by **keyword**. + +**Query Parameters** + +- `keyword` (string, required): Project keyword + +**Response** + +- `200 OK` — `SearchResult[]` +- `422` — Validation error + +--- + +#### `GET /api/search/people` +Search for people, i.e., PIs and Co-PIs by **name**. + +**Query Parameters** + +- `name` (string, required): Person name +- `orcid_id` (string, optional): Person ORCID ID which will be populated automatically. + +**Response** + +- `200 OK` — `SearchResult[]` +- `422` — Validation error + +--- + +### Projects + + +#### `GET /api/projects` +Get all projects with pagination and advanced filtering. + +**Query Parameters** + +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `50`, min `1`, max `200`) +- `search` (string, optional): Text filter +- `fiscal_year` (int, optional): Filter by fiscal year +- `min_funding` (number, optional) +- `max_funding` (number, optional) +- `organization` (string, optional) +- `pi_name` (string, optional) +- `skill` (string, optional) +- `research_area` (string, optional) + +**Response** + +- `200 OK` — JSON (paginated; schema not specified in OpenAPI snippet) +- `422` — Validation error + +--- + +### Get Project Detail + +#### `GET /api/project/{project_id}` +Get detailed information about a project. + +**Parameter(s)** + +- `project_id` (string, required) + +**Response** + +- `200 OK` — `SearchResult` +- `422` — Validation error + +--- + +### Update Project Detail + +#### `PUT /api/project/{project_id}` +Update project info (title/description) with provenance tracking. + +**Parameter(s)** + +- `project_id` (string, required) + +**Request Body** (`UpdateProjectRequest`) + +- `title` (string, optional) +- `description` (string, optional) +- `editor_name` (string, optional, default `"User"`) + +**Response** + +- `200 OK` — `SearchResult` +- `422` — Validation error + +--- + +## Grant PIs/Co-PIs + +Note: The term people, person are used to refer to PIs/Co-PIs. + +### List People + +#### `GET /api/people` +Get all people (PIs and co-PIs) with pagination and advanced filtering. + +**Query Parameters** + +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `50`, min `1`, max `200`) +- `search` (string, optional) +- `skill` (string, optional) +- `research_area` (string, optional) +- `organization` (string, optional) + +**Response** + +- `200 OK` — JSON (paginated; schema not specified in OpenAPI snippet) +- `422` — Validation error + +--- + +### Get Person Detail + +#### `GET /api/person/{person_id}` +Get detailed information about a person. + +**Parameter(s)** + +- `person_id` (string, required) + +**Response** + +- `200 OK` — `SearchResult` +- `422` — Validation error + +--- + +### Get Person Projects + +#### `GET /api/person/{person_id}/projects` +Get all projects for a person (as main PI or co-PI). + +**Parameter(s)** +- `person_id` (string, required) + +**Query Parameters** +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `50`, min `1`, max `200`) + +**Response** +- `200 OK` — `SearchResult[]` +- `422` — Validation error + +--- + +### Person Evolution + +#### `GET /api/person/{person_id}/evolution` +Temporal evolution of skills and research areas over time. + +**Parameter(s)** + +- `person_id` (string, required) + +**Query Parameters** + +- `entity_type` (string, optional, default `all`): `skills | research_areas | all` + +**Response** + +- `200 OK` — `PersonEvolution` +- `422` — Validation error + +--- + +## Skills & Research Areas + +### List Skills + +#### `GET /api/skills` + +Get all skills with pagination and optional year filter. + +**Query Parameters** + +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `50`, min `1`, max `200`) +- `search` (string, optional) +- `year` (int, optional): Temporal filter + +**Response** + +- `200 OK` — JSON (paginated; schema not specified in OpenAPI snippet) +- `422` — Validation error + +--- + +### Skill Detail + +#### `GET /api/skill/{skill_name}` +Get skill detail, including associated projects and people. + +**Parameter(s)** + +- `skill_name` (string, required) + +**Response** + +- `200 OK` — JSON (schema not specified in OpenAPI snippet) +- `422` — Validation error + +--- + +### List Research Areas + +#### `GET /api/research-areas` +Get all research areas with pagination and optional year filter. + +**Query Parameters** + +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `50`, min `1`, max `200`) +- `search` (string, optional) +- `year` (int, optional) + +**Response** + +- `200 OK` — JSON (paginated; schema not specified in OpenAPI snippet) +- `422` — Validation error + +--- + +### Research Area Detail + +#### `GET /api/research-area/{area_name}` +Get research area detail, including associated projects and people. + +**Parameter(s)** + +- `area_name` (string, required) + +**Response** + +- `200 OK` — JSON (schema not specified in OpenAPI snippet) +- `422` — Validation error + +--- + +## Related Entities + +### Get Related Entities + +#### `GET /api/related/{entity_type}/{entity_id}` +Fetch related entities based on shared skills, projects, or research areas. + +**Parameter(s)** + +- `entity_type` (string, required) +- `entity_id` (string, required) + +**Query Parameters** + +- `limit` (int, optional, default `10`, min `1`, max `50`) + +**Response** + +- `200 OK` — `SearchResult[]` +- `422` — Validation error + +--- + +## Chat + + +#### `POST /api/chat` +Chat endpoint backed by retrieval-augmented generation. + +**Request Body** (`ChatRequest`) +- `query` (string, required) + +**Response** +- `200 OK` — JSON +- `422` — Validation error + +--- + +## Graph & Network + +### Entity Network + +#### `GET /api/network/{entity_id}` +Get network visualization data for an entity. + +**Parameter(s)** + +- `entity_id` (string, required) + +**Query Parameters** + +- `depth` (int, optional, default `1`, min `1`, max `3`) + +**Response** + +- `200 OK` — `NetworkGraph` +- `422` — Validation error + +--- + +### Knowledge Graph (Global) + +#### `GET /api/knowledge-graph` +Get the full knowledge graph (nodes/edges) with a node limit. + +**Query Parameters** + +- `limit` (int, optional, default `100`, min `10`, max `500`) + +**Response** + +- `200 OK` — `NetworkGraph` +- `422` — Validation error + +--- + +## Sync & Stats + +### Sync Data (Public) + +#### `POST /api/sync` +Sync data from NIH Reporter API. + +**Query Parameters** + +- `limit` (int, optional, default `10`, min `1`, max `100`) + +**Request Body** + +- `project_ids` (string[] | null) + +**Response** + +- `200 OK` — JSON +- `422` — Validation error + +--- + +### Graph Stats + +#### `GET /api/stats` +Get knowledge graph statistics. + +**Response** +- `200 OK` — JSON (schema not specified in OpenAPI snippet) + +--- + +# Admin Endpoints + + +## Admin Sync + +### Start Admin Sync (Async) + +#### `POST /api/admin/sync` + +Starts a background job to sync projects from NIH. + +**Request Body** (`SyncRequest`) + +- `project_ids` (string[] | null) +- `limit` (int | null, default `10`) +- `fiscal_years` (int[] | null) +- `organization` (string | null) +- `keywords` (string[] | null) + +**Response** + +- `200 OK` — JSON (returns a job id; schema not specified in snippet) +- `422` — Validation error + +--- + +### Check Sync Status + +#### `GET /api/admin/sync/status/{job_id}` +Get status of a sync operation. + +**Parameter(s)** + +- `job_id` (string, required) + +**Response** + +- `200 OK` — JSON +- `422` — Validation error + +--- + +## Admin Data Access + +### Admin List Projects + +#### `GET /api/admin/data/projects` +Admin list projects with filtering and pagination. + +**Query Parameters** + +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `50`, min `1`, max `200`) +- `search` (string, optional) +- `fiscal_year` (int, optional) + +**Response** + +- `200 OK` — `PaginatedResponse` +- `422` — Validation error + +--- + +### Debug Project + +#### `GET /api/admin/debug/project/{project_id}` +Check what’s stored for a project in the knowledge graph. + +**Parameter(s)** + +- `project_id` (string, required) + +**Response** +- `200 OK` — JSON +- `422` — Validation error + +--- + +### Admin List People + +#### `GET /api/admin/data/people` +Admin list people with filtering and pagination. + +**Query Parameters** + +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `50`, min `1`, max `200`) +- `search` (string, optional) + +**Response** + +- `200 OK` — `PaginatedResponse` +- `422` — Validation error + +--- + +### Admin List Skills + +#### `GET /api/admin/data/skills` +Admin list skills with filtering and pagination. + +**Query Parameters** + +- `page` (int, optional, default `1`, min `1`) +- `page_size` (int, optional, default `50`, min `1`, max `200`) +- `search` (string, optional) + +**Response** + +- `200 OK` — `PaginatedResponse` +- `422` — Validation error + +--- + +## Admin Delete + +### Delete Entities + +#### `POST /api/admin/delete` + +Delete entities from the knowledge graph. + +**Request Body** (`DeleteRequest`) + +- `entity_type` (string, required) +- `entity_id` (string | null) + +**Response** + +- `200 OK` — JSON +- `422` — Validation error + +--- + +## Admin Pull All NIH (Async) + +### Start Pull-All Job + +#### `POST /api/admin/pull-all-nih` + +Starts a background job to fetch projects from NIH and skip already-stored projects. + +**Request Body** + +- `fiscal_years` (int[] | null): If null, pulls all projects +- `batch_size` (int, default `500`, min `100`, max `1000`) + +**Response** + +- `200 OK` — JSON (returns job id; schema not specified) +- `422` — Validation error + +--- + +### Check Pull-All Status + +#### `GET /api/admin/pull-all-nih/status/{job_id}` +Get status of pull operation. + +**Parameter(s)** + +- `job_id` (string, required) + +**Response** + +- `200 OK` — JSON +- `422` — Validation error + +--- + +## Admin Import / Export + +### Export Knowledge Graph + +#### `GET /api/admin/export` +Export graph data. + +**Query Parameters** + +- `format` (string, optional, default `turtle`): `turtle | xml | json-ld` + +**Response** + +- `200 OK` — JSON (or file-like payload depending on implementation) +- `422` — Validation error + +--- + +### Import Knowledge Graph + +#### `POST /api/admin/import` +Import graph data. This avoids re-fetching of data and KG construction. + +**Query Parameters** + +- `format` (string, optional, default `turtle`): `turtle | xml | json-ld` + +**Request Body** (`ImportRequest`) + +- `data` (string, required) + +**Response** +- `200 OK` — JSON +- `422` — Validation error + +--- + +## Admin Enrichment (Async) + +### Enrich Author(s) + +#### `POST /api/admin/enrich-author` +Enrich author(s) with publications and ORCID. Returns a job id. + +**Request Body** (`EnrichAuthorRequest`) + +- `person_id` (string | null) +- `person_name` (string | null) +- `project_id` (string | null) +- `fetch_publications` (bool, default `true`) +- `fetch_orcid` (bool, default `true`) +- `publication_limit` (int, default `20`) +- `skip_enriched` (bool, default `true`) + +**Response** + +- `200 OK` — JSON (job id) +- `422` — Validation error + +--- + +### Enrich Author Status + +#### `GET /api/admin/enrich-author/status/{job_id}` +Check status of an enrichment job. + +**Parameter(s)** + +- `job_id` (string, required) + +**Response** + +- `200 OK` — JSON +- `422` — Validation error + +--- + +### Debug API Keys + +#### `GET /api/admin/debug/api-keys` + +This endpoint is to check API key configuration. + +**Response** + +- `200 OK` — JSON +- `422` — Validation error + +--- + +### Autocomplete People + +#### `GET /api/admin/autocomplete/people` +Provides lightweight, prefix-based autocomplete suggestions for person (PI/co-PI) names stored in the knowledge graph. + +This endpoint performs a case-insensitive substring match against `foaf:name` values for all entities of type `foaf:Person`. It returns a limited list of matching people formatted for UI autocomplete components. + +**Query Parameters** + +- `q` (string, required, min length `1`) +- `limit` (int, optional, default `10`, min `1`, max `50`) + +**Response** + +- `200 OK` — JSON +- `422` — Validation error + +--- + +### Autocomplete Projects + +#### `GET /api/admin/autocomplete/projects` +Autocomplete for project IDs and titles. We will use the our schema -- to be developed. + +**Query Parameters** + +- `q` (string, required, min length `1`) +- `limit` (int, optional, default `10`, min `1`, max `50`) + +**Response** + +- `200 OK` — JSON +- `422` — Validation error + +--- + +### Enrich All Authors (Bulk, Async) + +#### `POST /api/admin/enrich-all-authors` + +Bulk enrichment for all authors, i.e., enrich with more information like skills and research areas by extracting the information. + +**Query Parameters** + +- `limit` (int | null, optional): Limit number of authors enriched +- `skip_enriched` (bool, default `true`) +- `fetch_publications` (bool, default `true`) +- `fetch_orcid` (bool, default `true`) +- `publication_limit` (int, default `20`, min `1`, max `100`) + +**Response** + +- `200 OK` — JSON (job id) +- `422` — Validation error + +--- \ No newline at end of file diff --git a/docs/design_docs/PI-Grant-Skills-Research-model.md b/docs/design_docs/PI-Grant-Skills-Research-model.md new file mode 100644 index 0000000..95d0a58 --- /dev/null +++ b/docs/design_docs/PI-Grant-Skills-Research-model.md @@ -0,0 +1 @@ +# Design Document for Project Grants and Research Findings Model \ No newline at end of file diff --git a/docs/design_docs/img/bbqs-projects.png b/docs/design_docs/img/bbqs-projects.png new file mode 100644 index 0000000..3be8f0e Binary files /dev/null and b/docs/design_docs/img/bbqs-projects.png differ diff --git a/docs/design_docs/img/poc-projects-brainkb.png b/docs/design_docs/img/poc-projects-brainkb.png new file mode 100644 index 0000000..787601b Binary files /dev/null and b/docs/design_docs/img/poc-projects-brainkb.png differ diff --git a/docs/design_docs/img/poc-projects-pi-brainkb.png b/docs/design_docs/img/poc-projects-pi-brainkb.png new file mode 100644 index 0000000..baa7a61 Binary files /dev/null and b/docs/design_docs/img/poc-projects-pi-brainkb.png differ diff --git a/docs/design_docs/img/poc-skills-brainkb.png b/docs/design_docs/img/poc-skills-brainkb.png new file mode 100644 index 0000000..2188cef Binary files /dev/null and b/docs/design_docs/img/poc-skills-brainkb.png differ diff --git a/docs/design_docs/img/poc-stats-brainkb.png b/docs/design_docs/img/poc-stats-brainkb.png new file mode 100644 index 0000000..56a656d Binary files /dev/null and b/docs/design_docs/img/poc-stats-brainkb.png differ diff --git a/env.template b/env.template index 9a6e4bf..28be2e2 100644 --- a/env.template +++ b/env.template @@ -59,12 +59,52 @@ JWT_ALGORITHM=HS256 QUERY_SERVICE_JWT_SECRET_KEY=your-query-service-jwt-secret-key-change-this-in-production ML_SERVICE_JWT_SECRET_KEY=your-ml-service-jwt-secret-key-change-this-in-production +# ---------------------------------------------------------------------------- +# User Management Service - JWT +# ---------------------------------------------------------------------------- +USERMANAGEMENT_SERVICE_JWT_SECRET_KEY=your-usermanagement-service-jwt-secret-key-change-this-in-production + +# ---------------------------------------------------------------------------- +# User Management Service - OAuth (unified flow for GitHub, ORCID, Globus) +# ---------------------------------------------------------------------------- +# Backend public base URL (used to build OAuth redirect_uri) +# e.g. http://localhost:8004 for local, https://api.brainkb.org for prod +USERMANAGEMENT_PUBLIC_BASE_URL=http://localhost:8004 + +# Frontend URL we redirect back to after OAuth callback (append ?token=...) +# e.g. http://localhost:3000/auth/callback +USERMANAGEMENT_FRONTEND_CALLBACK_URL=http://localhost:3000/auth/callback + +# Fernet key for encrypting OAuth access/refresh tokens at rest. +# Generate once: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY=Z85D3iJe4XCfJ5f8DExKXW3DfznyE4HzJ7XmmaOYtUQ= + +# Comma-separated emails that are bootstrapped as SuperAdmin on first startup. +# SuperAdmins also receive the regular Admin role for permissions, but the +# SuperAdmin marker protects them from being banned, deleted, or having that +# role stripped via the admin UI/API. Regular Admins (assigned through the +# admin UI) remain fully manageable. +USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS=tekraj@mit.edu + +# GitHub OAuth App +GITHUB_CLIENT_ID= +GITHUB_CLIENT_SECRET= + +# ORCID OAuth +ORCID_CLIENT_ID= +ORCID_CLIENT_SECRET= +# Use https://sandbox.orcid.org for dev, https://orcid.org for prod +ORCID_BASE_URL=https://orcid.org + +# Globus OAuth (register at https://app.globus.org/settings/developers) +GLOBUS_CLIENT_ID= +GLOBUS_CLIENT_SECRET= + # ---------------------------------------------------------------------------- # Are not used, kept it for future # ---------------------------------------------------------------------------- # CHAT_SERVICE_JWT_SECRET_KEY=your-chat-service-jwt-secret-key-change-this-in-production # INGESTION_SERVICE_JWT_SECRET_KEY=your-ingestion-service-jwt-secret-key-change-this-in-production -# USERMANAGEMENT_SERVICE_JWT_SECRET_KEY=your-usermanagement-service-jwt-secret-key-change-this-in-production # ---------------------------------------------------------------------------- # Service Ports - do not change unless you update docker/compose @@ -73,6 +113,7 @@ NGINX_PORT=80 API_TOKEN_PORT=8000 QUERY_SERVICE_PORT=8010 ML_SERVICE_PORT=8007 +USERMANAGEMENT_SERVICE_PORT=8004 OXIGRAPH_PORT=7878 UI_PORT=3000 PGADMIN_PORT=5051 diff --git a/scripts/check-env.sh b/scripts/check-env.sh new file mode 100755 index 0000000..6c54e3b --- /dev/null +++ b/scripts/check-env.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# ----------------------------------------------------------------------------- +# check-env.sh +# +# Detect env-key drift between env.template (the source of truth for what +# config the stack expects) and .env (the live values). +# +# Compares ONLY KEY NAMES, not values — values diverge intentionally +# (secrets, host-specific URLs, etc.). The check is one-directional by +# default: every key in the template must exist in .env, but extras in .env +# are reported as a non-fatal note. +# +# Usage: +# ./scripts/check-env.sh # default paths: /env.template, /.env +# ./scripts/check-env.sh TEMPLATE ENVFILE # explicit paths +# +# Exit codes: +# 0 no drift (every template key exists in .env) +# 1 drift detected (one or more template keys missing in .env) +# 2 invocation error (template or .env file missing) +# +# Wiring suggestions: +# - Add a call near the top of start_services.sh so deployments fail loudly +# when the live .env has fallen behind the template. +# - Add a pre-commit hook that runs it whenever env.template changes. +# ----------------------------------------------------------------------------- + +set -euo pipefail + +# Resolve repo root from this script's own location so relative invocations +# (e.g. `bash scripts/check-env.sh` from any cwd) still find env.template. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +TEMPLATE="${1:-$REPO_ROOT/env.template}" +ENV_FILE="${2:-$REPO_ROOT/.env}" + +if [[ ! -f "$TEMPLATE" ]]; then + echo "check-env: template not found: $TEMPLATE" >&2 + exit 2 +fi +if [[ ! -f "$ENV_FILE" ]]; then + echo "check-env: env file not found: $ENV_FILE" >&2 + exit 2 +fi + +if [[ -t 1 ]]; then + RED=$'\033[0;31m'; YELLOW=$'\033[0;33m'; GREEN=$'\033[0;32m'; BOLD=$'\033[1m'; RESET=$'\033[0m' +else + RED=''; YELLOW=''; GREEN=''; BOLD=''; RESET='' +fi + +# Extract dotenv KEY names from a file. Accepts: +# KEY=value +# export KEY=value +# KEY="value with spaces" +# leading whitespace, # comments, blanks, all ignored +extract_keys() { + grep -E '^[[:space:]]*(export[[:space:]]+)?[A-Z_][A-Z0-9_]*=' "$1" \ + | sed -E 's/^[[:space:]]*(export[[:space:]]+)?([A-Z_][A-Z0-9_]*)=.*/\2/' \ + | sort -u +} + +template_keys=$(extract_keys "$TEMPLATE") +env_keys=$(extract_keys "$ENV_FILE") + +missing_in_env=$(comm -23 <(echo "$template_keys") <(echo "$env_keys")) +extra_in_env=$(comm -13 <(echo "$template_keys") <(echo "$env_keys")) + +failed=0 +template_count=$(echo "$template_keys" | grep -c . || true) + +if [[ -n "$missing_in_env" ]]; then + failed=1 + echo "${BOLD}${RED}env drift:${RESET} keys present in ${TEMPLATE#$REPO_ROOT/} but missing from ${ENV_FILE#$REPO_ROOT/}" + while IFS= read -r k; do + echo " ${RED}-${RESET} $k" + done <<< "$missing_in_env" +fi + +if [[ -n "$extra_in_env" ]]; then + echo "${YELLOW}note:${RESET} keys in ${ENV_FILE#$REPO_ROOT/} but not in ${TEMPLATE#$REPO_ROOT/} (probably fine; consider adding to template):" + while IFS= read -r k; do + echo " ${YELLOW}+${RESET} $k" + done <<< "$extra_in_env" +fi + +if [[ "$failed" -eq 1 ]]; then + echo + echo "fix by adding the missing keys (with appropriate values) to ${BOLD}${ENV_FILE#$REPO_ROOT/}${RESET}." + echo "values can be copied from ${TEMPLATE#$REPO_ROOT/}, but secrets should be regenerated, not reused." + exit 1 +fi + +if [[ -z "$extra_in_env" ]]; then + echo "${GREEN}env keys aligned${RESET} (${template_count} keys checked, no drift)" +else + echo "${GREEN}all template keys present${RESET} (${template_count} checked); see notes above for extras" +fi +exit 0 diff --git a/start_services.sh b/start_services.sh index 028bec8..8a6bcd1 100755 --- a/start_services.sh +++ b/start_services.sh @@ -262,6 +262,9 @@ get_supervisor_name() { api-token-manager|api-tokenmanager|token-manager) echo "api_tokenmanager" ;; + usermanagement-service|user-management-service|usermanagement|user-management) + echo "usermanagement_service" + ;; *) echo "" ;; @@ -281,6 +284,7 @@ handle_microservice() { echo " - query-service (or query_service)" echo " - ml-service (or ml_service)" echo " - api-token-manager (or api_tokenmanager, token-manager)" + echo " - usermanagement-service (or user-management-service, usermanagement)" return 1 fi diff --git a/usermanagement_service/core/configuration.py b/usermanagement_service/core/configuration.py index 74933c4..03b32ba 100644 --- a/usermanagement_service/core/configuration.py +++ b/usermanagement_service/core/configuration.py @@ -37,23 +37,21 @@ def load_environment(env_name="env"): dict: A dictionary containing the loaded environment variables. """ # Determine the path to the .env file based on the environment - # Look in core directory first, then fall back to root directory + # Always fall back to root .env file - service-specific .env files are removed core_dir = os.path.dirname(os.path.abspath(__file__)) - root_dir = os.path.dirname(core_dir) - # Try core directory first - env_file = os.path.join(core_dir, f".{env_name}") - if not os.path.exists(env_file): - # Fall back to root directory - env_file = os.path.join(root_dir, f".{env_name}") - - # Load environment variables from the .env file - load_dotenv(dotenv_path=env_file) + # Traverse up to project root (BrainKB/) + # usermanagement_service/core/ -> usermanagement_service/ -> BrainKB/ + project_root = os.path.dirname(os.path.dirname(core_dir)) + + # Always load from root .env file (used by docker-compose) + root_env_file = os.path.join(project_root, ".env") + if os.path.exists(root_env_file): + load_dotenv(dotenv_path=root_env_file, override=False) # Return a dictionary containing the loaded environment variables return { "ENV_STATE": os.getenv("ENV_STATE"), - "DATABASE_URL": os.getenv("DATABASE_URL"), "LOGTAIL_API_KEY": os.getenv("LOGTAIL_API_KEY"), # PostgreSQL Database Configuration @@ -65,12 +63,31 @@ def load_environment(env_name="env"): # JWT Configuration "JWT_ALGORITHM": os.getenv("JWT_ALGORITHM", "HS256"), - "JWT_SECRET_KEY": os.getenv("JWT_SECRET_KEY"), + "JWT_SECRET_KEY": os.getenv("USERMANAGEMENT_SERVICE_JWT_SECRET_KEY"), "JWT_BEARER_TOKEN_URL": os.getenv("JWT_BEARER_TOKEN_URL"), "JWT_LOGIN_EMAIL": os.getenv("JWT_LOGIN_EMAIL"), "JWT_LOGIN_PASSWORD": os.getenv("JWT_LOGIN_PASSWORD"), + # OAuth / Admin Bootstrap + "USERMANAGEMENT_PUBLIC_BASE_URL": os.getenv("USERMANAGEMENT_PUBLIC_BASE_URL", "http://localhost:8004"), + "USERMANAGEMENT_FRONTEND_CALLBACK_URL": os.getenv("USERMANAGEMENT_FRONTEND_CALLBACK_URL", "http://localhost:3000/auth/callback"), + "USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY": os.getenv("USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY"), + # SuperAdmin bootstrap allowlist. Comma-separated emails. Seeded users + # get the SuperAdmin + Admin roles on first sight; the SuperAdmin role + # is protected against ban/delete/role-strip via the admin endpoints. + "USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS": os.getenv("USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS", ""), + + "GITHUB_CLIENT_ID": os.getenv("GITHUB_CLIENT_ID"), + "GITHUB_CLIENT_SECRET": os.getenv("GITHUB_CLIENT_SECRET"), + + "ORCID_CLIENT_ID": os.getenv("ORCID_CLIENT_ID"), + "ORCID_CLIENT_SECRET": os.getenv("ORCID_CLIENT_SECRET"), + "ORCID_BASE_URL": os.getenv("ORCID_BASE_URL", "https://orcid.org"), + + "GLOBUS_CLIENT_ID": os.getenv("GLOBUS_CLIENT_ID"), + "GLOBUS_CLIENT_SECRET": os.getenv("GLOBUS_CLIENT_SECRET"), + # Logging Configuration "LOG_LEVEL": os.getenv("LOG_LEVEL", "INFO"), "LOG_FORMAT": os.getenv("LOG_FORMAT", "%(asctime)s - %(name)s - %(levelname)s - %(message)s"), @@ -97,11 +114,6 @@ def env_state(self) -> Optional[str]: """Get the current environment state.""" return self._env_vars.get("ENV_STATE") - @property - def database_url(self) -> Optional[str]: - """Get the database URL.""" - return self._env_vars.get("DATABASE_URL") - @property def postgres_host(self) -> Optional[str]: """Get the PostgreSQL host.""" @@ -167,6 +179,53 @@ def logtail_api_key(self) -> Optional[str]: """Get the Logtail API key.""" return self._env_vars.get("LOGTAIL_API_KEY") + @property + def public_base_url(self) -> str: + return self._env_vars.get("USERMANAGEMENT_PUBLIC_BASE_URL", "http://localhost:8004") + + @property + def frontend_callback_url(self) -> str: + return self._env_vars.get("USERMANAGEMENT_FRONTEND_CALLBACK_URL", "http://localhost:3000/auth/callback") + + @property + def oauth_token_enc_key(self) -> Optional[str]: + return self._env_vars.get("USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY") + + @property + def bootstrap_superadmin_emails(self) -> list: + """Emails that get seeded as SuperAdmin (and Admin) on startup / first + OAuth login. Read from USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS.""" + raw = self._env_vars.get("USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS", "") or "" + return [e.strip().lower() for e in raw.split(",") if e.strip()] + + @property + def github_client_id(self) -> Optional[str]: + return self._env_vars.get("GITHUB_CLIENT_ID") + + @property + def github_client_secret(self) -> Optional[str]: + return self._env_vars.get("GITHUB_CLIENT_SECRET") + + @property + def orcid_client_id(self) -> Optional[str]: + return self._env_vars.get("ORCID_CLIENT_ID") + + @property + def orcid_client_secret(self) -> Optional[str]: + return self._env_vars.get("ORCID_CLIENT_SECRET") + + @property + def orcid_base_url(self) -> str: + return self._env_vars.get("ORCID_BASE_URL", "https://orcid.org") + + @property + def globus_client_id(self) -> Optional[str]: + return self._env_vars.get("GLOBUS_CLIENT_ID") + + @property + def globus_client_secret(self) -> Optional[str]: + return self._env_vars.get("GLOBUS_CLIENT_SECRET") + def get_postgres_settings(self) -> dict: """ Get PostgreSQL settings as a dictionary. diff --git a/usermanagement_service/core/database.py b/usermanagement_service/core/database.py index bcb3c3e..660d7ae 100644 --- a/usermanagement_service/core/database.py +++ b/usermanagement_service/core/database.py @@ -23,7 +23,12 @@ from fastapi import HTTPException from core.configuration import config -from core.models.database_models import Base, JWTUser, UserProfile, UserActivity, UserContribution, UserRole, UserCountry, UserOrganization, UserEducation, UserExpertise, AvailableRole, AvailableCountry +from core.models.database_models import ( + Base, JWTUser, UserProfile, UserActivity, UserContribution, UserRole, + UserCountry, UserOrganization, UserEducation, UserExpertise, AvailableRole, AvailableCountry, + OAuthIdentity, OAuthState, Permission, RolePermission, PageAccess, PageAccessRole, PageAccessUser, + AdminSetting, +) from core.models.user import ActivityType, ContributionStatus logger = logging.getLogger(__name__) @@ -206,30 +211,43 @@ def __init__(self): super().__init__(JWTUser) async def get_by_email(self, session: AsyncSession, email: str) -> Optional[JWTUser]: - """Get JWT user by email""" + """Get an *active* JWT user by email. Used by the password-login path, + which must reject deactivated accounts. OAuth flows that just need to + find an existing shell row (regardless of activation) should use + :py:meth:`get_by_email_any_status` instead.""" + return await self._fetch_by_email(session, email, active_only=True) + + async def get_by_email_any_status(self, session: AsyncSession, email: str) -> Optional[JWTUser]: + """Get a JWT user by email regardless of `is_active`. Used by the OAuth + callback (`_ensure_jwt_user_shell`), where a shell row is created with + `is_active=False` because OAuth users have no usable password — the + shell exists only to provide a stable `user_id` claim for the JWT. + Filtering by `is_active=true` here would miss the shell on every + repeat sign-in and trigger a duplicate-email INSERT.""" + return await self._fetch_by_email(session, email, active_only=False) + + async def _fetch_by_email(self, session: AsyncSession, email: str, *, active_only: bool) -> Optional[JWTUser]: try: - result = await session.execute( - text('SELECT * FROM "Web_jwtuser" WHERE email = :email AND is_active = true'), - {"email": email} - ) + sql = 'SELECT * FROM "Web_jwtuser" WHERE email = :email' + if active_only: + sql += ' AND is_active = true' + result = await session.execute(text(sql), {"email": email}) row = result.fetchone() if row: - # Convert raw result to JWTUser object - jwt_user = JWTUser( + return JWTUser( id=row.id, full_name=row.full_name, email=row.email, password=row.password, is_active=row.is_active, created_at=row.created_at, - updated_at=row.updated_at + updated_at=row.updated_at, ) - return jwt_user return None except SQLAlchemyError as e: logger.error(f"Error getting JWT user by email: {str(e)}") raise HTTPException(status_code=400, detail=str(e)) - + async def create_user(self, session: AsyncSession, full_name: str, email: str, password: str) -> JWTUser: """Create a new JWT user""" try: @@ -1288,6 +1306,249 @@ async def delete_by_name(self, session: AsyncSession, name: str) -> bool: raise HTTPException(status_code=400, detail=str(e)) +class OAuthIdentityRepository(UserBaseRepository): + """Repository for Web_oauth_identity — provider-linked identities.""" + + def __init__(self): + super().__init__(OAuthIdentity) + + async def get_by_provider_user(self, session: AsyncSession, provider: str, provider_user_id: str) -> Optional[OAuthIdentity]: + try: + result = await session.execute( + select(OAuthIdentity).where( + OAuthIdentity.provider == provider, + OAuthIdentity.provider_user_id == provider_user_id, + ) + ) + return result.scalar_one_or_none() + except SQLAlchemyError as e: + logger.error(f"Error getting oauth identity: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) + + async def list_for_profile(self, session: AsyncSession, profile_id: int) -> List[OAuthIdentity]: + try: + result = await session.execute( + select(OAuthIdentity).where(OAuthIdentity.profile_id == profile_id) + ) + return list(result.scalars().all()) + except SQLAlchemyError as e: + logger.error(f"Error listing oauth identities: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) + + async def upsert( + self, + session: AsyncSession, + *, + provider: str, + provider_user_id: str, + profile_id: int, + email: Optional[str], + access_token_enc: Optional[str], + refresh_token_enc: Optional[str], + token_expires_at: Optional[datetime], + raw_profile: Optional[dict], + ) -> OAuthIdentity: + existing = await self.get_by_provider_user(session, provider, provider_user_id) + if existing: + existing.profile_id = profile_id + existing.email = email + existing.access_token_enc = access_token_enc + existing.refresh_token_enc = refresh_token_enc + existing.token_expires_at = token_expires_at + existing.raw_profile = raw_profile + existing.updated_at = datetime.utcnow() + await session.flush() + await session.refresh(existing) + return existing + return await self.create( + session, + provider=provider, + provider_user_id=provider_user_id, + profile_id=profile_id, + email=email, + access_token_enc=access_token_enc, + refresh_token_enc=refresh_token_enc, + token_expires_at=token_expires_at, + raw_profile=raw_profile, + ) + + +class OAuthStateRepository(UserBaseRepository): + """Repository for short-lived OAuth state+PKCE tokens.""" + + def __init__(self): + super().__init__(OAuthState) + + async def get_by_state(self, session: AsyncSession, state: str) -> Optional[OAuthState]: + try: + result = await session.execute( + select(OAuthState).where(OAuthState.state == state) + ) + return result.scalar_one_or_none() + except SQLAlchemyError as e: + logger.error(f"Error getting oauth state: {str(e)}") + raise HTTPException(status_code=400, detail=str(e)) + + async def consume(self, session: AsyncSession, state: str) -> Optional[OAuthState]: + """Fetch and delete a state token — state must only be usable once.""" + row = await self.get_by_state(session, state) + if row is None: + return None + await session.delete(row) + await session.flush() + return row + + async def purge_expired(self, session: AsyncSession) -> None: + try: + await session.execute( + text('DELETE FROM "Web_oauth_state" WHERE expires_at < :now'), + {"now": datetime.utcnow()}, + ) + await session.flush() + except SQLAlchemyError as e: + logger.error(f"Error purging expired oauth states: {str(e)}") + + +class PermissionRepository(UserBaseRepository): + def __init__(self): + super().__init__(Permission) + + async def list_all(self, session: AsyncSession) -> List[Permission]: + result = await session.execute(select(Permission).order_by(Permission.resource, Permission.action)) + return list(result.scalars().all()) + + async def get_by_name(self, session: AsyncSession, name: str) -> Optional[Permission]: + result = await session.execute(select(Permission).where(Permission.name == name)) + return result.scalar_one_or_none() + + +class RolePermissionRepository(UserBaseRepository): + def __init__(self): + super().__init__(RolePermission) + + async def get_permissions_for_role(self, session: AsyncSession, role_id: int) -> List[Permission]: + result = await session.execute( + select(Permission) + .join(RolePermission, RolePermission.permission_id == Permission.id) + .where(RolePermission.role_id == role_id) + ) + return list(result.scalars().all()) + + async def get_permissions_for_role_names(self, session: AsyncSession, role_names: List[str]) -> List[Permission]: + if not role_names: + return [] + result = await session.execute( + select(Permission) + .join(RolePermission, RolePermission.permission_id == Permission.id) + .join(AvailableRole, AvailableRole.id == RolePermission.role_id) + .where(AvailableRole.name.in_(role_names)) + .distinct() + ) + return list(result.scalars().all()) + + async def set_role_permissions(self, session: AsyncSession, role_id: int, permission_ids: List[int]) -> None: + """Replace the set of permissions for a role.""" + await session.execute( + text('DELETE FROM "Web_role_permission" WHERE role_id = :rid'), + {"rid": role_id}, + ) + for pid in permission_ids: + session.add(RolePermission(role_id=role_id, permission_id=pid)) + await session.flush() + + +class PageAccessRepository(UserBaseRepository): + def __init__(self): + super().__init__(PageAccess) + + async def get_by_key(self, session: AsyncSession, page_key: str) -> Optional[PageAccess]: + result = await session.execute(select(PageAccess).where(PageAccess.page_key == page_key)) + return result.scalar_one_or_none() + + async def list_all(self, session: AsyncSession) -> List[PageAccess]: + result = await session.execute(select(PageAccess).order_by(PageAccess.page_key)) + return list(result.scalars().all()) + + async def upsert_with_members( + self, + session: AsyncSession, + *, + page_key: str, + description: Optional[str], + is_public: bool, + allowed_role_names: List[str], + allowed_profile_ids: List[int], + ) -> PageAccess: + existing = await self.get_by_key(session, page_key) + if existing: + existing.description = description + existing.is_public = is_public + existing.updated_at = datetime.utcnow() + page = existing + else: + page = PageAccess(page_key=page_key, description=description, is_public=is_public) + session.add(page) + await session.flush() + + await session.execute( + text('DELETE FROM "Web_page_access_role" WHERE page_access_id = :pid'), + {"pid": page.id}, + ) + await session.execute( + text('DELETE FROM "Web_page_access_user" WHERE page_access_id = :pid'), + {"pid": page.id}, + ) + for role_name in allowed_role_names: + session.add(PageAccessRole(page_access_id=page.id, role_name=role_name)) + for profile_id in allowed_profile_ids: + session.add(PageAccessUser(page_access_id=page.id, profile_id=profile_id)) + await session.flush() + await session.refresh(page) + return page + + async def delete_by_key(self, session: AsyncSession, page_key: str) -> bool: + result = await session.execute( + text('DELETE FROM "Web_page_access" WHERE page_key = :k'), + {"k": page_key}, + ) + await session.flush() + return result.rowcount > 0 + + async def get_allowed_roles(self, session: AsyncSession, page_access_id: int) -> List[str]: + result = await session.execute( + select(PageAccessRole.role_name).where(PageAccessRole.page_access_id == page_access_id) + ) + return [row[0] for row in result.all()] + + async def get_allowed_user_profile_ids(self, session: AsyncSession, page_access_id: int) -> List[int]: + result = await session.execute( + select(PageAccessUser.profile_id).where(PageAccessUser.page_access_id == page_access_id) + ) + return [row[0] for row in result.all()] + + async def check_access( + self, + session: AsyncSession, + *, + page_key: str, + profile_id: Optional[int], + role_names: List[str], + ) -> tuple[bool, str]: + page = await self.get_by_key(session, page_key) + if page is None: + return (False, "not_found") + if page.is_public: + return (True, "public") + if profile_id is not None: + user_ids = await self.get_allowed_user_profile_ids(session, page.id) + if profile_id in user_ids: + return (True, "user_override") + allowed_roles = await self.get_allowed_roles(session, page.id) + if any(r in allowed_roles for r in role_names): + return (True, "role") + return (False, "denied") + + # Repository instances jwt_user_repo = JWTUserRepository() user_profile_repo = UserProfileRepository() @@ -1299,4 +1560,68 @@ async def delete_by_name(self, session: AsyncSession, name: str) -> bool: user_education_repo = UserEducationRepository() user_expertise_repo = UserExpertiseRepository() available_role_repo = AvailableRoleRepository() -available_country_repo = AvailableCountryRepository() \ No newline at end of file +available_country_repo = AvailableCountryRepository() +oauth_identity_repo = OAuthIdentityRepository() +oauth_state_repo = OAuthStateRepository() +permission_repo = PermissionRepository() +role_permission_repo = RolePermissionRepository() +page_access_repo = PageAccessRepository() + +class AdminSettingRepository: + """Repository for the Web_admin_setting key/value store. Values are + Fernet-encrypted at rest using the same key as OAuth tokens — see + `core.security.encrypt_token` / `decrypt_token`. The repo stores + encrypted blobs only; encryption/decryption is the caller's responsibility + so we never accidentally leak plaintext through SQL logs.""" + + async def get(self, session: AsyncSession, key: str) -> Optional[AdminSetting]: + # Use ORM select so JSON deserialization is handled by SQLAlchemy. + result = await session.execute( + select(AdminSetting).where(AdminSetting.key == key).limit(1) + ) + return result.scalar_one_or_none() + + async def upsert( + self, + session: AsyncSession, + key: str, + value_enc: Optional[str], + allowed_role_names: Optional[List[str]], + updated_by: Optional[int], + ) -> AdminSetting: + # Use the ORM for both insert and update so SQLAlchemy serializes the + # JSON column for us — passing a Python list directly through raw SQL + # via asyncpg requires explicit json.dumps and is brittle. + existing_row = await session.execute( + select(AdminSetting).where(AdminSetting.key == key).limit(1) + ) + existing = existing_row.scalar_one_or_none() + now = datetime.utcnow() + if existing: + existing.value_enc = value_enc + existing.allowed_role_names = allowed_role_names + existing.updated_by = updated_by + existing.updated_at = now + await session.flush() + await session.refresh(existing) + return existing + row = AdminSetting( + key=key, + value_enc=value_enc, + allowed_role_names=allowed_role_names, + updated_by=updated_by, + ) + session.add(row) + await session.flush() + await session.refresh(row) + return row + + async def delete(self, session: AsyncSession, key: str) -> bool: + result = await session.execute( + text('DELETE FROM "Web_admin_setting" WHERE key = :k'), + {"k": key}, + ) + return (result.rowcount or 0) > 0 + + +admin_setting_repo = AdminSettingRepository() diff --git a/usermanagement_service/core/main.py b/usermanagement_service/core/main.py index bff70c4..251e0cc 100644 --- a/usermanagement_service/core/main.py +++ b/usermanagement_service/core/main.py @@ -13,9 +13,13 @@ from core.routers.index import router as index_router from core.routers.jwt_auth import router as jwt_router from core.routers.user_management import router as user_management_router +from core.routers.oauth import router as oauth_router +from core.routers.admin import router as admin_router +from core.routers.access import router as access_router from core.database import user_db_manager, user_activity_repo from core.models.user import ActivityType from core.security import verify_token +from core.bootstrap import run_bootstrap from fastapi.middleware.cors import CORSMiddleware class ActivityLoggingMiddleware(BaseHTTPMiddleware): @@ -105,7 +109,10 @@ async def lifespan(app: FastAPI): except Exception as e: logger.error(f"User database initialization failed: {str(e)}") raise - + + # Seed baseline roles, permissions, page access, and promote bootstrap admins. + await run_bootstrap() + yield # Shutdown logger.info("Shutting down FastAPI") @@ -137,6 +144,9 @@ async def lifespan(app: FastAPI): app.include_router(index_router) app.include_router(jwt_router, prefix="/api", tags=["Security Endpoints"]) app.include_router(user_management_router, prefix="/api", tags=["User Management"]) +app.include_router(oauth_router, prefix="/api", tags=["OAuth"]) +app.include_router(admin_router, prefix="/api/admin", tags=["Admin"]) +app.include_router(access_router, prefix="/api", tags=["Access Control"]) # log all HTTP exception when raised diff --git a/usermanagement_service/core/models/database_models.py b/usermanagement_service/core/models/database_models.py index f94640b..1a89b06 100644 --- a/usermanagement_service/core/models/database_models.py +++ b/usermanagement_service/core/models/database_models.py @@ -94,9 +94,19 @@ class UserProfile(Base): website: Mapped[Optional[str]] = mapped_column(String(500)) conflict_of_interest_statement: Mapped[Optional[str]] = mapped_column(Text) biography: Mapped[Optional[str]] = mapped_column(Text) + # Ban metadata. is_banned=True suspends the user — get_current_user + # rejects every authenticated request from the user with 403. The profile + # row itself is preserved (history, ORCID, etc. remain queryable). To + # cleanly remove a user use DELETE /api/admin/users/{id} instead. + is_banned: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + banned_at: Mapped[Optional[datetime]] = mapped_column(DateTime) + banned_by: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("Web_user_profile.id", ondelete="SET NULL") + ) + ban_reason: Mapped[Optional[str]] = mapped_column(Text) created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - + # Relationships - other tables link to this profile activities = relationship("UserActivity", back_populates="profile", cascade="all, delete-orphan") contributions = relationship("UserContribution", back_populates="profile", cascade="all, delete-orphan") @@ -319,7 +329,7 @@ class AvailableRole(Base): class AvailableCountry(Base): """Available countries for user profiles - Management table""" __tablename__ = "Web_available_country" - + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) name: Mapped[str] = mapped_column(String(100), unique=True, nullable=False) code: Mapped[Optional[str]] = mapped_column(String(3)) # ISO 3166-1 alpha-3 @@ -328,7 +338,7 @@ class AvailableCountry(Base): is_active: Mapped[bool] = mapped_column(Boolean, default=True) created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - + # Constraints and Indexes __table_args__ = ( Index('idx_available_country_name', 'name'), @@ -336,4 +346,181 @@ class AvailableCountry(Base): Index('idx_available_country_code_2', 'code_2'), Index('idx_available_country_region', 'region'), Index('idx_available_country_active', 'is_active'), - ) \ No newline at end of file + ) + + +class OAuthIdentity(Base): + """OAuth identity linking - one row per (provider, provider_user_id). + Unifies GitHub, ORCID, Globus logins and links them back to a UserProfile.""" + __tablename__ = "Web_oauth_identity" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + provider: Mapped[str] = mapped_column(String(32), nullable=False) # 'github' | 'orcid' | 'globus' + provider_user_id: Mapped[str] = mapped_column(String(255), nullable=False) + profile_id: Mapped[int] = mapped_column(Integer, ForeignKey("Web_user_profile.id", ondelete="CASCADE"), nullable=False) + email: Mapped[Optional[str]] = mapped_column(String(255)) + # Tokens stored encrypted at rest (Fernet). Nullable because some providers won't return a refresh_token. + access_token_enc: Mapped[Optional[str]] = mapped_column(Text) + refresh_token_enc: Mapped[Optional[str]] = mapped_column(Text) + token_expires_at: Mapped[Optional[datetime]] = mapped_column(DateTime) + raw_profile: Mapped[Optional[dict]] = mapped_column(JSONB) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + profile = relationship("UserProfile", backref="oauth_identities") + + __table_args__ = ( + UniqueConstraint('provider', 'provider_user_id', name='uq_oauth_provider_user'), + Index('idx_oauth_identity_provider', 'provider'), + Index('idx_oauth_identity_profile_id', 'profile_id'), + Index('idx_oauth_identity_email', 'email'), + ) + + +class OAuthState(Base): + """Short-lived OAuth state+PKCE verifier store, keyed by opaque state token. + Survives across horizontally-scaled instances (vs. in-memory dict).""" + __tablename__ = "Web_oauth_state" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + state: Mapped[str] = mapped_column(String(128), unique=True, nullable=False) + provider: Mapped[str] = mapped_column(String(32), nullable=False) + code_verifier: Mapped[Optional[str]] = mapped_column(String(256)) + redirect_after_login: Mapped[Optional[str]] = mapped_column(String(500)) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + expires_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + __table_args__ = ( + Index('idx_oauth_state_state', 'state'), + Index('idx_oauth_state_expires_at', 'expires_at'), + ) + + +class Permission(Base): + """Permission registry. A permission is a (resource, action) tuple, e.g. ('user', 'delete').""" + __tablename__ = "Web_permission" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + name: Mapped[str] = mapped_column(String(100), unique=True, nullable=False) + resource: Mapped[str] = mapped_column(String(100), nullable=False) + action: Mapped[str] = mapped_column(String(50), nullable=False) + description: Mapped[Optional[str]] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + __table_args__ = ( + UniqueConstraint('resource', 'action', name='uq_permission_resource_action'), + Index('idx_permission_name', 'name'), + Index('idx_permission_resource', 'resource'), + ) + + +class RolePermission(Base): + """Role <-> Permission many-to-many. role_name matches Web_available_role.name.""" + __tablename__ = "Web_role_permission" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + role_id: Mapped[int] = mapped_column(Integer, ForeignKey("Web_available_role.id", ondelete="CASCADE"), nullable=False) + permission_id: Mapped[int] = mapped_column(Integer, ForeignKey("Web_permission.id", ondelete="CASCADE"), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + + role = relationship("AvailableRole", backref="role_permissions") + permission = relationship("Permission", backref="role_permissions") + + __table_args__ = ( + UniqueConstraint('role_id', 'permission_id', name='uq_role_permission'), + Index('idx_role_permission_role', 'role_id'), + Index('idx_role_permission_permission', 'permission_id'), + ) + + +class PageAccess(Base): + """Page/route-level access control. One row per page_key (e.g. 'admin.users', 'curate.submit'). + is_public=True → anyone (even unauthenticated) can access. + Otherwise: user must be either in an allowed role (PageAccessRole) or explicitly whitelisted (PageAccessUser).""" + __tablename__ = "Web_page_access" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + page_key: Mapped[str] = mapped_column(String(100), unique=True, nullable=False) + description: Mapped[Optional[str]] = mapped_column(Text) + is_public: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + allowed_roles = relationship("PageAccessRole", back_populates="page", cascade="all, delete-orphan") + allowed_users = relationship("PageAccessUser", back_populates="page", cascade="all, delete-orphan") + + __table_args__ = ( + Index('idx_page_access_key', 'page_key'), + Index('idx_page_access_public', 'is_public'), + ) + + +class PageAccessRole(Base): + """Roles allowed on a page (role_name matches Web_available_role.name).""" + __tablename__ = "Web_page_access_role" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + page_access_id: Mapped[int] = mapped_column(Integer, ForeignKey("Web_page_access.id", ondelete="CASCADE"), nullable=False) + role_name: Mapped[str] = mapped_column(String(100), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + + page = relationship("PageAccess", back_populates="allowed_roles") + + __table_args__ = ( + UniqueConstraint('page_access_id', 'role_name', name='uq_page_access_role'), + Index('idx_page_access_role_page', 'page_access_id'), + Index('idx_page_access_role_role', 'role_name'), + ) + + +class PageAccessUser(Base): + """Per-user overrides for page access — grant a specific user access regardless of roles.""" + __tablename__ = "Web_page_access_user" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + page_access_id: Mapped[int] = mapped_column(Integer, ForeignKey("Web_page_access.id", ondelete="CASCADE"), nullable=False) + profile_id: Mapped[int] = mapped_column(Integer, ForeignKey("Web_user_profile.id", ondelete="CASCADE"), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + + page = relationship("PageAccess", back_populates="allowed_users") + profile = relationship("UserProfile") + + __table_args__ = ( + UniqueConstraint('page_access_id', 'profile_id', name='uq_page_access_user'), + Index('idx_page_access_user_page', 'page_access_id'), + Index('idx_page_access_user_profile', 'profile_id'), + ) + +class AdminSetting(Base): + """Admin-managed key/value settings stored encrypted at rest. + + Used today for the shared OpenRouter API key. Generic enough that future + shared secrets (Slack webhooks, S3 keys, etc.) can be added without a + schema change — just pick a new `key` value and reuse the same table. + + `value_enc` is Fernet-encrypted using + `USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY` (re-using the same key as the + OAuth-token-at-rest helper so we don't multiply key-management surface). + + `allowed_role_names` is a JSON list of role names that may consume the + setting. Empty list / NULL means "any signed-in user with a profile"; + `["Admin"]` would lock it back down to admin-only. Members of the listed + roles get the decrypted value via the user-facing endpoint; everyone + else gets a 'no shared key' response so they fall back to their own. + """ + __tablename__ = "Web_admin_setting" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + key: Mapped[str] = mapped_column(String(100), unique=True, nullable=False) + value_enc: Mapped[Optional[str]] = mapped_column(Text) + allowed_role_names: Mapped[Optional[list]] = mapped_column(JSON) + updated_by: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("Web_user_profile.id", ondelete="SET NULL") + ) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + updated_at: Mapped[datetime] = mapped_column( + DateTime, default=datetime.utcnow, onupdate=datetime.utcnow + ) + + __table_args__ = (Index("idx_admin_setting_key", "key"),) diff --git a/usermanagement_service/core/models/user.py b/usermanagement_service/core/models/user.py index 413aa33..4267297 100644 --- a/usermanagement_service/core/models/user.py +++ b/usermanagement_service/core/models/user.py @@ -10,6 +10,7 @@ # software or the use or other dealings in the software. # ----------------------------------------------------------------------------- +import re from datetime import datetime from typing import Optional, List, Dict, Any from enum import Enum @@ -19,22 +20,30 @@ class UserRoleEnum(str, Enum): """This is the user role for the BrainKB platform. Each user who are registered will have one of these roles, with default being the curator""" + # SuperAdmin: protected, immutable from the UI/API. + # Bootstrap-seeded only (USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS) and + # cannot be banned, deleted, or have its role stripped through admin endpoints. + SUPERADMIN = "SuperAdmin" + # Admin: regular platform administrator. Multiple admins can coexist; + # each can ban/delete/demote any other Admin (but not a SuperAdmin). + ADMIN = "Admin" + # Content Contribution Roles SUBMITTER = "Submitter" ANNOTATOR = "Annotator" MAPPER = "Mapper" CURATOR = "Curator" - + # Quality Control Roles REVIEWER = "Reviewer" VALIDATOR = "Validator" CONFLICT_RESOLVER = "Conflict Resolver" - + # Knowledge Management Roles KNOWLEDGE_CONTRIBUTOR = "Knowledge Contributor" EVIDENCE_TRACER = "Evidence Tracer" PROVENANCE_TRACKER = "Provenance Tracker" - + # Community Management Roles MODERATOR = "Moderator" AMBASSADOR = "Ambassador" @@ -57,6 +66,8 @@ class ActivityType(str, Enum): CONFLICT_RESOLUTION = "conflict_resolution" DISCUSSION_PARTICIPATION = "discussion_participation" COMMUNITY_OUTREACH = "community_outreach" + USER_BAN = "user_ban" + USER_UNBAN = "user_unban" class ContributionType(str, Enum): @@ -86,13 +97,6 @@ class ContributionStatus(str, Enum): # Authentication Models -class UserIn(BaseModel): - """JWT User registration input model""" - full_name: str = Field(..., min_length=1, max_length=255, description="User's full name") - email: EmailStr = Field(..., description="User's email address") - password: str = Field(..., min_length=8, description="User's password") - - class LoginUserIn(BaseModel): """JWT User login input model""" email: EmailStr = Field(..., description="User's email address") @@ -209,17 +213,29 @@ class UserExpertiseInput(BaseModel): class UserRoleInput(BaseModel): - """User role input model (for creating/updating)""" + """User role input model (for creating/updating). + + Accepts any role name that matches the canonical format. We don't check + membership in Web_available_role here because that's a DB-layer concern + — and admins should be able to assign roles they've just defined via + /api/admin/roles without two requests racing each other through + Pydantic. The format pattern matches the one in AvailableRoleInput so + the two endpoints stay consistent. + """ role: str = Field(..., description="Role name") is_active: bool = Field(default=True, description="Whether the role is active") expires_at: Optional[datetime] = Field(None, description="Role expiration date") - + @validator('role') def validate_role(cls, v): - """Validate that the role is one of the allowed values""" - valid_roles = [role.value for role in UserRoleEnum] - if v not in valid_roles: - raise ValueError(f'Role must be one of: {", ".join(valid_roles)}') + v = (v or "").strip() + if not v: + raise ValueError("Role name is required") + if not _ROLE_NAME_PATTERN.match(v): + raise ValueError( + "Role name must start with a letter and contain only letters, " + "numbers, spaces, hyphens, or underscores (max 100 chars)" + ) return v @@ -397,28 +413,62 @@ class AvailableRole(BaseModel): updated_at: Optional[datetime] = None +# Letters, numbers, spaces, hyphen, underscore. Must start with a letter to +# keep names sortable / paste-safe in URLs and JSON keys downstream. +_ROLE_NAME_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9 _-]{0,99}$") +_ROLE_CATEGORY_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9 _-]{0,49}$") + + class AvailableRoleInput(BaseModel): - """Available role input model for creating/updating""" + """Available role input model for creating/updating. + + Roles fall into two buckets: + + * **Canonical roles** (UserRoleEnum) — Admin, Curator, Reviewer, etc. + These are referenced by name in code (default OAuth role assignment, + admin checks, bootstrap seeding). They must exist and must keep their + names. Bootstrap re-seeds them on every startup if missing. + + * **Custom roles** — anything else admins create for organisational + purposes ("MIT User", "Lab X Member", "External Collaborator", ...). + These are free-form and only used for role-based page access; no code + path references them by name. + + The validator therefore enforces *format* (printable, length 1–100, + sensible characters) rather than membership in the canonical enum — + otherwise admins can't introduce organisational roles via the UI. Same + relaxation applies to ``category``: a small set of canonical buckets are + suggested, but custom values are accepted. + """ name: str = Field(..., max_length=100, description="Role name") description: Optional[str] = Field(None, description="Role description") category: Optional[str] = Field(None, max_length=50, description="Role category") is_active: bool = Field(default=True, description="Whether the role is active") - + @validator('name') def validate_role_name(cls, v): - """Validate that the role name is one of the allowed values""" - valid_roles = [role.value for role in UserRoleEnum] - if v not in valid_roles: - raise ValueError(f'Role name must be one of: {", ".join(valid_roles)}') + v = (v or "").strip() + if not v: + raise ValueError("Role name is required") + if not _ROLE_NAME_PATTERN.match(v): + raise ValueError( + "Role name must start with a letter and contain only letters, " + "numbers, spaces, hyphens, or underscores (max 100 chars)" + ) return v - + @validator('category') def validate_category(cls, v): - """Validate that the category is one of the allowed values""" - if v is not None: - valid_categories = ['Content', 'Quality', 'Knowledge', 'Community'] - if v not in valid_categories: - raise ValueError(f'Category must be one of: {", ".join(valid_categories)}') + if v is None: + return v + v = v.strip() + if not v: + return None + if not _ROLE_CATEGORY_PATTERN.match(v): + raise ValueError( + "Category must start with a letter and contain only letters, " + "numbers, spaces, hyphens, or underscores (max 50 chars)" + ) return v @@ -450,3 +500,88 @@ def validate_region(cls, v): if v not in valid_regions: raise ValueError(f'Region must be one of: {", ".join(valid_regions)}') return v + + +# Permission models +class PermissionInput(BaseModel): + name: str = Field(..., max_length=100) + resource: str = Field(..., max_length=100) + action: str = Field(..., max_length=50) + description: Optional[str] = None + + +class Permission(BaseModel): + id: Optional[int] = None + name: str + resource: str + action: str + description: Optional[str] = None + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + + +class RolePermissionAssignment(BaseModel): + """Assign a set of permissions to a role.""" + permission_ids: List[int] = Field(default_factory=list) + + +# Page access models +class PageAccessInput(BaseModel): + page_key: str = Field(..., max_length=100, description="Stable identifier for a UI page/feature") + description: Optional[str] = None + is_public: bool = Field(default=False) + allowed_roles: List[str] = Field(default_factory=list, description="Role names allowed to access this page") + allowed_user_emails: List[EmailStr] = Field(default_factory=list, description="Individual users allowed via email override") + + +class PageAccess(BaseModel): + id: Optional[int] = None + page_key: str + description: Optional[str] = None + is_public: bool = False + allowed_roles: List[str] = Field(default_factory=list) + allowed_user_emails: List[str] = Field(default_factory=list) + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + + +class PageAccessCheck(BaseModel): + page_key: str + allowed: bool + reason: str = Field(..., description="'public' | 'role' | 'user_override' | 'denied' | 'not_found'") + + +# OAuth models +class OAuthLoginStart(BaseModel): + authorize_url: str + state: str + + +class OAuthProviderEnum(str, Enum): + GITHUB = "github" + ORCID = "orcid" + GLOBUS = "globus" + + +class OAuthIdentityOut(BaseModel): + id: int + provider: str + provider_user_id: str + profile_id: int + email: Optional[str] = None + created_at: Optional[datetime] = None + + +# Admin-facing user list +class AdminUserListItem(BaseModel): + profile_id: int + name: str + email: str + orcid_id: Optional[str] = None + roles: List[str] = Field(default_factory=list) + providers: List[str] = Field(default_factory=list) + created_at: Optional[datetime] = None + is_banned: bool = False + banned_at: Optional[datetime] = None + banned_by: Optional[int] = None + ban_reason: Optional[str] = None diff --git a/usermanagement_service/core/routers/jwt_auth.py b/usermanagement_service/core/routers/jwt_auth.py index 2c54b92..f6ad117 100644 --- a/usermanagement_service/core/routers/jwt_auth.py +++ b/usermanagement_service/core/routers/jwt_auth.py @@ -1,54 +1,16 @@ import logging -from datetime import datetime -from fastapi import APIRouter, HTTPException, status, Depends, Request +from fastapi import APIRouter, HTTPException, status, Request -from core.database import user_db_manager, jwt_user_repo -from core.models.user import UserIn, LoginUserIn, ActivityType -from core.security import get_password_hash, authenticate_user, create_access_token_with_user_id, verify_token +from core.database import user_db_manager, jwt_user_repo, user_profile_repo, user_role_repo +from core.models.user import LoginUserIn +from core.security import authenticate_user, create_access_token_v2 logger = logging.getLogger(__name__) router = APIRouter() -@router.post("/register", status_code=201) -async def register(user: UserIn): - async with user_db_manager.get_async_session() as session: - # Check if JWT user already exists - existing_jwt_user = await jwt_user_repo.get_by_email(session, user.email) - if existing_jwt_user: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="A user with that email already exists", - ) - - hashed_password = get_password_hash(user.password) - - # Create JWT user - try: - new_jwt_user = await jwt_user_repo.create_user( - session=session, - full_name=user.full_name, - email=user.email, - password=hashed_password - ) - - return { - "detail": "Registration completed successfully! Admin will activate your account after verification." - } - except Exception as e: - logger.error(f"Error creating user: {str(e)}") - # Check if it's a duplicate email error - if "duplicate key value violates unique constraint" in str(e) and "email" in str(e): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="A user with that email already exists", - ) - else: - raise HTTPException(status_code=400, detail="Registration failed. Please try again.") - - @router.post("/token") async def login(user: LoginUserIn, request: Request): async with user_db_manager.get_async_session() as session: @@ -66,9 +28,20 @@ async def login(user: LoginUserIn, request: Request): if not scopes: scopes = ["read"] # Default scope if no scopes found - access_token = create_access_token_with_user_id(user_record.email, scopes, user_record.id) + # Attach profile-level roles if a profile exists for this email. + profile = await user_profile_repo.get_by_email(session, user_record.email) + profile_id = profile.id if profile else None + role_names = await user_role_repo.get_user_role_names(session, profile.id) if profile else [] + + access_token = create_access_token_v2( + email=user_record.email, + jwt_user_id=user_record.id, + profile_id=profile_id, + roles=role_names, + scopes=scopes, + auth_source="password", + ) - return {"access_token": access_token, "token_type": "bearer"} diff --git a/usermanagement_service/core/routers/user_management.py b/usermanagement_service/core/routers/user_management.py index cc928d2..d69e8c8 100644 --- a/usermanagement_service/core/routers/user_management.py +++ b/usermanagement_service/core/routers/user_management.py @@ -29,6 +29,7 @@ user_contribution_repo, user_role_repo, user_country_repo, user_organization_repo, user_education_repo, user_expertise_repo, available_role_repo, available_country_repo ) +from core.models.database_models import UserProfile as UserProfileModel from core.security import get_current_user, require_scopes, require_all_scopes from core.shared import convert_row_to_dict logger = logging.getLogger(__name__) @@ -37,6 +38,35 @@ +@router.get("/me") +async def get_me(jwt_user: Annotated[dict, Depends(get_current_user)]): + """Return the JWT-carrying user's profile summary (email, profile_id, roles, + auth_source, orcid_id, github). Useful for the UI to decide what to render + without an extra DB lookup per page.""" + orcid_id = None + github_username = None + name = None + profile_id = jwt_user.get("profile_id") + if profile_id: + async with user_db_manager.get_async_session() as session: + profile = await session.get(UserProfileModel, profile_id) + if profile: + orcid_id = profile.orcid_id + github_username = profile.github + name = profile.name + return { + "email": jwt_user.get("email"), + "name": name, + "profile_id": profile_id, + "user_id": jwt_user.get("user_id"), + "orcid_id": orcid_id, + "github": github_username, + "roles": jwt_user.get("roles", []), + "scopes": jwt_user.get("scopes", []), + "auth_source": jwt_user.get("auth_source", "password"), + } + + # User Profile Endpoints @router.get("/profile", response_model=UserProfile) async def get_profile( @@ -354,6 +384,30 @@ async def create_profile(jwt_user: Annotated[dict, Depends(get_current_user)], raise HTTPException(status_code=500, detail=f"Internal server error {e}") +@router.delete("/profile", status_code=204) +async def delete_own_profile( + jwt_user: Annotated[dict, Depends(get_current_user)], +): + """Delete the authenticated user's own profile. Admins should use + DELETE /api/admin/users/{profile_id} to delete other users.""" + try: + email = jwt_user.get("email") + if not email: + raise HTTPException(status_code=400, detail="JWT is missing email claim") + async with user_db_manager.get_async_session() as session: + profile = await user_profile_repo.get_by_email(session, email) + if not profile: + raise HTTPException(status_code=404, detail="Profile not found") + await session.delete(profile) + await session.commit() + return None + except HTTPException: + raise + except Exception as e: + logger.error(f"Error deleting own profile: {str(e)}") + raise HTTPException(status_code=500, detail="Internal server error") + + @router.put("/profile", response_model=dict) async def update_profile( jwt_user: Annotated[dict, Depends(get_current_user)], diff --git a/usermanagement_service/core/security.py b/usermanagement_service/core/security.py index 27d7804..bfc4ebe 100644 --- a/usermanagement_service/core/security.py +++ b/usermanagement_service/core/security.py @@ -11,18 +11,21 @@ # ----------------------------------------------------------------------------- import logging +import base64 +import hashlib from datetime import datetime, timedelta -from typing import Optional, Union, Any, Annotated +from typing import Optional, Union, Any, Annotated, List from jose import JWTError, jwt from passlib.context import CryptContext from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import text from fastapi import Depends, HTTPException, status from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from cryptography.fernet import Fernet, InvalidToken from core.configuration import config from core.database import jwt_user_repo -from core.models.user import LoginUserIn +from core.models.user import LoginUserIn, UserRoleEnum logger = logging.getLogger(__name__) @@ -30,7 +33,7 @@ pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") # JWT settings -SECRET_KEY = config.jwt_secret_key +SECRET_KEY = config.jwt_secret_key # Uses USERMANAGEMENT_SERVICE_JWT_SECRET_KEY ALGORITHM = config.jwt_algorithm ACCESS_TOKEN_EXPIRE_MINUTES = 30 @@ -77,7 +80,7 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) - def create_access_token_with_user_id(email: str, scopes: list, user_id: int) -> str: - """Create a JWT access token with user ID""" + """Create a JWT access token with user ID (legacy - no profile context).""" to_encode = { "sub": email, "scopes": scopes, @@ -88,6 +91,74 @@ def create_access_token_with_user_id(email: str, scopes: list, user_id: int) -> return encoded_jwt +def create_access_token_v2( + *, + email: str, + jwt_user_id: Optional[int], + profile_id: Optional[int], + roles: List[str], + scopes: List[str], + auth_source: str = "password", +) -> str: + """Create a JWT carrying both JWT-level scopes and profile-level roles. + auth_source: 'password' | 'github' | 'orcid' | 'globus'.""" + to_encode = { + "sub": email, + "scopes": scopes, + "user_id": jwt_user_id, + "profile_id": profile_id, + "roles": roles, + "auth_source": auth_source, + "exp": datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES), + } + return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) + + +# ---- Fernet helpers for encrypting OAuth tokens at rest ---- + +_fernet_instance: Optional[Fernet] = None + + +def _get_fernet() -> Fernet: + """Lazy Fernet loader. Derives a 32-byte key from the configured secret if it + isn't a valid Fernet key, so small configs don't fail hard. In prod, set a + proper key via USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY.""" + global _fernet_instance + if _fernet_instance is not None: + return _fernet_instance + + raw = config.oauth_token_enc_key + if not raw: + # Fall back to JWT secret so things boot; warn loudly. + logger.warning("USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY is not set; deriving a key from JWT secret. Set it in production.") + raw = SECRET_KEY or "change-me-unsafe-default" + + try: + # Try to use as-is if it's already a valid Fernet key. + _fernet_instance = Fernet(raw.encode() if isinstance(raw, str) else raw) + except Exception: + # Derive a 32-byte urlsafe key from the input. + digest = hashlib.sha256(raw.encode() if isinstance(raw, str) else raw).digest() + _fernet_instance = Fernet(base64.urlsafe_b64encode(digest)) + return _fernet_instance + + +def encrypt_token(plain: Optional[str]) -> Optional[str]: + if plain is None: + return None + return _get_fernet().encrypt(plain.encode()).decode() + + +def decrypt_token(ciphertext: Optional[str]) -> Optional[str]: + if ciphertext is None: + return None + try: + return _get_fernet().decrypt(ciphertext.encode()).decode() + except InvalidToken: + logger.error("Failed to decrypt OAuth token (InvalidToken)") + return None + + def verify_token(token: str) -> Union[dict, None]: """Verify and decode a JWT token""" try: @@ -142,32 +213,78 @@ def has_all_scopes(token: str, required_scopes: list) -> bool: security = HTTPBearer() -def get_current_user(credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)]) -> dict: - """Get current user from JWT token - FastAPI dependency""" +async def get_current_user(credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)]) -> dict: + """Get current user from JWT token — FastAPI dependency. Async because + we hit the DB to check ban status; the JWT itself is validated synchronously. + + Three failure modes: + * Invalid/expired token → 401 + * Token valid but the user has been banned (is_banned=True) → 403 + * Token valid, user not banned → returns the claims dict + """ token = credentials.credentials user_data = get_current_user_from_token(token) - + if user_data is None: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Could not validate credentials", headers={"WWW-Authenticate": "Bearer"}, ) - + + await _enforce_not_banned(user_data) return user_data +async def _enforce_not_banned(user_data: dict) -> None: + """Look up the user's profile and 403 if `is_banned` is True. We re-read + the flag on every request rather than baking it into the JWT — that way + an admin's ban takes effect immediately, without having to wait for the + user's JWT to expire / re-issue. + + Cost: one cheap indexed SELECT per authenticated request. If this becomes + hot we can cache the answer for a few seconds per (profile_id) key.""" + profile_id = user_data.get("profile_id") + if profile_id is None: + return + # Lazy import to avoid a circular import — security ↔ database both + # ultimately import models, and pulling user_db_manager at module load + # time creates the cycle. + from core.database import user_db_manager + from core.models.database_models import UserProfile as UserProfileModel + async with user_db_manager.get_async_session() as session: + profile = await session.get(UserProfileModel, profile_id) + if profile and getattr(profile, "is_banned", False): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": "account_suspended", + "message": "Your account has been suspended. Contact an administrator if you believe this is in error.", + "ban_reason": profile.ban_reason, + "banned_at": profile.banned_at.isoformat() if profile.banned_at else None, + }, + ) + + def get_current_user_from_token(token: str) -> Optional[dict]: - """Get current user from token string""" + """Get current user from token string. Returns claims including profile_id and roles (new fields) + alongside the legacy scopes/user_id fields.""" payload = verify_token(token) if payload is None: return None - + email: str = payload.get("sub") if email is None: return None - - return {"email": email, "user_id": payload.get("user_id"), "scopes": payload.get("scopes", [])} + + return { + "email": email, + "user_id": payload.get("user_id"), + "profile_id": payload.get("profile_id"), + "scopes": payload.get("scopes", []), + "roles": payload.get("roles", []), + "auth_source": payload.get("auth_source", "password"), + } def require_scopes(required_scopes: list): @@ -191,14 +308,64 @@ def require_all_scopes(required_scopes: list): """Dependency to require all specific scopes""" def scope_checker(current_user: Annotated[dict, Depends(get_current_user)]) -> dict: user_scopes = current_user.get("scopes", []) - + # Check if user has all required scopes if not all(scope in user_scopes for scope in required_scopes): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="Not enough permissions", ) - + return current_user - + return scope_checker + + +# Optional credentials — won't 401 if absent. Used by endpoints that work for +# both authed and unauthed users (e.g. page access checks for public pages). +_optional_security = HTTPBearer(auto_error=False) + + +async def get_current_user_optional( + credentials: Annotated[Optional[HTTPAuthorizationCredentials], Depends(_optional_security)], +) -> Optional[dict]: + """Like `get_current_user` but won't 401 when the request is anonymous. + Used for endpoints that serve both authed and unauthed callers (page access + checks for public pages, etc.). Banned users are silently demoted to + anonymous here rather than 403'd — that way a public page they could + have read while signed out is still readable, but they can't act as an + authenticated user.""" + if credentials is None: + return None + user_data = get_current_user_from_token(credentials.credentials) + if user_data is None: + return None + profile_id = user_data.get("profile_id") + if profile_id is None: + return user_data + from core.database import user_db_manager + from core.models.database_models import UserProfile as UserProfileModel + async with user_db_manager.get_async_session() as session: + profile = await session.get(UserProfileModel, profile_id) + if profile and getattr(profile, "is_banned", False): + return None + return user_data + + +def require_admin(current_user: Annotated[dict, Depends(get_current_user)]) -> dict: + """Dependency: caller must have the Admin or SuperAdmin role (profile-level). + Checks JWT 'roles' claim, falling back to the bootstrap superadmin email + allowlist for the very first sign-in (before any role is assigned in the DB).""" + email = (current_user.get("email") or "").lower() + roles = current_user.get("roles", []) or [] + + if UserRoleEnum.ADMIN.value in roles or UserRoleEnum.SUPERADMIN.value in roles: + return current_user + + if email and email in config.bootstrap_superadmin_emails: + return current_user + + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Admin role required", + ) diff --git a/usermanagement_service/requirements.txt b/usermanagement_service/requirements.txt index 5ea5aeb..787c7dd 100644 --- a/usermanagement_service/requirements.txt +++ b/usermanagement_service/requirements.txt @@ -23,6 +23,10 @@ eval-type-backport==0.1.3 python-jose==3.3.0 python-multipart==0.0.18 passlib[bcrypt]==1.7.4 +cryptography==42.0.5 + +# OAuth HTTP client +httpx==0.27.0 python-dotenv==1.0.0