docker-compose.workers.dev.yml

# docker-compose.workers.dev.yml
#
# DEVELOPMENT version of docker-compose.workers.yml
#
# Project name: comses-agentspace-workers
# IMPORTANT: Use -p comses-agentspace-workers flag to group separately in Docker Desktop
# Note: The 'name:' field is being ignored, so you MUST use -p flag
#
# Key differences from production (docker-compose.workers.yml):
# - Mounts source code as volumes for live editing (no rebuild needed!)
# - Mounts pyproject.toml and uv.lock for dependency changes
# - Adds debugpy port mappings for remote debugging
# - Disables health checks for faster startup
# - Uses 'restart: "no"' for easier debugging (manual restart)
# - Adds DEBUG environment variables
# - Uses host networking mode for easier debugging
#
# Usage (MUST use -p flag or use Makefile commands):
#   # Development with live code reload
#   make dw-dev
#   # OR manually:
#   docker compose -p comses-agentspace-workers -f docker-compose.workers.dev.yml up -d
#
#   # Watch logs
#   make dw-dev-logs
#   # OR manually:
#   docker compose -p comses-agentspace-workers -f docker-compose.workers.dev.yml logs -f
#
#   # Restart a specific worker after code changes
#   make dw-dev-restart WORKER=postgres-worker
#   # OR manually:
#   docker compose -p comses-agentspace-workers -f docker-compose.workers.dev.yml restart postgres-worker
#
#   # Stop all workers
#   make kdw-dev
#   # OR manually:
#   docker compose -p comses-agentspace-workers -f docker-compose.workers.dev.yml down
#
# Benefits:
#   ✅ Edit code on host, changes reflected immediately in container
#   ✅ No need to rebuild Docker images during development
#   ✅ Remote debugging support with debugpy
#   ✅ Faster iteration cycle
#   ✅ Test worker code changes without full rebuild
#
# Note: You'll still need to rebuild if you change:
#   - System dependencies in Dockerfile
#   - Python dependencies in pyproject.toml (run `uv sync` in container)

# =============================================================================
# Common Worker Environment Variables (YAML Anchor)
# =============================================================================
x-workers-common-env: &workers-common-env # Temporal
  TEMPORAL_ADDRESS: temporal-nginx:7233
  TEMPORAL_NAMESPACE: default

  # Database (use container name from main docker-compose)
  # Use postgresql+asyncpg:// scheme for async database operations
  COMSES_RAG_DB_URL: postgresql+asyncpg://${COMSES_RAG_DB_USER}:${COMSES_RAG_DB_PASSWORD}@comses-rag-db:${COMSES_RAG_DB_PORT:-5432}/${COMSES_RAG_DB_NAME}

  # MinIO (use container name from main docker-compose)
  MINIO_HOST: minio
  MINIO_PORT: 9000
  MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
  MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
  MINIO_BUCKET_NAME: ${MINIO_BUCKET_NAME:-models}

  # Qdrant (use container name from main docker-compose)
  QDRANT_HOST: qdrant
  QDRANT_PORT: 6333
  QDRANT_API_KEY: ${QDRANT_API_KEY:-}

  # Redis (use container name from main docker-compose)
  REDIS_HOST: redis
  REDIS_PORT: 6379
  REDIS_PASSWORD: ${REDIS_PASSWORD:-}

  # LiteLLM (override to use Docker service name instead of localhost)
  LITELLM_PROXY_API_BASE: http://litellm-proxy:4000
  LITELLM_PROXY_API_KEY: ${LITELLM_PROXY_API_KEY:-}
  DEFAULT_AGENT_MODEL: ${DEFAULT_AGENT_MODEL}
  DEFAULT_CHAT_COMPLETION_MODEL: ${DEFAULT_CHAT_COMPLETION_MODEL}
  DEFAULT_SMART_CHAT_COMPLETION_MODEL: ${DEFAULT_SMART_CHAT_COMPLETION_MODEL}
  DEFAULT_LONG_CONTEXT_CHAT_COMPLETION_MODEL: ${DEFAULT_LONG_CONTEXT_CHAT_COMPLETION_MODEL}
  DEFAULT_EMBEDDING_MODEL: ${DEFAULT_EMBEDDING_MODEL}
  DEFAULT_RERANKING_MODEL: ${DEFAULT_RERANKING_MODEL}

  # File Storage (override to use container path, not host path)
  FILE_STORAGE_ROOT: /app/storage

  # Metrics & Monitoring (Global Flags)
  ENABLE_WORKER_METRICS: ${ENABLE_WORKER_METRICS:-true}

  # Logging Configuration
  WORKER_LOGGING_LEVEL: ${WORKER_LOGGING_LEVEL:-INFO}
  LOG_FORMAT: ${LOG_FORMAT:-json}

  # OpenTelemetry Tracing
  ENABLE_TRACING: ${ENABLE_TRACING:-false}
  OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://jaeger:4317}

  # Sentry Error Tracking (Shared DSN)
  ENABLE_SENTRY: ${ENABLE_SENTRY:-false}
  SENTRY_DSN: ${SENTRY_DSN:-}
  SENTRY_ENVIRONMENT: ${SENTRY_ENVIRONMENT:-development}
  SENTRY_TRACES_SAMPLE_RATE: ${SENTRY_TRACES_SAMPLE_RATE:-1.0}

  # Development flags
  PYTHONUNBUFFERED: 1
  PYTHONDONTWRITEBYTECODE: 1

# =============================================================================
# Common Development Volumes (can't use YAML anchor for lists with additions)
# Each service will duplicate these mounts + service-specific ones
# =============================================================================

# =============================================================================
# Services
# =============================================================================
services:
  # ---------------------------------------------------------------------------
  # Postgres Worker - Database operations
  # ---------------------------------------------------------------------------
  postgres-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile
    command: python -m src.shared.services.postgres.worker
    environment:
      <<: *workers-common-env
      POSTGRES_WORKER_MAX_CONCURRENT_ACTIVITIES: ${POSTGRES_WORKER_MAX_CONCURRENT_ACTIVITIES:-10}
      POSTGRES_WORKER_MAX_ACTIVITIES_PER_SECOND: ${POSTGRES_WORKER_MAX_ACTIVITIES_PER_SECOND:-100.0}
      POSTGRES_WORKER_METRICS_PORT: ${POSTGRES_WORKER_METRICS_PORT:-10090}
    ports:
      - "${POSTGRES_WORKER_METRICS_PORT:-10090}:${POSTGRES_WORKER_METRICS_PORT:-10090}" # Prometheus metrics
      - "5690:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-venv:/app/.venv
    restart: "no" # Manual restart for debugging
    networks:
      - comses-agentspace-network
      - temporal-network

  # ---------------------------------------------------------------------------
  # LiteLLM Worker - LLM operations
  # ---------------------------------------------------------------------------
  litellm-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile
    command: python -m src.shared.services.llms.worker
    environment:
      <<: *workers-common-env
      LITELLM_WORKER_METRICS_PORT: ${LITELLM_WORKER_METRICS_PORT:-10091}
    ports:
      - "${LITELLM_WORKER_METRICS_PORT:-10091}:${LITELLM_WORKER_METRICS_PORT:-10091}" # Prometheus metrics
      - "5691:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-venv:/app/.venv
    restart: "no"
    networks:
      - comses-agentspace-network
      - temporal-network

  # ---------------------------------------------------------------------------
  # Qdrant Worker - Vector operations
  # ---------------------------------------------------------------------------
  qdrant-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile
    command: python -m src.shared.services.vectors.worker
    environment:
      <<: *workers-common-env
      QDRANT_WORKER_MAX_CONCURRENT_ACTIVITIES: ${QDRANT_WORKER_MAX_CONCURRENT_ACTIVITIES:-4}
      QDRANT_WORKER_MAX_ACTIVITIES_PER_SECOND: ${QDRANT_WORKER_MAX_ACTIVITIES_PER_SECOND:-4.0}
      QDRANT_WORKER_METRICS_PORT: ${QDRANT_WORKER_METRICS_PORT:-10092}
    ports:
      - "${QDRANT_WORKER_METRICS_PORT:-10092}:${QDRANT_WORKER_METRICS_PORT:-10092}" # Prometheus metrics
      - "5692:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-venv:/app/.venv
    restart: "no"
    networks:
      - comses-agentspace-network
      - temporal-network

  # ---------------------------------------------------------------------------
  # MinIO Worker - File operations
  # ---------------------------------------------------------------------------
  minio-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile
    command: python -m src.shared.services.files.worker
    environment:
      <<: *workers-common-env
      MINIO_WORKER_MAX_CONCURRENT_ACTIVITIES: ${MINIO_WORKER_MAX_CONCURRENT_ACTIVITIES:-1}
      MINIO_WORKER_MAX_ACTIVITIES_PER_SECOND: ${MINIO_WORKER_MAX_ACTIVITIES_PER_SECOND:-1.0}
      MINIO_WORKER_METRICS_PORT: ${MINIO_WORKER_METRICS_PORT:-10093}
    ports:
      - "${MINIO_WORKER_METRICS_PORT:-10093}:${MINIO_WORKER_METRICS_PORT:-10093}" # Prometheus metrics
      - "5693:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-venv:/app/.venv
      - ${WORKER_STORAGE_PATH:-./local_data/worker-storage/minio}:/app/storage
    restart: "no"
    networks:
      - comses-agentspace-network
      - temporal-network

  # ---------------------------------------------------------------------------
  # Redis Worker - Cache operations
  # ---------------------------------------------------------------------------
  redis-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile
    command: python -m src.shared.services.store.worker
    environment:
      <<: *workers-common-env
      REDIS_WORKER_MAX_CONCURRENT_ACTIVITIES: ${REDIS_WORKER_MAX_CONCURRENT_ACTIVITIES:-50}
      REDIS_WORKER_MAX_ACTIVITIES_PER_SECOND: ${REDIS_WORKER_MAX_ACTIVITIES_PER_SECOND:-1000.0}
      REDIS_WORKER_METRICS_PORT: ${REDIS_WORKER_METRICS_PORT:-10094}
    ports:
      - "${REDIS_WORKER_METRICS_PORT:-10094}:${REDIS_WORKER_METRICS_PORT:-10094}" # Prometheus metrics
      - "5694:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-venv:/app/.venv
    restart: "no"
    networks:
      - comses-agentspace-network
      - temporal-network

  # ---------------------------------------------------------------------------
  # Ingestion Worker - Model ingestion pipeline
  # ---------------------------------------------------------------------------
  ingestion-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile
    command: python -m src.modules.ingestion.workers.ingestion_worker
    environment:
      <<: *workers-common-env
      INGEST_WORKER_MAX_CONCURRENT_ACTIVITIES: ${INGEST_WORKER_MAX_CONCURRENT_ACTIVITIES:-12}
      INGEST_WORKER_MAX_ACTIVITIES_PER_SECOND: ${INGEST_WORKER_MAX_ACTIVITIES_PER_SECOND:-100.0}
      INGESTION_WORKER_METRICS_PORT: ${INGESTION_WORKER_METRICS_PORT:-10095}
    ports:
      - "${INGESTION_WORKER_METRICS_PORT:-10095}:${INGESTION_WORKER_METRICS_PORT:-10095}" # Prometheus metrics
      - "5695:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-venv:/app/.venv
      - ${WORKER_STORAGE_PATH:-./local_data/worker-storage/ingestion}:/app/storage
    restart: "no"
    networks:
      - comses-agentspace-network
      - temporal-network

  # ---------------------------------------------------------------------------
  # PDF Converter Worker - PDF to Markdown conversion
  # Uses specialized Dockerfile with marker-pdf dependencies (~9 GB)
  # ---------------------------------------------------------------------------
  pdf-converter-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile.pdf-worker
    command: python -m src.modules.ingestion.workers.pdf_converter_worker
    environment:
      <<: *workers-common-env
      PDF_CONVERTER_WORKER_MAX_CONCURRENT_ACTIVITIES: ${PDF_CONVERTER_WORKER_MAX_CONCURRENT_ACTIVITIES:-1}
      PDF_CONVERTER_WORKER_MAX_ACTIVITIES_PER_SECOND: ${PDF_CONVERTER_WORKER_MAX_ACTIVITIES_PER_SECOND:-0.2}
      PDF_CONVERTER_WORKER_METRICS_PORT: ${PDF_CONVERTER_WORKER_METRICS_PORT:-10096}
      # Explicitly set XDG cache directory for marker-pdf models
      # Using /app/.cache instead of /home/worker/.cache to avoid permission issues
      XDG_CACHE_HOME: /app/.cache
    ports:
      - "${PDF_CONVERTER_WORKER_METRICS_PORT:-10096}:${PDF_CONVERTER_WORKER_METRICS_PORT:-10096}" # Prometheus metrics
      - "5696:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-pdf-venv:/app/.venv
      - ${WORKER_STORAGE_PATH:-./local_data/worker-storage/pdf-converter}:/app/storage
      # Persistent cache for marker-pdf models and other XDG cache data
      # Mounted to /app/.cache (owned by worker) instead of /home/worker/.cache
      - marker-pdf-models-cache:/app/.cache
    restart: "no"
    networks:
      - comses-agentspace-network
      - temporal-network

  # ---------------------------------------------------------------------------
  # Retrieval Worker - Workflow-only retrieval operations
  # ---------------------------------------------------------------------------
  retrieval-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile
    command: python -m src.modules.retrieval.worker
    environment:
      <<: *workers-common-env
      RETRIEVAL_WORKER_MAX_CONCURRENT_ACTIVITIES: ${RETRIEVAL_WORKER_MAX_CONCURRENT_ACTIVITIES:-10}
      RETRIEVAL_WORKER_MAX_ACTIVITIES_PER_SECOND: ${RETRIEVAL_WORKER_MAX_ACTIVITIES_PER_SECOND:-50.0}
      RETRIEVAL_WORKER_METRICS_PORT: ${RETRIEVAL_WORKER_METRICS_PORT:-10097}
    ports:
      - "${RETRIEVAL_WORKER_METRICS_PORT:-10097}:${RETRIEVAL_WORKER_METRICS_PORT:-10097}" # Prometheus metrics
      - "5697:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-venv:/app/.venv
    restart: "no"
    networks:
      - comses-agentspace-network
      - temporal-network

  # ---------------------------------------------------------------------------
  # COMSES Agent Worker - Chat agent operations
  # ---------------------------------------------------------------------------
  comses-agent-worker:
    build:
      context: .
      dockerfile: src/shared/worker-base/Dockerfile
    command: python -m src.apps.chat.backend.agents.comses_library_agent.worker
    environment:
      <<: *workers-common-env
      COMSES_AGENT_WORKER_MAX_CONCURRENT_ACTIVITIES: ${COMSES_AGENT_WORKER_MAX_CONCURRENT_ACTIVITIES:-10}
      COMSES_AGENT_WORKER_MAX_ACTIVITIES_PER_SECOND: ${COMSES_AGENT_WORKER_MAX_ACTIVITIES_PER_SECOND:-50.0}
      COMSES_AGENT_WORKER_METRICS_PORT: ${COMSES_AGENT_WORKER_METRICS_PORT:-10098}
    ports:
      - "${COMSES_AGENT_WORKER_METRICS_PORT:-10098}:${COMSES_AGENT_WORKER_METRICS_PORT:-10098}" # Prometheus metrics
      - "5698:5678" # debugpy (optional)
    volumes:
      - ./src:/app/src:rw
      - ./pyproject.toml:/app/pyproject.toml:ro
      - ./uv.lock:/app/uv.lock:ro
      - worker-dev-venv:/app/.venv
    restart: "no"
    networks:
      - comses-agentspace-network
      - temporal-network

# =============================================================================
# Networks
# =============================================================================
networks:
  comses-agentspace-network:
    name: comses-agentspace-network
    external: true
  temporal-network:
    name: temporal-network
    external: true

# =============================================================================
# Volumes
# =============================================================================
volumes:
  # Shared .venv volume for all workers (except pdf-converter)
  # This preserves installed packages across restarts
  # To update dependencies: docker compose -f docker-compose.workers.dev.yml exec <worker> uv sync
  worker-dev-venv:
    driver: local

  # Separate .venv volume for pdf-converter-worker
  # PDF converter has heavy dependencies (marker-pdf, PyTorch) that other workers don't need
  worker-dev-pdf-venv:
    driver: local

  # Persistent cache for marker-pdf models (~2-3 GB)
  # Prevents re-downloading models on every container restart
  marker-pdf-models-cache:
    driver: local