docker-compose.yml

name: starlake



x-starlake-ui-common:
  &starlake-ui-common
  image: starlakeai/starlake-${SL_MAJOR_VERSION:-1.5}-ui:${SL_IMAGE_VERSION:-1.5}
  pull_policy: always
  container_name: starlake-ui
  restart: on-failure
  depends_on:
    &starlake-ui-common-depends-on
    starlake-db:
      condition: service_healthy
  networks:
    &airflow-common-networks
    - starlake-network
  healthcheck:
    test: [ "CMD", "curl", "-s", "-I", "http://starlake-ui:9900/api/v1/health" ]
    interval: "5s"
    retries: 60
  ports:
    - ${SL_PORT:-80}:9900  # starlake-ui default port
  environment:
    &starlake-ui-common-env
    SL_DEBUG: 1
    SL_API_GIZMO_ON_DEMAND_URL: http://starlake-gizmo:10900
    SL_API_APP_TYPE: ${SL_API_APP_TYPE:-web}
    SL_HOME: /app/starlake
    SL_FS: file://
    SL_ENV:
    SL_ROOT:
    SL_USE_LOCAL_FILE_SYSTEM: false
    SL_API_GIT_COMMAND_ROOT: /git
    SL_PYTHON_LIBS_DIR: /app/starlake/bin/deps/python-libs
    SL_API_SECURE: false
    SL_API: true
    SL_API_SESSION_AS_HEADER: true
    SL_API_HTTP_FRONT_URL: ${SL_API_HTTP_FRONT_URL:-http://localhost:${SL_PORT:-80}}
    SL_API_HTTP_INTERFACE: 0.0.0.0
    SL_API_HTTP_PORT: 9900
    SL_LOG_LEVEL: ${SL_LOG_LEVEL:-info}
    SL_API_JDBC_DRIVER: org.postgresql.Driver
    SL_API_JDBC_USER: ${SL_POSTGRES_USER:-dbuser}
    SL_API_JDBC_PASSWORD: ${SL_POSTGRES_PASSWORD:-dbuser123}
    SL_API_JDBC_URL: jdbc:postgresql://starlake-db:5432/${SL_POSTGRES_DB:-starlake}?user=${SL_POSTGRES_USER:-dbuser}&password=${SL_POSTGRES_PASSWORD:-dbuser123} # JDBC URL to connect to the database
    SL_API_DOMAIN: ${SL_API_DOMAIN:-localhost}
    SL_API_PROJECTS_ROOT: /projects
    SL_API_AI_URL: http://starlake-agent:8000
    SL_AI_APPLICATION_KEY: Starlake7157
    ENVIRONMENT: local # local environment
    FILESTORE_MNT_DIR: /projects  # Environment variable to specify the mount path inside starlake-ui container
    EXTERNAL_PROJECTS_MNT_DIR: /external_projects
    POSTGRES_HOST: starlake-db
    POSTGRES_DB: ${SL_POSTGRES_DB:-starlake}
    POSTGRES_USER: ${SL_POSTGRES_USER:-dbuser}
    POSTGRES_PASSWORD: ${SL_POSTGRES_PASSWORD:-dbuser123}
    SL_UI_DEMO: ${SL_UI_DEMO:-false}
    SL_API_MAIL_HOST: ${SL_API_MAIL_HOST:-smtp.sendgrid.net}
    SL_API_MAIL_PORT: ${SL_API_MAIL_PORT:-587}
    SL_API_MAIL_USER: ${SL_API_MAIL_USER:-apikey}
    SL_API_MAIL_PASSWORD: ${SL_API_MAIL_PASSWORD:-}
    SL_API_MAIL_FROM: ${SL_API_MAIL_FROM:-contact@starlake.ai}
    SL_API_MAX_USER_SPACE_MB: ${SL_API_MAX_USER_SPACE_MB:-0}
    SL_API_FILE_UPLOAD_MAX_CONTENT_LENGTH: ${SL_API_FILE_UPLOAD_MAX_CONTENT_LENGTH:-21474836480} # 20 GB
    SL_API_UI_FOLDER: /app/ui
    SL_API_DOCS_USERS: starlake
    SL_API_DOCS_USER: starlake
    SL_API_DOCS_PASSWORD: s3cret.Paw
  volumes:
    - ./scripts/starlake:/usr/local/bin/starlake:ro
    - ${PROJECTS_DATA_PATH:-./projects}:/projects
    - ${EXTERNAL_PROJECTS_DATA_PATH:-./external_projects}:/external_projects


services:

# Database service
  starlake-db:
    profiles:
      - airflow3
      - dagster
      - snowflake
    image: postgres:17
    restart: on-failure
    container_name: starlake-db
    ports:
      - ${SL_DB_PORT:-5432}:5432
    environment:
      POSTGRES_USER: ${SL_POSTGRES_USER:-dbuser}
      POSTGRES_PASSWORD: ${SL_POSTGRES_PASSWORD:-dbuser123}
      POSTGRES_DB: ${SL_POSTGRES_DB:-starlake}
      AIRFLOW_DB: ${AIRFLOW_DB:-airflow}
      DAGSTER_DB: ${DAGSTER_DB:-dagster}
    command: postgres -c 'config_file=/etc/postgresql/postgresql.conf'
    healthcheck:
      test: ["CMD", "pg_isready", "-q", "-U", "${SL_POSTGRES_USER:-dbuser}", "-d", "${SL_POSTGRES_DB:-starlake}"]
      interval: "5s"
      retries: 5
    volumes:
      - ./pgdata:/var/lib/postgresql/data
      - ./conf/postgres/postgresql.conf:/etc/postgresql/postgresql.conf
      - ./scripts/airflow/init-database.sh:/docker-entrypoint-initdb.d/init-airflow-database.sh
      - ./scripts/dagster/init-database.sh:/docker-entrypoint-initdb.d/init-dagster-database.sh
    networks:
      - starlake-network


  starlake-airflow3:
    profiles:
      - airflow3
    container_name: starlake-airflow
    restart: on-failure
    image: starlakeai/starlake-airflow3:latest
    build:
      context: .  # Assuming Dockerfile_airflow is in the current directory
      dockerfile: Dockerfile_airflow3
    depends_on:
      starlake-db:
        condition: service_healthy
    networks:
      - starlake-network

    volumes:
      - /var/run/docker.sock:/var/run/docker.sock  # Mount Docker socket to run Docker commands from the container
      - ${PROJECTS_DATA_PATH:-./projects}:/projects
      - ${PROJECTS_DATA_PATH:-./projects}/dags:/opt/airflow/dags
      - ${AIRFLOW_LOGS:-./airflow/logs}:/opt/airflow/logs
      #- /Users/hayssams/.config:/home/airflow/.config
      - ./scripts/airflow/run-airflow3.sh:/app/run-airflow3.sh
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://starlake-airflow:8080/airflow/api/v2/version"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${SL_POSTGRES_USER:-dbuser}:${SL_POSTGRES_PASSWORD:-dbuser123}@starlake-db:5432/${AIRFLOW_DB:-airflow}
      AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
      SL_HOME: /app/starlake
      AIRFLOW__API__SECRET_KEY: ${AIRFLOW__API__SECRET_KEY:-a_very_secret_key}
      AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW__API_AUTH__JWT_SECRET:-a_very_secret_key}
      AIRFLOW__WEBSERVER__EXPOSE_CONFIG: 'true'
      AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: 'true'
      AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
      AIRFLOW__SCHEDULER__STANDALONE_DAG_PROCESSOR: 'true'
      AIRFLOW__API__EXPOSE_CONFIG: 'true'
      AIRFLOW__API__BASE_URL: ${SL_API_HTTP_FRONT_URL:-http://localhost:${SL_PORT:-80}}/airflow
      AIRFLOW__LOGGING__LOGGING_LEVEL: INFO
      AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://starlake-airflow:8080/airflow/execution
      #AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: ${AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL:-60}
      #AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL: ${AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL:-10}
      AIRFLOW__DAG_PROCESSOR__REFRESH_INTERVAL: ${AIRFLOW__DAG_PROCESSOR__REFRESH_INTERVAL:-30}
      AIRFLOW__FAB__AUTH_BACKENDS: airflow.providers.fab.auth_manager.api.auth.backend.session,airflow.providers.fab.auth_manager.api.auth.backend.basic_auth
      AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
      PYTHONPATH: /opt/airflow/dags
    entrypoint: /app/run-airflow3.sh

  starlake-ui-airflow3:
    profiles:
      - airflow3
    <<: *starlake-ui-common
    depends_on:
      <<: *starlake-ui-common-depends-on
      starlake-airflow3:
        condition: service_healthy
    environment:
      <<: *starlake-ui-common-env
      SL_API_ORCHESTRATOR_PRIVATE_URL: http://starlake-airflow:8080/airflow/
      LOAD_DAG_REF: airflow_load_shell
      TRANSFORM_DAG_REF: airflow_transform_shell
      SL_API_AIRFLOW_VERSION: 3
    entrypoint: >
      /bin/bash -c "
      sleep 10 &&
      airflow db migrate &&
      /usr/bin/tini -- /app/run-api.sh"

  starlake-dagster:
    profiles:
      - dagster
    image: starlakeai/starlake-dagster:latest
    build:
      context: .
      dockerfile: Dockerfile_dagster
    container_name: starlake-dagster
    restart: on-failure
    depends_on:
      starlake-db:
        condition: service_healthy
    networks:
      - starlake-network
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ${PROJECTS_DATA_PATH:-./projects}:/projects
      - ${PROJECTS_DATA_PATH:-./projects}/dags:/opt/dagster/app/dags
      - ${DAGSTER_LOGS:-./dagster/logs}:/opt/dagster/app/logs
      - ${DAGSTER_STORAGE:-./dagster/storage}:/opt/dagster/home/storage
    ports:
      - ${SL_DAGSTER_PORT:-3000}:3000
    environment:
      DAGSTER_PG_USERNAME: ${SL_POSTGRES_USER:-dbuser}
      DAGSTER_PG_PASSWORD: ${SL_POSTGRES_PASSWORD:-dbuser123}
      DAGSTER_PG_HOST: starlake-db
      DAGSTER_PG_DB: ${DAGSTER_DB:-dagster}
      SL_HOME: /app/starlake
    entrypoint: >
      /bin/bash -c "
      sleep 10 &&
      pip install --no-cache-dir starlake-dagster docker --upgrade --force-reinstall &&
      python3 dagster_code_locations.py &&
      service cron --full-restart &
      exec dagster-webserver -h 0.0.0.0 -p 3000 --path-prefix /dagster"

  starlake-ui-dagster:
    profiles:
      - dagster
    <<: *starlake-ui-common
    depends_on:
      <<: *starlake-ui-common-depends-on
      starlake-dagster:
        condition: service_started
    environment:
      <<: *starlake-ui-common-env
      SL_API_ORCHESTRATOR_PRIVATE_URL: http://starlake-dagster:3000/dagster/
      LOAD_DAG_REF: dagster_load_shell
      TRANSFORM_DAG_REF: dagster_transform_shell
    entrypoint: >
      /bin/bash -c "
      sleep 10 &&
      python3 -m pip install --break-system-packages --no-cache-dir starlake-dagster docker --upgrade &&
      /usr/bin/tini -- /app/run-api.sh"

  starlake-ui-snowflake:
    profiles:
      - snowflake
    <<: *starlake-ui-common
    environment:
      <<: *starlake-ui-common-env
      SL_API_ORCHESTRATOR_PRIVATE_URL:
      LOAD_DAG_REF: snowflake_load_sql
      TRANSFORM_DAG_REF: snowflake_transform_sql
    entrypoint: >
      /bin/bash -c "
      sleep 10 &&
      python3 -m pip install --break-system-packages --no-cache-dir starlake-snowflake --upgrade &&
      /usr/bin/tini -- /app/run-api.sh"

# Starlake Agent service
  starlake-agent:
    profiles:
      - ai
    image: starlakeai/starlake-1.5-ask:0.1
    pull_policy: always
    container_name: starlake-agent
    restart: on-failure
    depends_on:
      starlake-db:
        condition: service_healthy
    networks:
      - starlake-network
    healthcheck:
      test: [ "CMD", "curl", "-s", "http://starlake-agent:8000/ask/health" ]
      interval: "5s"
      retries: 60
    environment:
      SL_ASK_APPLICATION_KEY: Starlake7157
      SL_ASK_STARLAKE_JSON_FOLDER: /app/starlake_ask/resources
      SL_ASK_PG_USER: ${SL_POSTGRES_USER:-dbuser}
      SL_ASK_PG_PASSWORD: ${SL_POSTGRES_PASSWORD:-dbuser123}
      SL_ASK_PG_DB: ${SL_POSTGRES_DB:-starlake}
      SL_ASK_PG_SERVER: starlake-db
      SL_ASK_PG_PORT: 5432
      SL_ASK_API_SERVER_URL: http://starlake-ui:9900
      SL_ASK_HOST: 0.0.0.0
      SL_ASK_DB_TYPE: postgres
      SL_ASK_API_DOCS_USER: starlake
      SL_ASK_API_DOCS_PASSWORD: s3cret.Paw

  starlake-gizmo:
    profiles:
      - gizmo
    image: starlakeai/gizmo-on-demand:${SL_GIZMO_IMAGE_VERSION:-snapshot-slim}
    pull_policy: always
    container_name: starlake-gizmo
    restart: on-failure
    depends_on:
      starlake-ui-airflow3:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "curl", "-s", "http://starlake-gizmo:10900/health"]
      interval: "5s"
      retries: 60
    ports:
      - ${SL_GIZMO_PORT:-10900}:10900
      - ${SL_GIZMO_MIN_PORT:-11900}-${SL_GIZMO_MAX_PORT:-12000}:${SL_GIZMO_MIN_PORT:-11900}-${SL_GIZMO_MAX_PORT:-12000}
    networks:
      - starlake-network
    environment:
      - SL_GIZMO_ON_DEMAND_HOST=0.0.0.0
      - SL_GIZMO_ON_DEMAND_PORT=10900
      - SL_GIZMO_MIN_PORT=${SL_GIZMO_MIN_PORT:-11900}
      - SL_GIZMO_MAX_PORT=${SL_GIZMO_MAX_PORT:-12000}
      - SL_GIZMO_MAX_PROCESSES=${SL_GIZMO_MAX_PROCESSES:-10}
      - SL_GIZMO_API_KEY=${SL_GIZMO_API_KEY:-a_secret_api_key}
      - PG_HOST=starlake-db
      - PG_PORT=5432
      - PG_USERNAME=${SL_POSTGRES_USER:-dbuser}
      - PG_PASSWORD=${SL_POSTGRES_PASSWORD:-dbuser123}
      - GIZMOSQL_USERNAME=${GIZMOSQL_USERNAME:-gizmosql_user}
      - GIZMOSQL_PASSWORD=${GIZMOSQL_PASSWORD:-gizmosql_password}
      - JWT_SECRET_KEY=${JWT_SECRET_KEY:-a_very_secret_key}
      - TLS_ENABLED=0
    volumes:
      - ${PROJECTS_DATA_PATH:-./projects}:/projects
      #- /Users/hayssams/.config:/root/.config


  minio:
    profiles:
      - minio
    image: "quay.io/minio/minio:latest"
    ports:
      - "${SL_MINIO_PORT:-9000}:9000"
      - "${SL_MINIO_CONSOLE_PORT:-9001}:9001"
    networks:
      - starlake-network
    command: [ "server", "--console-address", ":9001", "/data" ]
    volumes:
      - ./minio/data:/data
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio123
  createbuckets:
    profiles:
      - minio
    image: quay.io/minio/mc:RELEASE.2025-03-12T17-29-24Z
    depends_on:
      - minio
    restart: on-failure
    environment:
      MINIO_ACCESS_KEY: minio
      MINIO_SECRET_KEY: minio123
      S3_ENDPOINT: http://minio:9000
    networks:
      - starlake-network
    entrypoint: >
      /bin/sh -c "
      sleep 5;
      /usr/bin/mc alias set dockerminio \$$S3_ENDPOINT \$$MINIO_ACCESS_KEY \$$MINIO_SECRET_KEY;
      /usr/bin/mc mb --ignore-existing dockerminio/ducklake-files;
      /usr/bin/mc mb --ignore-existing dockerminio/warehouse;
      exit 0;
      "

networks:
  starlake-network:
    driver: bridge