compose.yml

name: bento
services:
  redis:
    hostname: ${REDIS_HOST}
    image: ${REDIS_IMG}
    restart: always
    ports:
      - 6379:6379
    volumes:
      - redis-data:/data

  postgres:
    hostname: ${POSTGRES_HOST}
    image: ${POSTGRES_IMG}
    restart: always
    environment:
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
    expose:
      - '${POSTGRES_PORT}'
    ports:
      - '${POSTGRES_PORT}:${POSTGRES_PORT}'
    volumes:
      - postgres-data:/var/lib/postgresql/data
    command: -p ${POSTGRES_PORT}

  minio:
    hostname: ${MINIO_HOST}
    image: ${MINIO_IMG}
    ports:
      - '9000:9000'
      - '9001:9001'
    volumes:
      - minio-data:/data
    command: server /data --console-address ":9001"
    environment:
      - MINIO_ROOT_USER=${MINIO_ROOT_USER}
      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASS}
      - MINIO_DEFAULT_BUCKETS=${MINIO_BUCKET}
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 5s
      timeout: 5s
      retries: 5

  grafana:
    image: ${GRAFANA_IMG}
    restart: unless-stopped
    ports:
     - '3000:3000'
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_LOG_LEVEL=WARN
      - POSTGRES_HOST=${POSTGRES_HOST}
      - POSTGRES_DB=${POSTGRES_DB}
      - POSTGRES_PORT={POSTGRES_PORT}
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASS=${POSTGRES_PASSWORD}
    volumes:
      - ./grafana:/etc/grafana/provisioning/
      - grafana-data:/var/lib/grafana
    depends_on:
      - postgres
      - redis
      - minio

  exec_agent:
    image: agent
    runtime: nvidia
    restart: always
    depends_on:
      - postgres
      - redis
      - minio
    build:
      context: .
      dockerfile: bento/dockerfiles/agent.dockerfile
      args:
        NVCC_APPEND_FLAGS: "--gpu-architecture=compute_86 --gpu-code=compute_86,sm_86 --generate-code arch=compute_86,code=sm_86"
        CUDA_OPT_LEVEL: 1
    mem_limit: 4G
    cpu_count: 4
    environment:
      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
      REDIS_URL: redis://${REDIS_HOST}:6379
      S3_URL: http://${MINIO_HOST}:9000
      S3_BUCKET: ${MINIO_BUCKET}
      # TODO: create a lower perm user on startup of minio to use for agents
      S3_ACCESS_KEY: ${MINIO_ROOT_USER}
      S3_SECRET_KEY: ${MINIO_ROOT_PASS}
      RUST_LOG: ${RUST_LOG}
      RUST_BACKTRACE: 1
      RISC0_KECCAK_PO2: ${RISC0_KECCAK_PO2}
      # Enable / disable along with gpu_*_agent*
      # JOIN_STREAM: 1
      # COPROC_STREAM: 1
    entrypoint: /app/agent -t exec --segment-po2 ${SEGMENT_SIZE}

  aux_agent:
    image: agent
    runtime: nvidia
    pull_policy: never
    restart: always
    depends_on:
      - postgres
      - redis
      - minio
    mem_limit: 256M
    cpu_count: 1
    environment:
      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
      REDIS_URL: redis://${REDIS_HOST}:6379
      S3_URL: http://${MINIO_HOST}:9000
      S3_BUCKET: ${MINIO_BUCKET}
      S3_ACCESS_KEY: ${MINIO_ROOT_USER}
      S3_SECRET_KEY: ${MINIO_ROOT_PASS}
      RUST_LOG: ${RUST_LOG}
      RUST_BACKTRACE: 1
    # NOTE: if adding more aux workers, only one needs --monitor-requeue (for task reaping)
    entrypoint: /app/agent -t aux --monitor-requeue

  gpu_prove_agent0: &gpu
    image: agent
    runtime: nvidia
    pull_policy: never
    restart: always
    depends_on:
      - postgres
      - redis
      - minio
    mem_limit: 4G
    cpu_count: 4
    environment:
      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
      REDIS_URL: redis://${REDIS_HOST}:6379
      S3_URL: http://${MINIO_HOST}:9000
      S3_BUCKET: ${MINIO_BUCKET}
      S3_ACCESS_KEY: ${MINIO_ROOT_USER}
      S3_SECRET_KEY: ${MINIO_ROOT_PASS}
      RUST_LOG: ${RUST_LOG}
      RUST_BACKTRACE: 1
    entrypoint: /app/agent -t prove
    # comment-out if running in CPU proving mode
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              # TODO: how to scale this with N gpus?
              device_ids: ['0']
              capabilities: [gpu]

  snark_agent:
    image: agent
    runtime: nvidia
    pull_policy: never
    restart: always
    depends_on:
      - postgres
      - redis
      - minio
    environment:
      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
      REDIS_URL: redis://${REDIS_HOST}:6379
      S3_URL: http://${MINIO_HOST}:9000
      S3_BUCKET: ${MINIO_BUCKET}
      S3_ACCESS_KEY: ${MINIO_ROOT_USER}
      S3_SECRET_KEY: ${MINIO_ROOT_PASS}
      RUST_LOG: ${RUST_LOG}
      RUST_BACKTRACE: 1
    entrypoint: /app/agent -t snark
    ulimits:
      # Needed for stark_verify bin
      # TODO: this number was just guess and check found
      stack: 90000000

  rest_api:
    restart: always
    depends_on:
      - postgres
      - minio
    build:
      context: .
      dockerfile: bento/dockerfiles/rest_api.dockerfile
    mem_limit: 1G
    cpu_count: 1
    environment:
      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
      S3_URL: http://${MINIO_HOST}:9000
      S3_BUCKET: ${MINIO_BUCKET}
      S3_ACCESS_KEY: ${MINIO_ROOT_USER}
      S3_SECRET_KEY: ${MINIO_ROOT_PASS}
      RUST_LOG: ${RUST_LOG}
      RUST_BACKTRACE: 1
    ports:
      - '8081:8081'
    entrypoint: /app/rest_api --bind-addr 0.0.0.0:8081

volumes:
  redis-data:
  postgres-data:
  minio-data:
  grafana-data: