embedding_infinity_docker/docker-compose.server.yml at main · AarhusAI/embedding_infinity_docker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
services:
  nginx:
    restart: unless-stopped
    ports: !reset []
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=frontend"
      - "traefik.http.routers.${COMPOSE_PROJECT_NAME}-http.rule=Host(`${COMPOSE_SERVER_DOMAIN}`)"
      - "traefik.http.routers.${COMPOSE_PROJECT_NAME}-http.entrypoints=web"
      - "traefik.http.routers.${COMPOSE_PROJECT_NAME}-http.middlewares=redirect-to-https"
      - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
      - "traefik.http.routers.${COMPOSE_PROJECT_NAME}.entrypoints=websecure"
      - "traefik.http.routers.${COMPOSE_PROJECT_NAME}.rule=Host(`${COMPOSE_SERVER_DOMAIN}`)"

  authz:
    restart: unless-stopped
    ports: !reset []

  bge-m3:
    restart: unless-stopped
    ports: !reset []
    command:
      - v2
      - --model-id=BAAI/bge-m3
      - --served-model-name=bge-m3
      - --engine=torch
      - --dtype=float16
      - --device=cuda
      - --port=7997
      # Lower than e5-large because bge-m3's 8192-token context can balloon
      # activation memory; safer when sharing a 16 GB card.
      - --batch-size=${BGE_M3_BATCH_SIZE:-16}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              # Pin to one GPU; set BGE_M3_GPU and E5_LARGE_GPU to split across cards.
              device_ids: ["${BGE_M3_GPU:-0}"]
              capabilities: [gpu]

  e5-large:
    restart: unless-stopped
    ports: !reset []
    command:
      - v2
      - --model-id=intfloat/multilingual-e5-large
      - --served-model-name=multilingual-e5-large
      - --engine=torch
      - --dtype=float16
      - --device=cuda
      - --port=7997
      - --batch-size=${E5_LARGE_BATCH_SIZE:-32}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["${E5_LARGE_GPU:-0}"]
              capabilities: [gpu]

  bge-reranker:
    restart: unless-stopped
    ports: !reset []
    command:
      - v2
      - --model-id=BAAI/bge-reranker-v2-m3
      - --served-model-name=bge-reranker-v2-m3
      - --engine=torch
      - --dtype=float16
      - --device=cuda
      - --port=7997
      # Conservative because the 16 GB card is already shared with bge-m3 and
      # e5-large; reranker pairs (query, doc) inflate effective batch.
      - --batch-size=${BGE_RERANKER_BATCH_SIZE:-8}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ["${BGE_RERANKER_GPU:-0}"]
              capabilities: [gpu]