StreamingRAG/docker-compose.yml at main · sharmajee499/StreamingRAG · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
services:
 broker:
   image: confluentinc/cp-kafka:8.0.0
   hostname: broker
   container_name: broker
   ports:
     - "9092:9092"
     - "9101:9101"
   environment:
     KAFKA_NODE_ID: 1
     KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
     KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092'
     KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
     KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
     KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
     KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
     KAFKA_JMX_PORT: 9101
     KAFKA_JMX_HOSTNAME: localhost
     KAFKA_PROCESS_ROLES: 'broker,controller'
     KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093'
     KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092'
     KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
     KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
     KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
     # Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid"
     # See https://docs.confluent.io/kafka/operations-tools/kafka-tools.html#kafka-storage-sh
     CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'

 connect:
   build: ./Docker/connect-cluster/
   hostname: connect
   container_name: connect
   depends_on:
     - broker
   ports:
     - "8083:8083"
   environment:
     CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
     CONNECT_REST_ADVERTISED_HOST_NAME: connect
     CONNECT_GROUP_ID: compose-connect-group
     CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
     CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
     CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
     CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
     CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
     CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
     CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
     CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
     CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"
     CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"

#  sql-client:
#    build: ./Docker/flink/
#    command: bin/sql-client.sh
#    depends_on:
#      - jobmanager
#    environment:
#      - |
#        FLINK_PROPERTIES=
#        jobmanager.rpc.address: jobmanager
#        rest.address: jobmanager

 jobmanager:
   build: ./Docker/flink/
   container_name: jobmanager
   ports:
     - "127.0.0.1:8081:8081"
   command: jobmanager
   volumes:
     - flink_data:/tmp/
     - ./FlinkJobs/:/opt/flink/FlinkJobs
   environment:
     - |
       FLINK_PROPERTIES=
       jobmanager.rpc.address: jobmanager
       state.backend: hashmap
       state.checkpoints.dir: file:///tmp/flink-checkpoints
       heartbeat.interval: 1000
       heartbeat.timeout: 5000
       rest.flamegraph.enabled: true
       web.backpressure.refresh-interval: 10000

 taskmanager:
   build: ./Docker/flink/
   container_name: taskmanager
   depends_on:
     - jobmanager
   command: taskmanager
   volumes:
     - flink_data:/tmp/
   environment:
     - |
       FLINK_PROPERTIES=
       jobmanager.rpc.address: jobmanager
       taskmanager.numberOfTaskSlots: 3
       state.backend: hashmap
       state.checkpoints.dir: file:///tmp/flink-checkpoints
       heartbeat.interval: 1000
       heartbeat.timeout: 5000

 ollama:
    build: ./Docker/ollama/
    container_name: ollama
    volumes:
      - ollama_data:/root/.ollama
    restart: unless-stopped
    ports:
      - "11434:11434"

 mongodb:
    hostname: mongodb
    build: ./Docker/mongodb/
    container_name: mongodb
    ports:
      - "27017:27017"
    volumes:
      - mongo_data:/data/db
      - mongo_config:/data/configdb
      - mongo_mongot:/data/mongot # Stores index for vector search
    environment:
      MONGO_INITDB_ROOT_USERNAME: root
      MONGO_INITDB_ROOT_PASSWORD: rootpassword

volumes:
  flink_data:
  ollama_data:
  mongo_data:
  mongo_config:
  mongo_mongot: