kafka-hackathon/docker-compose.yaml at master · PGCSEDS-IIITH/kafka-hackathon · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
version: '3'

services:
  zookeeper:
    image: wurstmeister/zookeeper
    ports:
      - "2181:2181"

  kafka:
    image: wurstmeister/kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: kafka
      KAFKA_CREATE_TOPICS: "tweets:1:1"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    depends_on:
      - zookeeper

  spark-master:
    image: bde2020/spark-master:3.1.1-hadoop3.2
    ports:
      - "8080:8080"
      - "7077:7077"
    environment:
      - INIT_DAEMON_STEP=setup_spark

  spark-worker:
    image: bde2020/spark-worker:3.1.1-hadoop3.2
    depends_on:
      - spark-master
    ports:
      - "8081:8081"
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"

  producer:
    build: .
    environment:
      BROKER: kafka:9092
    command: sh -c "spark-submit --master spark://spark-master:7077 send_data.py"
    depends_on:
      - spark-master
      - kafka

  consumer:
    build: .
    environment:
      BROKER: kafka:9092
    command: "spark-submit --master spark://spark-master:7077 consumer.py "
    depends_on:
      - spark-master
      - kafka
      - producer