LargeScaleDataManagement/docker-compose.yml at master · IakMastro/LargeScaleDataManagement · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
version: '3.9'

services:
    namenode:
        image: bde2020/hadoop-namenode
        container_name: namenode
        hostname: namenode
        ports:
            - 9870:9870
        volumes:
            - hadoop_namenode:/hadoop/dfs/name
        environment:
            - CLUSTER_NAME=test
        env_file:
            - ./hadoop.env
        networks:
            - hadoop_ecosystem

    datanode:
        image: bde2020/hadoop-datanode
        container_name: datanode
        volumes:
            - hadoop_datanode:/hadoop/dfs/data
        environment:
            SERVICE_PRECONDITION: "namenode:9870"
        env_file:
            - ./hadoop.env
        networks:
            - hadoop_ecosystem

    resourcemanager:
        image: bde2020/hadoop-resourcemanager
        container_name: resourcemanager
        hostname: resourcemanager
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
        env_file:
            - ./hadoop.env
        networks:
            - hadoop_ecosystem

    nodemanager:
        image: bde2020/hadoop-nodemanager
        container_name: nodemanager
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        env_file:
            - ./hadoop.env
        networks:
            - hadoop_ecosystem

    historyserver:
        image: bde2020/hadoop-historyserver
        container_name: historyserver
        environment:
            SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
        volumes:
            - hadoop_historyserver:/hadoop/yarn/timeline
        env_file:
            - ./hadoop.env
        networks:
            - hadoop_ecosystem

    zookeeper:
        image: docker.io/bitnami/zookeeper:3.7
        container_name: zookeeper
        hostname: zookeeper
        volumes:
            - "zookeeper_data:/bitnami"
        environment:
            - ALLOW_ANONYMOUS_LOGIN=yes
        networks:
            - hadoop_ecosystem

    kafka:
        image: docker.io/bitnami/kafka:2
        container_name: kafka
        hostname: kafka
        volumes:
            - "kafka_data:/bitnami"
        environment:
            - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
            - ALLOW_PLAINTEXT_LISTENER=yes
            - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
        depends_on:
            - zookeeper
        networks:
            - hadoop_ecosystem

    flume:
        image: lokkju/flume-hdfs
        container_name: flume
        hostname: flume
        volumes:
            - ./my_data/:/conf/
        environment:
            - FLUME_AGENT_NAME=KafkaAgent
            - FLUME_CONF_DIR=conf/
            - FLUME_CONF_FILE=conf/twitter.conf
        depends_on:
            - kafka
            - namenode
        networks:
            - hadoop_ecosystem

    java:
        build: .
        container_name: java
        depends_on:
            - namenode
        ports:
            - 8080:8080
        volumes:
            - ./my_files:/my_files
        networks:
            - hadoop_ecosystem

volumes:
    hadoop_namenode:
    hadoop_datanode:
    hadoop_historyserver:
    zookeeper_data:
    kafka_data:
    my_data:
    my_files:

networks:
    hadoop_ecosystem:
        name: hadoop_ecosystem