docker-compose-hbase-prod.yml

version: "3"

services:

    master:
        image: sauloricci/docker-spark-master-hbase
        command: bin/spark-class org.apache.spark.deploy.master.Master -h master
        hostname: master
        deploy:
            mode: global
            placement:
                constraints:
                    - node.labels.type == master
                    - node.labels.spark == true
        environment:
            MASTER: spark://master:7077
            SPARK_PUBLIC_DNS: $EXTERNAL_IP
        expose:
            - 7001
            - 7002
            - 7003
            - 7004
            - 7005
            - 7006
            - 7077
            - 6066
            - 8080
            - 4040
            - 5005
        volumes:
            - /mnt/spark_data:/data
            - /data1:/data1
            - /data2:/data2
            - /data3:/data3
            - /data4:/data4
            - /spark_logs:/spark_logs
        extra_hosts:
            - "kafka1:13.65.242.208"
            - "zk1:13.65.242.208"
            - "kafka2:13.84.50.174"
            - "zk2:13.84.50.174"
            - "kafka3:104.210.144.204"
            - "zk3:104.210.144.204"
        ports:
            - 5005:5005

    worker:
        image: sauloricci/docker-spark-worker-hbase
        command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
        hostname: worker
        deploy:
            mode: global
            placement:
                constraints: 
                    - node.labels.type == worker
                    - node.labels.spark == true
        environment:
            SPARK_WORKER_PORT: 8881
            SPARK_WORKER_WEBUI_PORT: 8081
            SPARK_PUBLIC_DNS: $EXTERNAL_IP
        links:
            - master
        expose:
            - 7012
            - 7013
            - 7014
            - 7015
            - 7016
            - 8881
        volumes:
            - /data1:/data1
            - /data2:/data2
            - /data3:/data3
            - /data4:/data4
            - /spark_logs:/spark_logs
        extra_hosts:
            - "kafka1:13.65.242.208"
            - "zk1:13.65.242.208"
            - "kafka2:13.84.50.174"
            - "zk2:13.84.50.174"
            - "kafka3:104.210.144.204"
            - "zk3:104.210.144.204"

    proxy:
       image: sauloricci/docker_spark_proxy:prod
       hostname: proxy
       deploy:
           placement:
               constraints: 
                - node.labels.type == master
                - node.labels.spark == true
       links:
           - master
       ports:
           - 666:666
           - 667:667
           - 4040:4040
           - 4041:4041
           - 4042:4042
           - 4043:4043
           - 4044:4044
           - 4045:4045
           - 4046:4046
           - 4047:4047
           - 4048:4048
           - 4049:4049
           - 4050:4050
           - 4051:4051
           - 4052:4052
           - 4053:4053
           - 4054:4054
           - 4055:4055