docker-compose.yml

version: "3"

services:

    master:
        image: sauloricci/docker-spark-master
        command: bin/spark-class org.apache.spark.deploy.master.Master -h master
        hostname: master
        deploy:
            mode: global
            placement:
                constraints:
                    - node.labels.type == master
                    - node.labels.spark == true
        environment:
            MASTER: spark://master:7077
            SPARK_PUBLIC_DNS: $EXTERNAL_IP
        expose:
            - 7001
            - 7002
            - 7003
            - 7004
            - 7005
            - 7006
            - 7077
            - 6066
            - 8080
            - 4040
            - 5005
        ports:
            - 5005:5005
        volumes:
            - /mnt/spark_data:/data
            - /data1:/data1
            - /data2:/data2
            - /data3:/data3
            - /data4:/data4
        extra_hosts:
            - "kafka1:13.65.242.208"
            - "zk1:13.65.242.208"
            - "kafka2:13.84.50.174"
            - "zk2:13.84.50.174"
            - "kafka3:104.210.144.204"
            - "zk3:104.210.144.204"

    worker:
        image: sauloricci/docker-spark-worker
        command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
        hostname: worker
        deploy:
            mode: global
            placement:
                constraints: 
                    - node.labels.type == worker
                    - node.labels.spark == true
        environment:
            SPARK_WORKER_PORT: 8881
            SPARK_WORKER_WEBUI_PORT: 8081
            SPARK_PUBLIC_DNS: $EXTERNAL_IP
        links:
            - master
        expose:
            - 7012
            - 7013
            - 7014
            - 7015
            - 7016
            - 8881
        volumes:
            - /data1:/data1
            - /data2:/data2
            - /data3:/data3
            - /data4:/data4
        extra_hosts:
            - "kafka1:13.65.242.208"
            - "zk1:13.65.242.208"
            - "kafka2:13.84.50.174"
            - "zk2:13.84.50.174"
            - "kafka3:104.210.144.204"
            - "zk3:104.210.144.204"

    notebook:
        image: sauloricci/all-spark-notebook
        command: start-notebook.sh --NotebookApp.token='' --NotebookApp.ip='notebook' --NotebookApp.allow_origin='*'
        hostname: notebook
        deploy:
            placement:
                constraints: 
                    - node.labels.type == master
                    - node.labels.spark == true

        environment:
            TINI_SUBREAPER: "true"
            GRANT_SUDO: "yes"
            SPARK_PUBLIC_DNS: $EXTERNAL_IP
            SPARK_OPTS: "--master=spark://master:7077"
        links:
            - master
        expose:
            - 8888
        pid: "host"
        volumes:
            - /notebook:/home/jovyan/work
    
    proxy:
       image: sauloricci/docker_spark_proxy
       hostname: proxy
       deploy:
           placement:
               constraints: 
                - node.labels.type == master
                - node.labels.spark == true
       links:
           - master
           - notebook
       ports:
           - 666:666
           - 667:667
           - 4040:4040
           - 4041:4041