docker-compose.yml

version: "3"

services:
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.1-java8
    container_name: namenode
    hostname: namenode
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
      - ./data:/data:rw
    environment:
      - CLUSTER_NAME=test
    ports:
      - 9870:9870
    env_file:
      - ./docker-compose/hadoop.env

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.1-java8
    container_name: datanode
    hostname: datanode
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    ports:
      - 9864:9864
    env_file:
      - ./docker-compose/hadoop.env
  
  resourcemanager:
    image: mnt/resourcemanager
    container_name: resourcemanager
    hostname: resourcemanager
    volumes:
      - hadoop_conf:/etc/hadoop
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
      MALLOC_ARENA_MAX: "1"
    ports:
      - 8088:8088
    env_file:
      - ./docker-compose/hadoop.env


  nodemanager-1:
    image: mnt/nodemanager
    container_name: nodemanager-1
    hostname: nodemanager-1
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
      MALLOC_ARENA_MAX: "1"
      YARN_CONF_yarn_nodemanager_webapp_address: "0.0.0.0:8041"
    ports:
      - "8041:8041"
    env_file:
      - ./docker-compose/hadoop.env

  nodemanager-2:
    image: mnt/nodemanager
    container_name: nodemanager-2
    hostname: nodemanager-2
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
      MALLOC_ARENA_MAX: "1"
      YARN_CONF_yarn_nodemanager_webapp_address: "0.0.0.0:8042"
    ports:
      - "8042:8042"
    env_file:
      - ./docker-compose/hadoop.env

  nodemanager-3:
    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.1-java8
    container_name: nodemanager-3
    hostname: nodemanager-3
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
      YARN_CONF_yarn_nodemanager_webapp_address: "0.0.0.0:8043"
    ports:
      - "8043:8043"
    env_file:
      - ./docker-compose/hadoop.env

  historyserver:
    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.1-java8
    container_name: historyserver
    hostname: historyserver
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
    ports:
      - 8188:8188
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    env_file:
      - ./docker-compose/hadoop.env

  spark-history:
    build:
      context: ./docker-compose/spark-history
    image: mnt/spark-history
    container_name: spark-history
    hostname: spark-history
    ports:
      - 18080:18080
    env_file:
      - ./docker-compose/hadoop.env

  spark-submit:
    build:
      context: ./docker-compose/spark-submit
    image: mnt/spark-submit
    container_name: spark-submit
    hostname: spark-submit
    environment:
      MALLOC_ARENA_MAX: "1"
      YARN_CONF_resourcemanager_address: "resourcemanager:8032"
      YARN_CONF_yarn_nodemanager_vmem__check__enabled: "false"
    env_file:
      - ./docker-compose/hadoop.env
    volumes:
      - ./jobs:/app:rw

volumes:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_historyserver:
  hadoop_conf: