docker-compose-hadoop.yml

version: "2.1"

services:
  namenode:
    build: ./namenode
    container_name: namenode
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
      - ./data/:/hadoop-data/input
      - ./map_reduce/:/hadoop-data/map_reduce
      - ./requirements.txt:/hadoop-data/requirements.txt
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop.env
    ports:
      - 9870:9870
      - 8020:8020
    networks:
      - hadoop_network

  resourcemanager:
    build: ./resourcemanager
    container_name: resourcemanager
    restart: on-failure
    depends_on:
      - namenode
      - datanode1
      - datanode2
      - datanode3
    env_file:
      - ./hadoop.env
    ports:
      - 8089:8088
    networks:
      - hadoop_network

  historyserver:
    build: ./historyserver
    container_name: historyserver
    depends_on:
      - namenode
      - datanode1
      - datanode2
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    env_file:
      - ./hadoop.env
    ports:
      - 8188:8188
    networks:
      - hadoop_network

  nodemanager1:
    build: ./nodemanager
    container_name: nodemanager1
    depends_on:
      - namenode
      - datanode1
      - datanode2
    env_file:
      - ./hadoop.env
    ports:
      - 8042:8042
    networks:
      - hadoop_network

  datanode1:
    build: ./datanode
    container_name: datanode1
    depends_on:
      - namenode
    volumes:
      - hadoop_datanode1:/hadoop/dfs/data
    env_file:
      - ./hadoop.env
    networks:
      - hadoop_network

  datanode2:
    build: ./datanode
    container_name: datanode2
    depends_on:
      - namenode
    volumes:
      - hadoop_datanode2:/hadoop/dfs/data
    env_file:
      - ./hadoop.env
    networks:
      - hadoop_network
  # datanode3:
  #   build: ./datanode
  #   container_name: datanode3
  #   depends_on:
  #     - namenode
  #   volumes:
  #     - hadoop_datanode3:/hadoop/dfs/data
  #   env_file:
  #     - ./hadoop.env
  #   networks:
  #     - hadoop_network

volumes:
  hadoop_namenode:
  hadoop_datanode1:
  hadoop_datanode2: # hadoop_datanode3:

  hadoop_historyserver:


networks:
  hadoop_network:
    name: hadoop_network
    external: true