diff --git a/group_vars/all b/group_vars/all index a363767..d10d866 100644 --- a/group_vars/all +++ b/group_vars/all @@ -7,6 +7,9 @@ # Do you have internet access on your cluster ? is_offline: false +# Build Hadoop from source +build_hadoop: false + # This should stay as false. Will Format HDFS otherwise bootstrap: false diff --git a/roles/hadoop/tasks/main.yml b/roles/hadoop/tasks/main.yml new file mode 100644 index 0000000..9aed61a --- /dev/null +++ b/roles/hadoop/tasks/main.yml @@ -0,0 +1,124 @@ +--- +- name: Download Hadoop's source code + url: "{{ hadoop_src_archive_url }}" + dest: "{{ download_dir }}/{{ hadoop_src_archive }}" + checksum: md5:"{{ hadoop_src_md5 }}" + when: build_hadoop == True + +- name: Unarchive Hadoop's source code archive + unarchive: + src: "{{ download_dir }}/{{ hadoop_archive }}" + dest: "{{ download_dir }}" + copy: no + creates: "{{ download_dir }}/hadoop-{{ hadoop_version }}" + when: build_hadoop == True + +- name: Install dependency packages + yum: + name: "{{ item }}" + state: latest + with_items: + - libselinux-python + - nmap-ncat + - maven + - gcc + - gcc-c++ + - snappy + - snappy-devel + - bzip2 + - bzip2-devel + - bzip2-libs + - lz4 + - lz4-devel + - zlib + - zlib-devel + - lzo + - lzo-devel + - automake + - libtool + - openssl + - openssl-devel + - fuse + - fuse-devel + - cmake + - protobuf + - protobuf-devel + - protobuf-compiler + - zstd + - libzstd + - libzstd-devel + - jansson-devel + - jansson + - lzop + - autoconf + when: build_hadoop == True + +- name: Install dependency packages + yum: + name: "{{ item }}" + state: latest + with_items: + - libselinux-python + - java-1.8.0-openjdk-devel + - nmap-ncat + - lzo-devel + +- name: Build Hadoop from source code + command: mvn package -Pdist,native -Drequire.snappy -Drequire.openssl -DskipTests -Dtar + args: + chdir: "{{ download_dir }}/hadoop-{{ hadoop_version }}" + when: build_hadoop == True + +- group: + name: hadoop + system: yes + +- user: + name: hadoop + group: hadoop + createhome: no + system: yes + +- name: Ensure Hadoop directories exist + file: + path: "{{ item }}" + state: directory + owner: hadoop + group: hadoop + mode: 0774 + with_items: + - "{{ hadoop_home }}" + - "{{ hadoop_conf_dir }}" + - "{{ hadoop_log_dir }}" + +- name: Copy Hadoop binary archive + copy: + src: "{{ download_dir }}/hadoop-{{ hadoop_version }}/hadoop-dist/target/hadoop-{{ hadoop_version }}.tar.gz" + dest: "{{ hadoop_home }}-{{ hadoop_version }}.tar.gz" + when: build_hadoop == True + +- name: Download hadoop binary + get_url: + url: "{{ hadoop_bin_archive_url }}" + dest: "{{ download_dir }}/{{ hadoop_bin_archive }}" + checksum: md5:"{{ hadoop_bin_md5 }}" + when: build_hadoop == False + +- name: Unarchive hadoop archive + unarchive: + src: "{{ hadoop_home }}-{{ hadoop_version }}.tar.gz" +# dest: "/usr/local" + dest: "{{ hadoop_home }}-{{ hadoop_version }}" + copy: no + owner: hadoop + group: hadoop +# creates: "{{ hadoop_home }}-{{ hadoop_version }}" + +- name: Link generic hadoop directory to specific version directory + file: + path: "{{ hadoop_home }}" + src: "{{ hadoop_home }}-{{ hadoop_version }}" + owner: hadoop + group: hadoop + state: link + force: yes diff --git a/roles/hadoop/vars/main.yml b/roles/hadoop/vars/main.yml new file mode 100644 index 0000000..94f282a --- /dev/null +++ b/roles/hadoop/vars/main.yml @@ -0,0 +1,16 @@ +--- +# Download configuration +hadoop_version: 2.7.4 +hadoop_url: "http://apache.mediamirrors.org/hadoop/common" +hadoop_src_archive: "hadoop-{{ hadoop_version }}-src.tar.gz" +hadoop_bin_archive: "hadoop-{{ hadoop_version }}.tar.gz" +hadoop_src_archive_url: "{{ hadoop_url }}/hadoop-{{ hadoop_version }}/{{ hadoop_src_archive }}" +hadoop_bin_archive_url: "{{ hadoop_url }}/hadoop-{{ hadoop_version }}/{{ hadoop_bin_archive }}" +hadoop_src_md5: 93424b23e4203db31ed4e7b4eaf0ac1e +hadoop_bin_md5: ee3cc9f384b787ac13ad3b1335920924 +download_dir: /opt/src + +# Hadoop directories +hadoop_home: /usr/local/hadoop +hadoop_conf_dir: /etc/hadoop +hadoop_log_dir: /var/log/hadoop diff --git a/roles/hdfs/tasks/main.yml b/roles/hdfs/tasks/main.yml index db0642d..49496a2 100644 --- a/roles/hdfs/tasks/main.yml +++ b/roles/hdfs/tasks/main.yml @@ -1,80 +1,19 @@ --- -- name: Install package - yum: name={{item}} state=latest - with_items: - - libselinux-python - - java-1.8.0-openjdk-devel - - nmap-ncat - - lzo-devel - -- group: name=hadoop system=yes -- group: name=hdfs system=yes -- group: name=yarn system=yes - - user: - name: hadoop + name: hdfs group: hadoop createhome: no system: yes -- user: - name: hdfs - group: hdfs - groups: hadoop - home: "{{hdfs_workdir}}" - system: yes - -- user: - name: yarn - group: yarn - groups: hadoop - home: "{{yarn_workdir}}" - system: yes - -- name: Ensure download directory exists - file: path={{hadoop_download_dir}} state=directory - -- name: Ensure hadoop directories exists - file: path={{item}} state=directory - with_items: - - "{{hadoop_lib_dir}}" - - "{{hadoop_conf_dir}}" - -- name: Ensure hadoop log directories exists - file: - path: "{{hadoop_log_dir}}" - state: directory - owner: hadoop - group: hadoop - mode: 0774 - -- name: Download hadoop binary - get_url: - url: "{{hadoop_archive_url}}" - dest: "{{hadoop_download_dir}}/{{hadoop_archive}}" - checksum: md5:{{hadoop_md5}} - -- name: Unarchive hadoop archive - unarchive: - src: "{{hadoop_download_dir}}/{{hadoop_archive}}" - dest: "{{hadoop_lib_dir}}" - copy: no - owner: hadoop - group: hadoop - creates: "{{hadoop_lib_dir}}/hadoop-{{hadoop_version}}" - -- name: Link generic hadoop dir to specific version directory - file: - path: "{{hadoop_home}}" - src: "{{hadoop_home}}-{{hadoop_version}}" - owner: hadoop - group: hadoop - state: link +- name: Add HADOOP_CONF_DIR as environment variable + lineinfile: + dest: /etc/profile + line: "export HADOOP_CONF_DIR={{ hadoop_conf_dir }}" - name: Write configuration files template: - src: "{{item}}.j2" - dest: "{{hadoop_conf_dir}}/{{item}}" + src: "{{ item }}.j2" + dest: "{{ hadoop_conf_dir }}/{{ item }}" owner: hadoop group: hadoop with_items: @@ -82,125 +21,124 @@ - core-site.xml - hdfs-site.xml - log4j.properties + - slaves + - masters -- name: Remove old conf - file: - path: "{{hadoop_home}}/etc/hadoop" - state: absent - -- name: Link configuration directories - file: - path: "{{hadoop_home}}/etc/hadoop" - src: "{{hadoop_conf_dir}}" - state: link - force: yes - -- name: add hadoop binaries in PATH +- name: Add hadoop binaries in PATH lineinfile: dest: /etc/profile - line: export PATH={{hadoop_home}}/bin:{{hadoop_home}}/sbin:$PATH + line: 'export PATH={{ hadoop_home }}/bin:{{ hadoop_home }}/sbin:$PATH' -- name: Create namenode data directory +- name: Create Namenode data directory file: - path: "{{item}}" + path: "{{ item }}" state: directory owner: hdfs group: hadoop recurse: yes mode: 0700 - with_items: "{{namenode_dirs}}" + with_items: "{{ namenode_dirs }}" when: "inventory_hostname == namenode_hostname" -- name: Create secondary data directory +- name: Create Secondary Namenode data directory file: - path: "{{item}}" + path: "{{ item }}" state: directory owner: hdfs group: hadoop recurse: yes mode: 0700 - with_items: "{{secondnn_dirs}}" + with_items: "{{ secondnn_dirs }}" when: "inventory_hostname == secondnn_hostname" -- name: Add namenode service file +- name: Ensure HDFS state information directory exists + file: + path: "{{ hdfs_workdir }}" + state: directory + owner: hdfs + group: hadoop + mode: 0755 + +- name: Add Namenode service file template: src: namenode.service.j2 dest: /etc/systemd/system/namenode.service when: "inventory_hostname == namenode_hostname" -- name: Add secondary service file +- name: Add Secondary Namenode service file template: - src: secondary.service.j2 - dest: /etc/systemd/system/secondary.service - when: "inventory_hostname == secondnn_hostname" + src: secondary-namenode.service.j2 + dest: /etc/systemd/system/secondary-namenode.service + when: "inventory_hostname == secondary_namenode_hostname" -- name: Create datanode data directory +- name: Create Datanode data directory file: - path: "{{item}}" + path: "{{ item }}" state: directory owner: hdfs group: hadoop recurse: yes mode: 0700 - with_items: "{{datanode_dirs}}" + with_items: "{{ datanode_dirs }}" when: "'slaves' in group_names" -- name: Add datanode service file +- name: Add Datanode service file template: src: datanode.service.j2 dest: /etc/systemd/system/datanode.service when: "'slaves' in group_names" -- name: Format namenode - shell: "{{hadoop_home}}/bin/hdfs namenode -format" +- name: Format Namenode + shell: "{{ hadoop_home }}/bin/hdfs namenode -format" become_user: hdfs when: - "inventory_hostname == namenode_hostname" - - bootstrap + - bootstrap == True -- name: Start namenode +- name: Start Namenode service: name: namenode enabled: yes state: started when: "inventory_hostname == namenode_hostname" -- name: Start secondary namenode +- name: Start Secondary Namenode service: - name: secondary + name: secondary-namenode enabled: yes state: started - when: "inventory_hostname == secondnn_hostname" + when: "inventory_hostname == secondary_namenode_hostname" -- name: Start datanode +- name: Start Datanode service: name: datanode enabled: yes state: started when: "'slaves' in group_names" -- name: add hadoop user to group +- name: Add Hadoop user to group user: - name: "{{item}}" + name: "{{ item }}" append: yes groups: hadoop - with_items: "{{hadoop_users}}" + with_items: "{{ hadoop_users }}" - name: Create base directories - shell: "{{hadoop_home}}/bin/hadoop fs -mkdir {{item}}" + shell: "{{ hadoop_home }}/bin/hadoop fs -mkdir {{ item }}" become_user: hdfs when: - "inventory_hostname == namenode_hostname" - - bootstrap + - bootstrap == True with_items: - /shared - /tmp - /user + - /var - name: Create user directories - shell: "{{hadoop_home}}/bin/hadoop fs -mkdir /user/{{item}}" - become_user: "{{item}}" + shell: "{{ hadoop_home }}/bin/hadoop fs -mkdir /user/{{ item }}" + become_user: "{{ item }}" when: - "inventory_hostname == namenode_hostname" - - bootstrap - with_items: "{{hadoop_users}}" + - bootstrap == True + with_items: "{{ hadoop_users }}" diff --git a/roles/hdfs/templates/core-site.xml.j2 b/roles/hdfs/templates/core-site.xml.j2 index d014957..6c3f56b 100644 --- a/roles/hdfs/templates/core-site.xml.j2 +++ b/roles/hdfs/templates/core-site.xml.j2 @@ -20,7 +20,7 @@ fs.defaultFS - {{namenode_uri}} + {{ namenode_uri }} fs.trash.interval diff --git a/roles/hdfs/templates/datanode.service.j2 b/roles/hdfs/templates/datanode.service.j2 index 5062e62..abd3bc5 100644 --- a/roles/hdfs/templates/datanode.service.j2 +++ b/roles/hdfs/templates/datanode.service.j2 @@ -9,8 +9,9 @@ After=network.target remote-fs.target Type=forking User=hdfs Group=hadoop -ExecStart={{hadoop_home}}/sbin/hadoop-daemon.sh start datanode -WorkingDirectory={{hdfs_workdir}} +ExecStart={{ hadoop_home }}/sbin/hadoop-daemon.sh start datanode +ExecStop={{ hadoop_home }}/sbin/hadoop-daemon.sh stop datanode +WorkingDirectory={{ hdfs_workdir }} [Install] WantedBy=multi-user.target diff --git a/roles/hdfs/templates/hadoop-env.sh.j2 b/roles/hdfs/templates/hadoop-env.sh.j2 index 44bf7ed..d5fc97d 100644 --- a/roles/hdfs/templates/hadoop-env.sh.j2 +++ b/roles/hdfs/templates/hadoop-env.sh.j2 @@ -91,7 +91,7 @@ export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} # NOTE: this should be set to a directory that can only be written to by # the user that will run the hadoop daemons. Otherwise there is the # potential for a symlink attack. -export HADOOP_PID_DIR=${HADOOP_PID_DIR} +export HADOOP_PID_DIR={{ hadoop_run_dir }} export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR} # A string representing this instance of hadoop. $USER by default. diff --git a/roles/hdfs/templates/hdfs-site.xml.j2 b/roles/hdfs/templates/hdfs-site.xml.j2 index 20904a3..eeb4fd4 100644 --- a/roles/hdfs/templates/hdfs-site.xml.j2 +++ b/roles/hdfs/templates/hdfs-site.xml.j2 @@ -20,19 +20,19 @@ dfs.namenode.name.dir - file://{{namenode_dirs | join(',file://')}} - > + file://{{ namenode_dirs | join(',file://') }} + dfs.namenode.checkpoint.dir - file://{{secondnn_dirs | join(',file://')}} + file://{{ secondar_namenode_dirs | join(',file://') }} dfs.datanode.data.dir - file://{{datanode_dirs | join(',file://')}} + file://{{ datanode_dirs | join(',file://') }} dfs.datanode.failed.volumes.tolerated - {{failed_volumes}} + {{ failed_volumes }} dfs.permissions.superusergroup diff --git a/roles/hdfs/templates/masters.j2 b/roles/hdfs/templates/masters.j2 new file mode 100644 index 0000000..bac0ed4 --- /dev/null +++ b/roles/hdfs/templates/masters.j2 @@ -0,0 +1,3 @@ +{% for host in groups['masters'] %} +{{ host }} +{% endfor %} diff --git a/roles/hdfs/templates/namenode.service.j2 b/roles/hdfs/templates/namenode.service.j2 index 9077c1a..4b53544 100644 --- a/roles/hdfs/templates/namenode.service.j2 +++ b/roles/hdfs/templates/namenode.service.j2 @@ -9,8 +9,9 @@ After=network.target remote-fs.target zookeeper.service Type=forking User=hdfs Group=hadoop -ExecStart={{hadoop_home}}/sbin/hadoop-daemon.sh start namenode -WorkingDirectory={{hdfs_workdir}} +ExecStart={{ hadoop_home }}/sbin/hadoop-daemon.sh start namenode +ExecStop={{ hadoop_home }}/sbin/hadoop-daemon.sh stop namenode +WorkingDirectory={{ hdfs_workdir }} [Install] WantedBy=multi-user.target diff --git a/roles/hdfs/templates/secondary.service.j2 b/roles/hdfs/templates/secondary-namenode.service.j2 similarity index 65% rename from roles/hdfs/templates/secondary.service.j2 rename to roles/hdfs/templates/secondary-namenode.service.j2 index 338a041..78e7262 100644 --- a/roles/hdfs/templates/secondary.service.j2 +++ b/roles/hdfs/templates/secondary-namenode.service.j2 @@ -9,8 +9,9 @@ After=network.target remote-fs.target zookeeper.service Type=forking User=hdfs Group=hadoop -ExecStart={{hadoop_home}}/sbin/hadoop-daemon.sh start secondarynamenode -WorkingDirectory={{hdfs_workdir}} +ExecStart={{ hadoop_home }}/sbin/hadoop-daemon.sh start secondarynamenode +ExecStop={{ hadoop_home }}/sbin/hadoop-daemon.sh stop secondarynamenode +WorkingDirectory={{ hdfs_workdir }} [Install] WantedBy=multi-user.target diff --git a/roles/hdfs/vars/main.yml b/roles/hdfs/vars/main.yml index 79c2572..5d5d6da 100644 --- a/roles/hdfs/vars/main.yml +++ b/roles/hdfs/vars/main.yml @@ -1,25 +1,16 @@ --- -# Download configuration -hadoop_version: 2.7.3 -hadoop_download_dir: /opt/src -hadoop_url: "{{mirror_base_url}}/hadoop/common" -hadoop_archive: hadoop-{{hadoop_version}}.tar.gz -hadoop_archive_url: "{{hadoop_url}}/hadoop-{{hadoop_version}}/{{hadoop_archive}}" -hadoop_md5: 3455BB57E4B4906BBEA67B58CCA78FA8 - -# Install configuration -hdfs_workdir: /var/lib/hdfs -yarn_workdir: /var/lib/yarn - -hadoop_lib_dir: /usr/local/lib -hadoop_home: /usr/local/lib/hadoop +# Hadoop directories +hadoop_home: /usr/local/hadoop hadoop_conf_dir: /etc/hadoop hadoop_log_dir: /var/log/hadoop +hdfs_workdir: /var/local/hdfs +hadoop_run_dir: /var/run/hadoop + # Global architecture isHA: no -namenode_hostname: "{{hostvars[groups['masters'][0]]['inventory_hostname']}}" -secondnn_hostname: "{{hostvars[groups['masters'][1]]['inventory_hostname']}}" +namenode_hostname: "{{ hostvars[groups['masters'][0]]['inventory_hostname'] }}" +secondary_namenode_hostname: "{{ hostvars[groups['masters'][1]]['inventory_hostname'] }}" datanode_group: slaves # hadoop-env.sh variables @@ -28,13 +19,13 @@ hadoop_heapsize: 1024 hadoop_namenode_heapsize: "-Xms1024m" # core-site.xml variables -namenode_uri: hdfs://{{hostvars[groups['masters'][0]]['inventory_hostname']}} +namenode_uri: "hdfs://{{ hostvars[groups['masters'][0]]['inventory_hostname'] }}" # hdfs-site.xml variables list here the relevant directories namenode_dirs: - - "{{hdfs_workdir}}/nn" -secondnn_dirs: - - "{{hdfs_workdir}}/sn" + - "{{ hdfs_workdir }}/nn" +secondary_namenode_dirs: + - "{{ hdfs_workdir }}/sn" datanode_dirs: # typically here you should have something like : # - /disk1/hdfs/dn diff --git a/roles/yarn/tasks/main.yml b/roles/yarn/tasks/main.yml new file mode 100644 index 0000000..129ccfc --- /dev/null +++ b/roles/yarn/tasks/main.yml @@ -0,0 +1,63 @@ +--- +- user: + name: yarn + group: hadoop + createhome: no + system: yes + +- name: Write configuration files + template: + src: "{{ item }}.j2" + dest: "{{ hadoop_conf_dir }}/{{ item }}" + owner: yarn + group: hadoop + with_items: + - mapred-site.xml + - yarn-site.xml + +- name: Ensure YARN state information directory exists + file: + path: "{{ yarn_workdir }}" + state: directory + owner: yarn + group: hadoop + mode: 0755 + +- name: Add Resource Manager service file + template: + src: resource-manager.service.j2 + dest: /etc/systemd/system/resource-manager.service + when: "inventory_hostname == resource_manager_hostname" + +- name: Add Node Manager service file + template: + src: node-manager.service.j2 + dest: /etc/systemd/system/node-manager.service + when: "'slaves' in group_names" + +- name: Add Job History Server service file + template: + src: job-history-server.service.j2 + dest: /etc/systemd/system/job-history-server.service + when: "inventory_hostname == job_history_server_hostname" + +- name: Start Resource Manager + service: + name: resource-manager + enabled: yes + state: started + when: "inventory_hostname == resource_manager_hostname" + +- name: Start Node manager + service: + name: node-manager + enabled: yes + state: started + when: "'slaves' in group_names" + +- name: Start Job History Server + service: + name: job-history-server + enabled: yes + state: started + when: "inventory_hostname == job_history_server_hostname" diff --git a/roles/yarn/templates/job-history-server.service.j2 b/roles/yarn/templates/job-history-server.service.j2 new file mode 100644 index 0000000..5eedc6e --- /dev/null +++ b/roles/yarn/templates/job-history-server.service.j2 @@ -0,0 +1,17 @@ +# {{ansible_managed}} +[Unit] +Description=Apache Hadoop Job History Server service +Documentation=http://hadoop.apache.org +Requires=network.target remote-fs.target +After=network.target remote-fs.target + +[Service] +Type=forking +User=yarn +Group=hadoop +ExecStart={{ hadoop_home }}/sbin/mr-jobhistory-daemon.sh start historyserver +ExecStop={{ hadoop_home }}/sbin/mr-jobhistory-daemon.sh stop historyserver +WorkingDirectory={{ yarn_workdir }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/hdfs/templates/mapred-site.xml.j2 b/roles/yarn/templates/mapred-site.xml.j2 similarity index 91% rename from roles/hdfs/templates/mapred-site.xml.j2 rename to roles/yarn/templates/mapred-site.xml.j2 index e653ada..6952d06 100644 --- a/roles/hdfs/templates/mapred-site.xml.j2 +++ b/roles/yarn/templates/mapred-site.xml.j2 @@ -24,11 +24,11 @@ limitations under the License. mapreduce.jobhistory.address - master03:10020 + {{ job_history_server_hostname }}:10020 mapreduce.jobhistory.webapp.address - master03:19888 + {{ job_history_server_hostname }}:19888 yarn.app.mapreduce.am.staging-dir diff --git a/roles/yarn/templates/node-manager.service.j2 b/roles/yarn/templates/node-manager.service.j2 new file mode 100644 index 0000000..f1f5028 --- /dev/null +++ b/roles/yarn/templates/node-manager.service.j2 @@ -0,0 +1,17 @@ +# {{ansible_managed}} +[Unit] +Description=Apache Hadoop Node Manager service +Documentation=http://hadoop.apache.org +Requires=network.target remote-fs.target +After=network.target remote-fs.target + +[Service] +Type=forking +User=yarn +Group=hadoop +ExecStart={{ hadoop_home }}/sbin/yarn-daemon.sh start nodemanager +ExecStop={{ hadoop_home }}/sbin/yarn-daemon.sh stop nodemanager +WorkingDirectory={{ yarn_workdir }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/yarn/templates/proxy-server.service.j2 b/roles/yarn/templates/proxy-server.service.j2 new file mode 100644 index 0000000..e452b85 --- /dev/null +++ b/roles/yarn/templates/proxy-server.service.j2 @@ -0,0 +1,17 @@ +# {{ansible_managed}} +[Unit] +Description=Apache Hadoop Proxy Server service +Documentation=http://hadoop.apache.org +Requires=network.target remote-fs.target zookeeper.service +After=network.target remote-fs.target zookeeper.service + +[Service] +Type=forking +User=yarn +Group=hadoop +ExecStart={{ hadoop_home }}/sbin/yarn-daemon.sh start proxyserver +ExecStop={{ hadoop_home }}/sbin/yarn-daemon.sh stop proxyserver +WorkingDirectory={{ yarn_workdir }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/yarn/templates/resource-manager.service.j2 b/roles/yarn/templates/resource-manager.service.j2 new file mode 100644 index 0000000..12b1db9 --- /dev/null +++ b/roles/yarn/templates/resource-manager.service.j2 @@ -0,0 +1,17 @@ +# {{ansible_managed}} +[Unit] +Description=Apache Hadoop Resource Manager service +Documentation=http://hadoop.apache.org +Requires=network.target remote-fs.target +After=network.target remote-fs.target + +[Service] +Type=forking +User=yarn +Group=hadoop +ExecStart={{ hadoop_home }}/sbin/yarn-daemon.sh start resourcemanager +ExecStop={{ hadoop_home }}/sbin/yarn-daemon.sh stop resourcemanager +WorkingDirectory={{ yarn_workdir }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/hdfs/templates/yarn-site.xml.j2 b/roles/yarn/templates/yarn-site.xml.j2 similarity index 98% rename from roles/hdfs/templates/yarn-site.xml.j2 rename to roles/yarn/templates/yarn-site.xml.j2 index 1d013d5..93698b0 100644 --- a/roles/hdfs/templates/yarn-site.xml.j2 +++ b/roles/yarn/templates/yarn-site.xml.j2 @@ -20,7 +20,7 @@ yarn.resourcemanager.hostname - master03.local + {{ resource_manager_hostname }} @@ -46,7 +46,7 @@ yarn.nodemanager.remote-app-log-dir hdfs:/var/log/hadoop-yarn/apps - + yarn.log-aggregation-enable true diff --git a/roles/yarn/vars/main.yml b/roles/yarn/vars/main.yml new file mode 100644 index 0000000..5faef81 --- /dev/null +++ b/roles/yarn/vars/main.yml @@ -0,0 +1,19 @@ +--- +# Hadoop directories +hadoop_home: /usr/local/hadoop +hadoop_conf_dir: /etc/hadoop +hadoop_log_dir: /var/log/hadoop + +yarn_workdir: /var/local/yarn + +# Global architecture +resource_manager_hostname: "{{ hostvars[groups['masters'][0]]['inventory_hostname'] }}" +job_history_server_hostname: "{{ hostvars[groups['masters'][1]]['inventory_hostname'] }}" + +# yarn-env.sh variables + +# mapred-env.sh variables + +# mapred-site.xml variables + +# yarn-site.xml variables list here the relevant directories