2024-05-08 hadoop 配置 version=1

namenode.yaml

startDelaySeconds: 0

hostPort: 192.168.12.131:1234

name: NameNode

rules:

  - pattern: 'Hadoop NameNode:<name=(NameNode)><>(.+)'

    name: hadoop_namenode_info


 mapred-site.xml


<?xml version="1.0"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!--

  Licensed under the Apache License, Version 2.0 (the "License");

  you may not use this file except in compliance with the License.

  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software

  distributed under the License is distributed on an "AS IS" BASIS,

  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  See the License for the specific language governing permissions and

  limitations under the License. See accompanying LICENSE file.

-->

<!-- Put site-specific property overrides in this file. -->

<configuration>

  <property>

                <name>mapreduce.framework.name</name>

                <value>yarn</value>

        </property>

        <property>

          <name>mapreduce.jobhistory.address</name>

          <value>datanebula9:10020</value>

        </property>

        <property>

          <name>mapreduce.jobhistory.webapp.address</name>

          <value>datanebula9:19888</value>

        </property>

    <!-- 指定MapReduce作业执行时,需要使用到的路径 -->

    <property>

        <name>yarn.app.mapreduce.am.env</name>

        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>

    </property>

    <property>

        <name>mapreduce.map.env</name>

        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>

    </property>

    <property>

        <name>mapreduce.reduce.env</name>

        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>

    </property>

    <property>

          <name>mapred.child.java.opts</name>

          <value>-Xmx2048m</value>

    </property>

</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!--

  Licensed under the Apache License, Version 2.0 (the "License");

  you may not use this file except in compliance with the License.

  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software

  distributed under the License is distributed on an "AS IS" BASIS,

  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  See the License for the specific language governing permissions and

  limitations under the License. See accompanying LICENSE file.

-->

<!-- Put site-specific property overrides in this file. -->

<configuration>

  <!-- 指定hdfs的nameservice为zszchdfs,需要和core-site.xml中的保持一致 -->

  <property>

    <name>dfs.nameservices</name>

    <value>zszchdfs</value>

  </property>

  <!-- hdfs namenode集群zszchdfs下的两个namenode实例别名 -->

  <property>

    <name>dfs.ha.namenodes.zszchdfs</name>

    <value>nn1,nn2</value>

  </property>

  <!-- nn1的RPC通信地址 -->

  <property>

    <name>dfs.namenode.rpc-address.zszchdfs.nn1</name>

    <value>datanebula1:8020</value>

  </property>

  <!-- nn2的RPC通信地址 -->

  <property>

    <name>dfs.namenode.rpc-address.zszchdfs.nn2</name>

    <value>datanebula2:8020</value>

  </property>

  <!-- datanode数据节点存储数据目录 -->

  <property>

    <name>dfs.datanode.data.dir</name>

    <value>file:///home/sdb1/hadoop/data/hdfs</value>

  </property>

  <!-- namenode节点存储元数据目录 -->

  <property>

    <name>dfs.namenode.name.dir</name>

    <value>file:///home/sdb1/hadoop/data/namenode</value>

  </property>

  <!-- nn1的http访问地址 -->

  <property>

    <name>dfs.namenode.http-address.zszchdfs.nn1</name>

    <value>datanebula1:9870</value>

  </property>

  <!-- nn2的http访问地址 -->

  <property>

    <name>dfs.namenode.http-address.zszchdfs.nn2</name>

    <value>datanebula2:9870</value>

  </property>

  <!-- hdfs副本个数 -->

  <property>

    <name>dfs.replication</name>

    <value>2</value>

  </property>

  <!-- jn服务 -->

  <property>

    <name>dfs.namenode.shared.edits.dir</name>

    <value>qjournal://datanebula7:8485;datanebula8:8485;datanebula9:8485/zszchdfs-joural</value>

  </property>

  <!-- jn edits 在本地磁盘存放数据的位置 -->

  <property>

    <name>dfs.journalnode.edits.dir</name>

    <value>/home/sdb1/hadoop/data/journal/edits</value>

  </property>

  <!-- 开启NameNode失败自动切换 -->

  <property>

    <name>dfs.ha.automatic-failover.enabled</name>

    <value>true</value>

  </property>

  <!-- 配置失败自动切换实现方式 -->

  <property>

    <name>dfs.client.failover.proxy.provider.zszchdfs</name>

    <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

  </property>

  <!-- 配置隔离机制 -->

  <property>

    <name>dfs.ha.fencing.methods</name>

    <value>sshfence</value>

  </property>

  <property>

    <name>dfs.ha.fencing.ssh.connect-timeout</name>

    <value>1000</value>

  </property>

  <!-- 使用隔离机制时需要ssh免登陆 -->

  <property>

    <name>dfs.ha.fencing.ssh.private-key-files</name>

    <value>/root/.ssh/id_rsa</value>

  </property>

  <!-- hadoop webhdfs服务开启和使用 -->

  <property>

    <name>dfs.webhdfs.enabled</name>

    <value>true</value>

  </property>

  <!-- DataNode用来连接NameNode的RPC请求的线程数量 -->

  <property>

    <name>dfs.namenode.handler.count</name>

    <value>30</value>

  </property>

  <!-- DataNode可以同时处理的数据传输连接数 -->

  <property>

    <name>dfs.datanode.max.xcievers</name>

    <value>1024</value>

  </property>

  <!-- 关闭权限检查-->

  <property>

    <name>dfs.permissions.enable</name>

    <value>false</value>

  </property>

</configuration>


 hadoop-env.sh

#

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements.  See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership.  The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License.  You may obtain a copy of the License at

#

#    http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# Set Hadoop-specific environment variables here.

##

## THIS FILE ACTS AS THE MASTER FILE FOR ALL HADOOP PROJECTS.

## SETTINGS HERE WILL BE READ BY ALL HADOOP COMMANDS.  THEREFORE,

## ONE CAN USE THIS FILE TO SET YARN, HDFS, AND MAPREDUCE

## CONFIGURATION OPTIONS INSTEAD OF xxx-env.sh.

##

## Precedence rules:

##

## {yarn-env.sh|hdfs-env.sh} > hadoop-env.sh > hard-coded defaults

##

## {YARN_xyz|HDFS_xyz} > HADOOP_xyz > hard-coded defaults

##

# Many of the options here are built from the perspective that users

# may want to provide OVERWRITING values on the command line.

# For example:

#

#  JAVA_HOME=/usr/java/testing hdfs dfs -ls

#

# Therefore, the vast majority (BUT NOT ALL!) of these defaults

# are configured for substitution and not append.  If append

# is preferable, modify this file accordingly.

###

# Generic settings for HADOOP

###

# Technically, the only required environment variable is JAVA_HOME.

# All others are optional.  However, the defaults are probably not

# preferred.  Many sites configure these options outside of Hadoop,

# such as in /etc/profile.d

# The java implementation to use. By default, this environment

# variable is REQUIRED on ALL platforms except OS X!

export JAVA_HOME=/opt/module/java/jdk1.8.0_212

# Location of Hadoop.  By default, Hadoop will attempt to determine

# this location based upon its execution path.

# export HADOOP_HOME=

# Location of Hadoop's configuration information.  i.e., where this

# file is living. If this is not defined, Hadoop will attempt to

# locate it based upon its execution path.

#

# NOTE: It is recommend that this variable not be set here but in

# /etc/profile.d or equivalent.  Some options (such as

# --config) may react strangely otherwise.

#

# export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop

# The maximum amount of heap to use (Java -Xmx).  If no unit

# is provided, it will be converted to MB.  Daemons will

# prefer any Xmx setting in their respective _OPT variable.

# There is no default; the JVM will autoscale based upon machine

# memory size.

# export HADOOP_HEAPSIZE_MAX=

# The minimum amount of heap to use (Java -Xms).  If no unit

# is provided, it will be converted to MB.  Daemons will

# prefer any Xms setting in their respective _OPT variable.

# There is no default; the JVM will autoscale based upon machine

# memory size.

# export HADOOP_HEAPSIZE_MIN=

# Enable extra debugging of Hadoop's JAAS binding, used to set up

# Kerberos security.

# export HADOOP_JAAS_DEBUG=true

# Extra Java runtime options for all Hadoop commands. We don't support

# IPv6 yet/still, so by default the preference is set to IPv4.

# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"

# For Kerberos debugging, an extended option set logs more invormation

# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug"

# Some parts of the shell code may do special things dependent upon

# the operating system.  We have to set this here. See the next

# section as to why....

export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}

# Extra Java runtime options for some Hadoop commands

# and clients (i.e., hdfs dfs -blah).  These get appended to HADOOP_OPTS for

# such commands.  In most cases, # this should be left empty and

# let users supply it on the command line.

# export HADOOP_CLIENT_OPTS=""

#

# A note about classpaths.

#

# By default, Apache Hadoop overrides Java's CLASSPATH

# environment variable.  It is configured such

# that it sarts out blank with new entries added after passing

# a series of checks (file/dir exists, not already listed aka

# de-deduplication).  During de-depulication, wildcards and/or

# directories are *NOT* expanded to keep it simple. Therefore,

# if the computed classpath has two specific mentions of

# awesome-methods-1.0.jar, only the first one added will be seen.

# If two directories are in the classpath that both contain

# awesome-methods-1.0.jar, then Java will pick up both versions.

# An additional, custom CLASSPATH. Site-wide configs should be

# handled via the shellprofile functionality, utilizing the

# hadoop_add_classpath function for greater control and much

# harder for apps/end-users to accidentally override.

# Similarly, end users should utilize ${HOME}/.hadooprc .

# This variable should ideally only be used as a short-cut,

# interactive way for temporary additions on the command line.

# export HADOOP_CLASSPATH="/some/cool/path/on/your/machine"

# Should HADOOP_CLASSPATH be first in the official CLASSPATH?

# export HADOOP_USER_CLASSPATH_FIRST="yes"

# If HADOOP_USE_CLIENT_CLASSLOADER is set, the classpath along

# with the main jar are handled by a separate isolated

# client classloader when 'hadoop jar', 'yarn jar', or 'mapred job'

# is utilized. If it is set, HADOOP_CLASSPATH and

# HADOOP_USER_CLASSPATH_FIRST are ignored.

# export HADOOP_USE_CLIENT_CLASSLOADER=true

# HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES overrides the default definition of

# system classes for the client classloader when HADOOP_USE_CLIENT_CLASSLOADER

# is enabled. Names ending in '.' (period) are treated as package names, and

# names starting with a '-' are treated as negative matches. For example,

# export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop."

# Enable optional, bundled Hadoop features

# This is a comma delimited list.  It may NOT be overridden via .hadooprc

# Entries may be added/removed as needed.

# export HADOOP_OPTIONAL_TOOLS="hadoop-kafka,hadoop-aws,hadoop-aliyun,hadoop-openstack,hadoop-azure,hadoop-azure-datalake"

###

# Options for remote shell connectivity

###

# There are some optional components of hadoop that allow for

# command and control of remote hosts.  For example,

# start-dfs.sh will attempt to bring up all NNs, DNS, etc.

# Options to pass to SSH when one of the "log into a host and

# start/stop daemons" scripts is executed

# export HADOOP_SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"

# The built-in ssh handler will limit itself to 10 simultaneous connections.

# For pdsh users, this sets the fanout size ( -f )

# Change this to increase/decrease as necessary.

# export HADOOP_SSH_PARALLEL=10

# Filename which contains all of the hosts for any remote execution

# helper scripts # such as workers.sh, start-dfs.sh, etc.

# export HADOOP_WORKERS="${HADOOP_CONF_DIR}/workers"

###

# Options for all daemons

###

#

#

# Many options may also be specified as Java properties.  It is

# very common, and in many cases, desirable, to hard-set these

# in daemon _OPTS variables.  Where applicable, the appropriate

# Java property is also identified.  Note that many are re-used

# or set differently in certain contexts (e.g., secure vs

# non-secure)

#

# Where (primarily) daemon log files are stored.

# ${HADOOP_HOME}/logs by default.

# Java property: hadoop.log.dir

# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs

# A string representing this instance of hadoop. $USER by default.

# This is used in writing log and pid files, so keep that in mind!

# Java property: hadoop.id.str

# export HADOOP_IDENT_STRING=$USER

# How many seconds to pause after stopping a daemon

# export HADOOP_STOP_TIMEOUT=5

# Where pid files are stored.  /tmp by default.

# export HADOOP_PID_DIR=/tmp

# Default log4j setting for interactive commands

# Java property: hadoop.root.logger

# export HADOOP_ROOT_LOGGER=INFO,console

# Default log4j setting for daemons spawned explicitly by

# --daemon option of hadoop, hdfs, mapred and yarn command.

# Java property: hadoop.root.logger

# export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA

# Default log level and output location for security-related messages.

# You will almost certainly want to change this on a per-daemon basis via

# the Java property (i.e., -Dhadoop.security.logger=foo). (Note that the

# defaults for the NN and 2NN override this by default.)

# Java property: hadoop.security.logger

# export HADOOP_SECURITY_LOGGER=INFO,NullAppender

# Default process priority level

# Note that sub-processes will also run at this level!

# export HADOOP_NICENESS=0

# Default name for the service level authorization file

# Java property: hadoop.policy.file

# export HADOOP_POLICYFILE="hadoop-policy.xml"

#

# NOTE: this is not used by default!  <-----

# You can define variables right here and then re-use them later on.

# For example, it is common to use the same garbage collection settings

# for all the daemons.  So one could define:

#

# export HADOOP_GC_SETTINGS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps"

#

# .. and then use it as per the b option under the namenode.

###

# Secure/privileged execution

###

#

# Out of the box, Hadoop uses jsvc from Apache Commons to launch daemons

# on privileged ports.  This functionality can be replaced by providing

# custom functions.  See hadoop-functions.sh for more information.

#

# The jsvc implementation to use. Jsvc is required to run secure datanodes

# that bind to privileged ports to provide authentication of data transfer

# protocol.  Jsvc is not required if SASL is configured for authentication of

# data transfer protocol using non-privileged ports.

# export JSVC_HOME=/usr/bin

#

# This directory contains pids for secure and privileged processes.

#export HADOOP_SECURE_PID_DIR=${HADOOP_PID_DIR}

#

# This directory contains the logs for secure and privileged processes.

# Java property: hadoop.log.dir

# export HADOOP_SECURE_LOG=${HADOOP_LOG_DIR}

#

# When running a secure daemon, the default value of HADOOP_IDENT_STRING

# ends up being a bit bogus.  Therefore, by default, the code will

# replace HADOOP_IDENT_STRING with HADOOP_xx_SECURE_USER.  If one wants

# to keep HADOOP_IDENT_STRING untouched, then uncomment this line.

# export HADOOP_SECURE_IDENT_PRESERVE="true"

###

# NameNode specific parameters

###

# Default log level and output location for file system related change

# messages. For non-namenode daemons, the Java property must be set in

# the appropriate _OPTS if one wants something other than INFO,NullAppender

# Java property: hdfs.audit.logger

# export HDFS_AUDIT_LOGGER=INFO,NullAppender

# Specify the JVM options to be used when starting the NameNode.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# a) Set JMX options

# export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"

#

# b) Set garbage collection logs

# export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"

#

# c) ... or set them directly

# export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"

# this is the default:

# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"

###

# SecondaryNameNode specific parameters

###

# Specify the JVM options to be used when starting the SecondaryNameNode.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# This is the default:

# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"

###

# DataNode specific parameters

###

# Specify the JVM options to be used when starting the DataNode.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# This is the default:

# export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"

# On secure datanodes, user to run the datanode as after dropping privileges.

# This **MUST** be uncommented to enable secure HDFS if using privileged ports

# to provide authentication of data transfer protocol.  This **MUST NOT** be

# defined if SASL is configured for authentication of data transfer protocol

# using non-privileged ports.

# This will replace the hadoop.id.str Java property in secure mode.

# export HDFS_DATANODE_SECURE_USER=hdfs

# Supplemental options for secure datanodes

# By default, Hadoop uses jsvc which needs to know to launch a

# server jvm.

# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server"

###

# NFS3 Gateway specific parameters

###

# Specify the JVM options to be used when starting the NFS3 Gateway.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# export HDFS_NFS3_OPTS=""

# Specify the JVM options to be used when starting the Hadoop portmapper.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# export HDFS_PORTMAP_OPTS="-Xmx512m"

# Supplemental options for priviliged gateways

# By default, Hadoop uses jsvc which needs to know to launch a

# server jvm.

# export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server"

# On privileged gateways, user to run the gateway as after dropping privileges

# This will replace the hadoop.id.str Java property in secure mode.

# export HDFS_NFS3_SECURE_USER=nfsserver

###

# ZKFailoverController specific parameters

###

# Specify the JVM options to be used when starting the ZKFailoverController.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# export HDFS_ZKFC_OPTS=""

###

# QuorumJournalNode specific parameters

###

# Specify the JVM options to be used when starting the QuorumJournalNode.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# export HDFS_JOURNALNODE_OPTS=""

###

# HDFS Balancer specific parameters

###

# Specify the JVM options to be used when starting the HDFS Balancer.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# export HDFS_BALANCER_OPTS=""

###

# HDFS Mover specific parameters

###

# Specify the JVM options to be used when starting the HDFS Mover.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# export HDFS_MOVER_OPTS=""

###

# Router-based HDFS Federation specific parameters

# Specify the JVM options to be used when starting the RBF Routers.

# These options will be appended to the options specified as HADOOP_OPTS

# and therefore may override any similar flags set in HADOOP_OPTS

#

# export HDFS_DFSROUTER_OPTS=""

###

###

# Advanced Users Only!

###

#

# When building Hadoop, one can add the class paths to the commands

# via this special env var:

# export HADOOP_ENABLE_BUILD_PATHS="true"

#

# To prevent accidents, shell commands be (superficially) locked

# to only allow certain users to execute certain subcommands.

# It uses the format of (command)_(subcommand)_USER.

#

# For example, to limit who can execute the namenode command,

# export HDFS_NAMENODE_USER=hdfs

export HADOOP_NAMENODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1234 -javaagent:/opt/module/hadoop-3.1.3/share/hadoop/common/lib/management-agent.jar=9200:/opt/module/hadoop-3.1.3/etc/hadoop/namenode.yaml"

core-site.xml

<?xml version="1.0" encoding="UTF-8"?>

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!--

  Licensed under the Apache License, Version 2.0 (the "License");

  you may not use this file except in compliance with the License.

  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software

  distributed under the License is distributed on an "AS IS" BASIS,

  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  See the License for the specific language governing permissions and

  limitations under the License. See accompanying LICENSE file.

-->

<configuration>

  <property>

    <name>fs.defaultFS</name>

    <value>hdfs://zszchdfs/</value>

  </property>

  <property>

    <name>ha.zookeeper.quorum</name>

    <value>datanebula7:2181,datanebual8:2181,datanebula9:2181</value>

  </property>

  <property>

    <name>hadoop.tmp.dir</name>

    <value>/home/sdb1/hadoop/data/tmp</value>

    <description>A base for other temporarydirectories.</description>

  </property>

  <property>

    <name>io.file.buffer.size</name>

    <value>131072</value>

  </property>

</configuration>

yarn-site.xml

<?xml version="1.0"?>

<!--

  Licensed under the Apache License, Version 2.0 (the "License");

  you may not use this file except in compliance with the License.

  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software

  distributed under the License is distributed on an "AS IS" BASIS,

  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

  See the License for the specific language governing permissions and

  limitations under the License. See accompanying LICENSE file.

-->

<configuration>

  <!-- Site specific YARN configuration properties -->

  <property>

          <name>yarn.nodemanager.aux-services</name>

              <value>mapreduce_shuffle,spark_shuffle</value>

        </property>

  <!--ha related configuration-->

<property>

    <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>

    <value>org.apache.spark.network.yarn.YarnShuffleService</value>

</property>

  <property>

    <name>yarn.resourcemanager.ha.enabled</name>

    <value>true</value>

  </property>

  <property>

    <name>yarn.resourcemanager.cluster-id</name>

    <value>zszcyarn</value>

  </property>

  <property>

    <name>yarn.resourcemanager.ha.rm-ids</name>

    <value>rm1,rm2</value>

  </property>

  <property>

    <name>yarn.resourcemanager.hostname.rm1</name>

    <value>datanebula1</value>

  </property>

  <property>

    <name>yarn.resourcemanager.hostname.rm2</name>

    <value>datanebula2</value>

  </property>

  <property>

    <name>yarn.resourcemanager.recovery.enabled</name>

    <value>true</value>

  </property>

  <property>

    <name>yarn.resourcemanager.webapp.address.rm1</name> 

        <value>datanebula1:8088</value>

  </property> 

  <property>

        <name>yarn.resourcemanager.webapp.address.rm2</name> 

        <value>datanebula2:8088</value>

  </property>

  <property>

    <name>yarn.resourcemanager.store.class</name>

    <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>

  </property>

  <property>

    <name>yarn.resourcemanager.zk-state-store.parent-path</name>

    <value>/rmstore</value>

  </property>

  <property>

    <name>yarn.resourcemanager.zk-address</name>

    <value>datanebula7:2181,datanebula8:2181,datanebula9:2181</value>

  </property>

  <!-- 该节点服务器上yarn可以使用的虚拟CPU个数,默认是8,此处配置24-->

  <property>

    <name>yarn.nodemanager.resource.cpu-vcores</name>

    <value>27</value>

  </property>

  <!-- 配置每台nodemanager的可使用内存大小,设置成64G(单位M) -->

  <property>

    <name>yarn.nodemanager.resource.memory-mb</name>

    <value>159744</value>

  </property>

  <!-- 单个container最小vcore个数,也可以说每个任务最小使用1核CPU -->

  <property>

    <name>yarn.scheduler.minimum-allocation-vcores</name>

    <value>1</value>

  </property>

  <!-- 单个container最大vcore个数,也可以说每个任务最大使用4核CPU -->

  <property>

    <name>yarn.scheduler.maximum-allocation-vcores</name>

    <value>4</value>

  </property>

  <!-- 单个container最小内存,也可以说是一个任务 -->

  <property>

    <name>yarn.scheduler.minimum-allocation-mb</name>

    <value>1024</value>

  </property>

  <!-- 单个container最大内存,也可以说是一个任务 -->

  <property>

    <name>yarn.scheduler.maximum-allocation-mb</name>

    <value>16384</value>

  </property>

        <!-- 关闭虚拟内存检查 -->

  <property>

        <name>yarn.nodemanager.vmem-check-enabled</name>

          <value>false</value>

        </property>


        <!-- 开启日志聚合相关参数 -->

  <property>

    <name>yarn.log-aggregation-enable</name>

    <value>true</value>

  </property>

  <!-- 多长时间删除一次聚合产生的日志,30 day -->

  <property>

    <name>yarn.log-aggregation.retain-seconds</name>

    <value>2592000</value>

  </property>

  <!-- 指定聚合产生的日志的压缩算法 -->

  <property>

    <name>yarn.nodemanager.log-aggregation.compression-type</name>

    <value>gz</value>

  </property>

  <property>

    <name>yarn.nodemanager.local-dirs</name>

    <value>/home/sdb1/yarn/logs/nmlocallogs</value>

  </property>

  <property>

    <name>yarn.resourcemanager.max-completed-applications</name>

    <value>10000</value>

  </property>

  <property>

    <name>yarn.log.server.url</name>

    <value>http://datanebula9:19888/jobhistory/logs</value>

  </property>

</configuration>

最后编辑于
©著作权归作者所有,转载或内容合作请联系作者
  • 序言:七十年代末,一起剥皮案震惊了整个滨河市,随后出现的几起案子,更是在滨河造成了极大的恐慌,老刑警刘岩,带你破解...
    沈念sama阅读 216,843评论 6 502
  • 序言:滨河连续发生了三起死亡事件,死亡现场离奇诡异,居然都是意外死亡,警方通过查阅死者的电脑和手机,发现死者居然都...
    沈念sama阅读 92,538评论 3 392
  • 文/潘晓璐 我一进店门,熙熙楼的掌柜王于贵愁眉苦脸地迎上来,“玉大人,你说我怎么就摊上这事。” “怎么了?”我有些...
    开封第一讲书人阅读 163,187评论 0 353
  • 文/不坏的土叔 我叫张陵,是天一观的道长。 经常有香客问我,道长,这世上最难降的妖魔是什么? 我笑而不...
    开封第一讲书人阅读 58,264评论 1 292
  • 正文 为了忘掉前任,我火速办了婚礼,结果婚礼上,老公的妹妹穿的比我还像新娘。我一直安慰自己,他们只是感情好,可当我...
    茶点故事阅读 67,289评论 6 390
  • 文/花漫 我一把揭开白布。 她就那样静静地躺着,像睡着了一般。 火红的嫁衣衬着肌肤如雪。 梳的纹丝不乱的头发上,一...
    开封第一讲书人阅读 51,231评论 1 299
  • 那天,我揣着相机与录音,去河边找鬼。 笑死,一个胖子当着我的面吹牛,可吹牛的内容都是我干的。 我是一名探鬼主播,决...
    沈念sama阅读 40,116评论 3 418
  • 文/苍兰香墨 我猛地睁开眼,长吁一口气:“原来是场噩梦啊……” “哼!你这毒妇竟也来了?” 一声冷哼从身侧响起,我...
    开封第一讲书人阅读 38,945评论 0 275
  • 序言:老挝万荣一对情侣失踪,失踪者是张志新(化名)和其女友刘颖,没想到半个月后,有当地人在树林里发现了一具尸体,经...
    沈念sama阅读 45,367评论 1 313
  • 正文 独居荒郊野岭守林人离奇死亡,尸身上长有42处带血的脓包…… 初始之章·张勋 以下内容为张勋视角 年9月15日...
    茶点故事阅读 37,581评论 2 333
  • 正文 我和宋清朗相恋三年,在试婚纱的时候发现自己被绿了。 大学时的朋友给我发了我未婚夫和他白月光在一起吃饭的照片。...
    茶点故事阅读 39,754评论 1 348
  • 序言:一个原本活蹦乱跳的男人离奇死亡,死状恐怖,灵堂内的尸体忽然破棺而出,到底是诈尸还是另有隐情,我是刑警宁泽,带...
    沈念sama阅读 35,458评论 5 344
  • 正文 年R本政府宣布,位于F岛的核电站,受9级特大地震影响,放射性物质发生泄漏。R本人自食恶果不足惜,却给世界环境...
    茶点故事阅读 41,068评论 3 327
  • 文/蒙蒙 一、第九天 我趴在偏房一处隐蔽的房顶上张望。 院中可真热闹,春花似锦、人声如沸。这庄子的主人今日做“春日...
    开封第一讲书人阅读 31,692评论 0 22
  • 文/苍兰香墨 我抬头看了看天上的太阳。三九已至,却和暖如春,着一层夹袄步出监牢的瞬间,已是汗流浃背。 一阵脚步声响...
    开封第一讲书人阅读 32,842评论 1 269
  • 我被黑心中介骗来泰国打工, 没想到刚下飞机就差点儿被人妖公主榨干…… 1. 我叫王不留,地道东北人。 一个月前我还...
    沈念sama阅读 47,797评论 2 369
  • 正文 我出身青楼,却偏偏与公主长得像,于是被迫代替她去往敌国和亲。 传闻我的和亲对象是个残疾皇子,可洞房花烛夜当晚...
    茶点故事阅读 44,654评论 2 354

推荐阅读更多精彩内容