HADOOP-13341. Deprecate HADOOP_SERVERNAME_OPTS; replace with (command…

…)_(subcommand)_OPTS This commit includes the following changes: HADOOP-13356. Add a function to handle command_subcommand_OPTS HADOOP-13355. Handle HADOOP_CLIENT_OPTS in a function HADOOP-13554. Add an equivalent of hadoop_subcmd_opts for secure opts HADOOP-13562. Change hadoop_subcommand_opts to use only uppercase HADOOP-13358. Modify HDFS to use hadoop_subcommand_opts HADOOP-13357. Modify common to use hadoop_subcommand_opts HADOOP-13359. Modify YARN to use hadoop_subcommand_opts HADOOP-13361. Modify hadoop_verify_user to be consistent with hadoop_subcommand_opts (ie more granularity) HADOOP-13564. modify mapred to use hadoop_subcommand_opts HADOOP-13563. hadoop_subcommand_opts should print name not actual content during debug HADOOP-13360. Documentation for HADOOP_subcommand_OPTS This closes #126
apache · Sep 12, 2016 · 58ed4fa · 58ed4fa
1 parent 9faccd1
commit 58ed4fa
Show file tree

Hide file tree

Showing 23 changed files with 477 additions and 166 deletions.
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -161,10 +161,6 @@ function hadoopcmd_case
  fi
  ;;
  esac
-
- # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
 }
 
 # This script runs the hadoop core commands.
@@ -194,6 +190,8 @@ fi
 HADOOP_SUBCMD=$1
 shift
 
+hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
 HADOOP_SUBCMD_ARGS=("$@")
 
 if declare -f hadoop_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
@@ -203,15 +201,20 @@ else
  hadoopcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
 fi
 
-hadoop_verify_user "${HADOOP_SUBCMD}"
+hadoop_add_client_opts
 
 if [[ ${HADOOP_WORKER_MODE} = true ]]; then
  hadoop_common_worker_mode_execute "${HADOOP_COMMON_HOME}/bin/hadoop" "${HADOOP_USER_PARAMS[@]}"
  exit $?
 fi
 
+hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
 if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
  HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+
+ hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
  hadoop_verify_secure_prereq
  hadoop_setup_secure_service
  priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"

diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -306,6 +306,13 @@ function hadoop_bootstrap
  HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
  HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
 
+ # by default, whatever we are about to run doesn't support
+ # daemonization
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION=false
+
+ # shellcheck disable=SC2034
+ HADOOP_SUBCMD_SECURESERVICE=false
+
  # usage output set to zero
  hadoop_reset_usage
 
@@ -1230,6 +1237,20 @@ function hadoop_translate_cygwin_path
  fi
 }
 
+## @description Adds the HADOOP_CLIENT_OPTS variable to
+## @description HADOOP_OPTS if HADOOP_SUBCMD_SUPPORTDAEMONIZATION is false
+## @audience public
+## @stability stable
+## @replaceable yes
+function hadoop_add_client_opts
+{
+ if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = false
+ || -z "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" ]]; then
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ fi
+}
+
 ## @description Finish configuring Hadoop specific system properties
 ## @description prior to executing Java
 ## @audience private
@@ -1963,17 +1984,130 @@ function hadoop_secure_daemon_handler
 ## @return will exit on failure conditions
 function hadoop_verify_user
 {
- local command=$1
- local uservar="HADOOP_${command}_USER"
+ declare program=$1
+ declare command=$2
+ declare uprogram
+ declare ucommand
+ declare uvar
+
+ if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+ uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
+ ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
+ else
+ uprogram=${program^^}
+ ucommand=${command^^}
+ fi
+
+ uvar="${uprogram}_${ucommand}_USER"
 
- if [[ -n ${!uservar} ]]; then
- if [[ ${!uservar} != "${USER}" ]]; then
- hadoop_error "ERROR: ${command} can only be executed by ${!uservar}."
+ if [[ -n ${!uvar} ]]; then
+ if [[ ${!uvar} != "${USER}" ]]; then
+ hadoop_error "ERROR: ${command} can only be executed by ${!uvar}."
  exit 1
  fi
  fi
 }
 
+## @description Add custom (program)_(command)_OPTS to HADOOP_OPTS.
+## @description Also handles the deprecated cases from pre-3.x.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param program
+## @param subcommand
+## @return will exit on failure conditions
+function hadoop_subcommand_opts
+{
+ declare program=$1
+ declare command=$2
+ declare uvar
+ declare depvar
+ declare uprogram
+ declare ucommand
+
+ if [[ -z "${program}" || -z "${command}" ]]; then
+ return 1
+ fi
+
+ # bash 4 and up have built-in ways to upper and lower
+ # case the contents of vars. This is faster than
+ # calling tr.
+
+ if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+ uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
+ ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
+ else
+ uprogram=${program^^}
+ ucommand=${command^^}
+ fi
+
+ uvar="${uprogram}_${ucommand}_OPTS"
+
+ # Let's handle all of the deprecation cases early
+ # HADOOP_NAMENODE_OPTS -> HDFS_NAMENODE_OPTS
+
+ depvar="HADOOP_${ucommand}_OPTS"
+
+ if [[ "${depvar}" != "${uvar}" ]]; then
+ if [[ -n "${!depvar}" ]]; then
+ hadoop_deprecate_envvar "${depvar}" "${uvar}"
+ fi
+ fi
+
+ if [[ -n ${!uvar} ]]; then
+ hadoop_debug "Appending ${uvar} onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
+ return 0
+ fi
+}
+
+## @description Add custom (program)_(command)_SECURE_EXTRA_OPTS to HADOOP_OPTS.
+## @description This *does not* handle the pre-3.x deprecated cases
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param program
+## @param subcommand
+## @return will exit on failure conditions
+function hadoop_subcommand_secure_opts
+{
+ declare program=$1
+ declare command=$2
+ declare uvar
+ declare uprogram
+ declare ucommand
+
+ if [[ -z "${program}" || -z "${command}" ]]; then
+ return 1
+ fi
+
+ # bash 4 and up have built-in ways to upper and lower
+ # case the contents of vars. This is faster than
+ # calling tr.
+
+ if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+ uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
+ ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
+ else
+ uprogram=${program^^}
+ ucommand=${command^^}
+ fi
+
+ # HDFS_DATANODE_SECURE_EXTRA_OPTS
+ # HDFS_NFS3_SECURE_EXTRA_OPTS
+ # ...
+ uvar="${uprogram}_${ucommand}_SECURE_EXTRA_OPTS"
+
+ if [[ -n ${!uvar} ]]; then
+ hadoop_debug "Appending ${uvar} onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
+ return 0
+ fi
+}
+
 ## @description Perform the 'hadoop classpath', etc subcommand with the given
 ## @description parameters
 ## @audience private

diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -294,16 +294,16 @@ esac
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
 # a) Set JMX options
-# export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
+# export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
 #
 # b) Set garbage collection logs
-# export HADOOP_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
+# export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
 #
 # c) ... or set them directly
-# export HADOOP_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
+# export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
 
 # this is the default:
-# export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
+# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
 
 ###
 # SecondaryNameNode specific parameters
@@ -313,7 +313,7 @@ esac
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
 # This is the default:
-# export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
+# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
 
 ###
 # DataNode specific parameters
@@ -323,7 +323,7 @@ esac
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
 # This is the default:
-# export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
+# export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
 
 # On secure datanodes, user to run the datanode as after dropping privileges.
 # This **MUST** be uncommented to enable secure HDFS if using privileged ports
@@ -336,7 +336,7 @@ esac
 # Supplemental options for secure datanodes
 # By default, Hadoop uses jsvc which needs to know to launch a
 # server jvm.
-# export HADOOP_DN_SECURE_EXTRA_OPTS="-jvm server"
+# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server"
 
 # Where datanode log files are stored in the secure data environment.
 # This will replace the hadoop.log.dir Java property in secure mode.
@@ -352,18 +352,18 @@ esac
 # These options will be appended to the options specified as HADOOP_OPTS
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
-# export HADOOP_NFS3_OPTS=""
+# export HDFS_NFS3_OPTS=""
 
 # Specify the JVM options to be used when starting the Hadoop portmapper.
 # These options will be appended to the options specified as HADOOP_OPTS
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
-# export HADOOP_PORTMAP_OPTS="-Xmx512m"
+# export HDFS_PORTMAP_OPTS="-Xmx512m"
 
 # Supplemental options for priviliged gateways
 # By default, Hadoop uses jsvc which needs to know to launch a
 # server jvm.
-# export HADOOP_NFS3_SECURE_EXTRA_OPTS="-jvm server"
+# export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server"
 
 # On privileged gateways, user to run the gateway as after dropping privileges
 # This will replace the hadoop.id.str Java property in secure mode.
@@ -376,7 +376,7 @@ esac
 # These options will be appended to the options specified as HADOOP_OPTS
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
-# export HADOOP_ZKFC_OPTS=""
+# export HDFS_ZKFC_OPTS=""
 
 ###
 # QuorumJournalNode specific parameters
@@ -385,7 +385,7 @@ esac
 # These options will be appended to the options specified as HADOOP_OPTS
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
-# export HADOOP_JOURNALNODE_OPTS=""
+# export HDFS_JOURNALNODE_OPTS=""
 
 ###
 # HDFS Balancer specific parameters
@@ -394,7 +394,7 @@ esac
 # These options will be appended to the options specified as HADOOP_OPTS
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
-# export HADOOP_BALANCER_OPTS=""
+# export HDFS_BALANCER_OPTS=""
 
 ###
 # HDFS Mover specific parameters
@@ -403,7 +403,7 @@ esac
 # These options will be appended to the options specified as HADOOP_OPTS
 # and therefore may override any similar flags set in HADOOP_OPTS
 #
-# export HADOOP_MOVER_OPTS=""
+# export HDFS_MOVER_OPTS=""
 
 ###
 # Advanced Users Only!
@@ -417,6 +417,7 @@ esac
 #
 # To prevent accidents, shell commands be (superficially) locked
 # to only allow certain users to execute certain subcommands.
+# It uses the format of (command)_(subcommand)_USER.
 #
 # For example, to limit who can execute the namenode command,
-# export HADOOP_namenode_USER=hdfs
+# export HDFS_NAMENODE_USER=hdfs
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
@@ -64,17 +64,17 @@ Administrators can configure individual daemons using the configuration options
 
 | Daemon | Environment Variable |
 |:---- |:---- |
-| NameNode | HADOOP\_NAMENODE\_OPTS |
-| DataNode | HADOOP\_DATANODE\_OPTS |
-| Secondary NameNode | HADOOP\_SECONDARYNAMENODE\_OPTS |
+| NameNode | HDFS\_NAMENODE\_OPTS |
+| DataNode | HDFS\_DATANODE\_OPTS |
+| Secondary NameNode | HDFS\_SECONDARYNAMENODE\_OPTS |
 | ResourceManager | YARN\_RESOURCEMANAGER\_OPTS |
 | NodeManager | YARN\_NODEMANAGER\_OPTS |
 | WebAppProxy | YARN\_PROXYSERVER\_OPTS |
-| Map Reduce Job History Server | HADOOP\_JOB\_HISTORYSERVER\_OPTS |
+| Map Reduce Job History Server | MAPRED\_HISTORYSERVER\_OPTS |
 
-For example, To configure Namenode to use parallelGC, the following statement should be added in hadoop-env.sh :
+For example, To configure Namenode to use parallelGC and a 4GB Java Heap, the following statement should be added in hadoop-env.sh :
 
-  export HADOOP_NAMENODE_OPTS="-XX:+UseParallelGC"
+  export HDFS_NAMENODE_OPTS="-XX:+UseParallelGC -Xmx4g"
 
 See `etc/hadoop/hadoop-env.sh` for other examples.
 
@@ -91,13 +91,6 @@ It is also traditional to configure `HADOOP_HOME` in the system-wide shell envir
   HADOOP_HOME=/path/to/hadoop
   export HADOOP_HOME
 
-| Daemon | Environment Variable |
-|:---- |:---- |
-| ResourceManager | YARN\_RESOURCEMANAGER\_HEAPSIZE |
-| NodeManager | YARN\_NODEMANAGER\_HEAPSIZE |
-| WebAppProxy | YARN\_PROXYSERVER\_HEAPSIZE |
-| Map Reduce Job History Server | HADOOP\_JOB\_HISTORYSERVER\_HEAPSIZE |
-
 ### Configuring the Hadoop Daemons
 
 This section deals with important parameters to be specified in the given configuration files: