Hadoop源碼分析三啟動及腳本剖析

第Hadoop源碼分析三啟動及腳本剖析#somevariables

exportHADOOP_LOGFILE=hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.log

exportHADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-"INFO,RFA"}

exportHADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-"INFO,RFAS"}

exportHDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-"INFO,NullAppender"}

log=$HADOOP_LOG_DIR/hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.out

pid=$HADOOP_PID_DIR/hadoop-$HADOOP_IDENT_STRING-$command.pid

HADOOP_STOP_TIMEOUT=${HADOOP_STOP_TIMEOUT:-5}

#Setdefaultschedulingpriority

if["$HADOOP_NICENESS"=""];then

exportHADOOP_NICENESS=0

case$startStopin

(start)

[-w"$HADOOP_PID_DIR"]||mkdir-p"$HADOOP_PID_DIR"

if[-f$pid];then

ifkill-0`cat$pid`/dev/null2then

echo$commandrunningasprocess`cat$pid`.Stopitfirst.

exit1

if["$HADOOP_MASTER"!=""];then

echorsyncfrom$HADOOP_MASTER

rsync-a-essh--delete--exclude=.svn--exclude='logs/*'--exclude='contrib/hod/logs/*'$HADOOP_MASTER/"$HADOOP_PREFIX"

hadoop_rotate_log$log

echostarting$command,loggingto$log

cd"$HADOOP_PREFIX"

case$commandin

namenode|secondarynamenode|datanode|journalnode|dfs|dfsadmin|fsck|balancer|zkfc)

if[-z"$HADOOP_HDFS_HOME"];then

hdfsScript="$HADOOP_PREFIX"/bin/hdfs

else

hdfsScript="$HADOOP_HDFS_HOME"/bin/hdfs

nohupnice-n$HADOOP_NICENESS$hdfsScript--config$HADOOP_CONF_DIR$command"$@""$log"21/dev/null

nohupnice-n$HADOOP_NICENESS$hadoopScript--config$HADOOP_CONF_DIR$command"$@""$log"21/dev/null

esac

echo$!$pid

sleep1

head"$log"

#capturetheulimitoutput

if["true"="$starting_secure_dn"];then

echo"ulimit-aforsecuredatanodeuser$HADOOP_SECURE_DN_USER"$log

#capturetheulimitinfofortheappropriateuser

su--shell=/bin/bash$HADOOP_SECURE_DN_USER-c'ulimit-a'$log21

elif["true"="$starting_privileged_nfs"];then

echo"ulimit-aforprivilegednfsuser$HADOOP_PRIVILEGED_NFS_USER"$log

su--shell=/bin/bash$HADOOP_PRIVILEGED_NFS_USER-c'ulimit-a'$log21

else

echo"ulimit-aforuser$USER"$log

ulimit-a$log21

sleep3;

if!ps-p$!/dev/null;then

exit1

(stop)

if[-f$pid];then

TARGET_PID=`cat$pid`

ifkill-0$TARGET_PID/dev/null2then

echostopping$command

kill$TARGET_PID

sleep$HADOOP_STOP_TIMEOUT

ifkill-0$TARGET_PID/dev/null2then

echo"$commanddidnotstopgracefullyafter$HADOOP_STOP_TIMEOUTseconds:killingwithkill-9"

kill-9$TARGET_PID

else

echono$commandtostop

rm-f$pid

else

echono$commandtostop

echo$usage

exit1

這段代碼的重點在第131行到結(jié)束。這里是真正在啟動服務(wù)的代碼，這個文件在調(diào)用的時候，會傳入兩個重要的參數(shù)start/stopxxx。用于啟動或停止某些服務(wù)。以啟動服務(wù)為例，其重點在第153行，這里會執(zhí)行一個hdfsScript腳本。這個參數(shù)的定義在第155行，

這里可以看見它實際是hadoop的bin目錄下的hdfs文件

文件的內(nèi)容如下：

#!/usr/bin/envbash

#LicensedtotheApacheSoftwareFoundation(ASF)underoneormore

#contributorlicenseagreements.SeetheNOTICEfiledistributedwith

#thisworkforadditionalinformationregardingcopyrightownership.

#TheASFlicensesthisfiletoYouundertheApacheLicense,Version2.0

#(the"License");youmaynotusethisfileexceptincompliancewith

#theLicense.YoumayobtainacopyoftheLicenseat

#/licenses/LICENSE-2.0

#Unlessrequiredbyapplicablelaworagreedtoinwriting,software

#distributedundertheLicenseisdistributedonan"ASIS"BASIS,

#WITHOUTWARRANTIESORCONDITIONSOFANYKIND,eitherexpressorimplied.

#SeetheLicenseforthespecificlanguagegoverningpermissionsand

#limitationsundertheLicense.

#EnvironmentVariables

#JSVC_HOMEhomedirectoryofjsvcbinary.Requiredforstartingsecure

#datanode.

#JSVC_OUTFILEpathtojsvcoutputfile.Defaultsto

#$HADOOP_LOG_DIR/jsvc.out.

#JSVC_ERRFILEpathtojsvcerrorfile.Defaultsto$HADOOP_LOG_DIR/jsvc.err.

bin=`which$0`

bin=`dirname${bin}`

bin=`cd"$bin"/dev/null;pwd`

DEFAULT_LIBEXEC_DIR="$bin"/../libexec

HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}

.$HADOOP_LIBEXEC_DIR/hdfs-config.sh

functionprint_usage(){

echo"Usage:hdfs[--configconfdir][--loglevelloglevel]COMMAND"

echo"whereCOMMANDisoneof:"

echo"dfsrunafilesystemcommandonthefilesystemssupportedinHadoop."

echo"classpathprintstheclasspath"

echo"namenode-formatformattheDFSfilesystem"

echo"secondarynamenoderuntheDFSsecondarynamenode"

echo"namenoderuntheDFSnamenode"

echo"journalnoderuntheDFSjournalnode"

echo"zkfcruntheZKFailoverControllerdaemon"

echo"datanoderunaDFSdatanode"

echo"dfsadminrunaDFSadminclient"

echo"haadminrunaDFSHAadminclient"

echo"fsckrunaDFSfilesystemcheckingutility"

echo"balancerrunaclusterbalancingutility"

echo"jmxgetgetJMXexportedvaluesfromNameNodeorDataNode."

echo"moverrunautilitytomoveblockreplicasacross"

echo"storagetypes"

echo"oivapplytheofflinefsimageviewertoanfsimage"

echo"oiv_legacyapplytheofflinefsimageviewertoanlegacyfsimage"

echo"oevapplytheofflineeditsviewertoaneditsfile"

echo"fetchdtfetchadelegationtokenfromtheNameNode"

echo"getconfgetconfigvaluesfromconfiguration"

echo"groupsgetthegroupswhichusersbelongto"

echo"snapshotDiffdifftwosnapshotsofadirectoryordiffthe"

echo"currentdirectorycontentswithasnapshot"

echo"lsSnapshottableDirlistallsnapshottabledirsownedbythecurrentuser"

echo"Use-helptoseeoptions"

echo"portmaprunaportmapservice"

echo"nfs3runanNFSversion3gateway"

echo"cacheadminconfiguretheHDFScache"

echo"cryptoconfigureHDFSencryptionzones"

echo"storagepolicieslist/get/setblockstoragepolicies"

echo"versionprinttheversion"

echo""

echo"Mostcommandsprinthelpwheninvokedw/oparameters."

#Therearealsodebugcommands,buttheydon'tshowupinthislisting.

if[$#=0];then

print_usage

exit

COMMAND=$1

shift

case$COMMANDin

#usageflags

--help|-help|-h)

print_usage

exit

#Determineifwe'restartingasecuredatanode,andifso,redefineappropriatevariables

if["$COMMAND"=="datanode"]["$EUID"-eq0][-n"$HADOOP_SECURE_DN_USER"];then

if[-n"$JSVC_HOME"];then

if[-n"$HADOOP_SECURE_DN_PID_DIR"];then

HADOOP_PID_DIR=$HADOOP_SECURE_DN_PID_DIR

if[-n"$HADOOP_SECURE_DN_LOG_DIR"];then

HADOOP_LOG_DIR=$HADOOP_SECURE_DN_LOG_DIR

HADOOP_OPTS="$HADOOP_OPTS-Dhadoop.log.dir=$HADOOP_LOG_DIR"

HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER

HADOOP_OPTS="$HADOOP_OPTS-Dhadoop.id.str=$HADOOP_IDENT_STRING"

starting_secure_dn="true"

else

echo"Itlookslikeyou'retryingtostartasecureDN,but\$JSVC_HOME"\

"isn'tset.FallingbacktostartinginsecureDN."

#Determineifwe'restartingaprivilegedNFSdaemon,andifso,redefineappropriatevariables

if["$COMMAND"=="nfs3"]["$EUID"-eq0][-n"$HADOOP_PRIVILEGED_NFS_USER"];then

if[-n"$JSVC_HOME"];then

if[-n"$HADOOP_PRIVILEGED_NFS_PID_DIR"];then

HADOOP_PID_DIR=$HADOOP_PRIVILEGED_NFS_PID_DIR

if[-n"$HADOOP_PRIVILEGED_NFS_LOG_DIR"];then

HADOOP_LOG_DIR=$HADOOP_PRIVILEGED_NFS_LOG_DIR

HADOOP_OPTS="$HADOOP_OPTS-Dhadoop.log.dir=$HADOOP_LOG_DIR"

HADOOP_IDENT_STRING=$HADOOP_PRIVILEGED_NFS_USER

HADOOP_OPTS="$HADOOP_OPTS-Dhadoop.id.str=$HADOOP_IDENT_STRING"

starting_privileged_nfs="true"

else

echo"Itlookslikeyou'retryingtostartaprivilegedNFSserver,but"\

"\$JSVC_HOMEisn'tset.FallingbacktostartingunprivilegedNFSserver."

if["$COMMAND"="namenode"];then

HADOOP_OPTS="$HADOOP_OPTS$HADOOP_NAMENODE_OPTS"

elif["$COMMAND"="zkfc"];then

HADOOP_OPTS="$HADOOP_OPTS$HADOOP_ZKFC_OPTS"

elif["$COMMAND"="secondarynamenode"];then

HADOOP_OPTS="$HADOOP_OPTS$HADOOP_SECONDARYNAMENODE_OPTS"

elif["$COMMAND"="datanode"];then

if["$starting_secure_dn"="true"];then

HADOOP_OPTS="$HADOOP_OPTS-jvmserver$HADOOP_DATANODE_OPTS"

else

HADOOP_OPTS="$HADOOP_OPTS-server$HADOOP_DATANODE_OPTS"

elif["$COMMAND"="journalnode"];then

HADOOP_OPTS="$HADOOP_OPTS$HADOOP_JOURNALNODE_OPTS"

elif["$COMMAND"="dfs"];then

"$HADOOP_OPTS$HADOOP_CLIENT_OPTS"

elif["$COMMAND"="dfsadmin"];then

"$HADOOP_OPTS$HADOOP_CLIENT_OPTS"

elif["$COMMAND"="haadmin"];then

"$HADOOP_OPTS$HADOOP_CLIENT_OPTS"

elif["$COMMAND"="fsck"];then

"$HADOOP_OPTS$HADOOP_CLIENT_OPTS"

elif["$COMMAND"="balancer"];then

"$HADOOP_OPTS$HADOOP_BALANCER_OPTS"

elif["$COMMAND"="mover"];then

"${HADOOP_OPTS}${HADOOP_MOVER_OPTS}"

elif["$COMMAND"="storagepolicies"];then

"$COMMAND"="jmxget"];then

"$COMMAND"="oiv"];then

"$COMMAND"="oiv_legacy"];then

"$COMMAND"="oev"];then

"$COMMAND"="fetchdt"];then

"$COMMAND"="getconf"];then

"$COMMAND"="groups"];then

"$COMMAND"="snapshotDiff"];then

"$COMMAND"="lsSnapshottableDir"];then

"$COMMAND"="portmap"];then

"$HADOOP_OPTS$HADOOP_PORTMAP_OPTS"

elif["$COMMAND"="nfs3"];then

"$HADOOP_OPTS$HADOOP_NFS3_OPTS"

elif["$COMMAND"="cacheadmin"];then

"$COMMAND"="crypto"];then

"$COMMAND"="version"];then

"$COMMAND"="debug"];then

"$COMMAND"="classpath"];then

if["$#"-gt0];then

"$CLASSPATH"2/dev/null)

echo$CLASSPATH

exit0

#cygwinpathtranslation

if$cygwin;then

CLASSPATH=$(cygpath-p-w"$CLASSPATH"2/dev/null)

HADOOP_LOG_DIR=$(cygpath-w"$HADOOP_LOG_DIR"2/dev/null)

HADOOP_PREFIX=$(cygpath-w"$HADOOP_PREFIX"2/dev/null)

HADOOP_CONF_DIR=$(cygpath-w"$HADOOP_CONF_DIR"2/dev/null)

HADOOP_COMMON_HOME=$(cygpath-w"$HADOOP_COMMON_HOME"2/dev/null)

HADOOP_HDFS_HOME=$(cygpath-w"$HADOOP_HDFS_HOME"2/dev/null)

HADOOP_YARN_HOME=$(cygpath-w"$HADOOP_YARN_HOME"2/dev/null)

HADOOP_MAPRED_HOME=$(cygpath-w"$HADOOP_MAPRED_HOME"2/dev/null)

exportCLASSPATH=$CLASSPATH

HADOOP_OPTS="$HADOOP_OPTS-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"

#Checktoseeifweshouldstartasecuredatanode

if["$starting_secure_dn"="true"];then

if["$HADOOP_PID_DIR"=""];then

HADOOP_SECURE_DN_PID="/tmp/hadoop_secure_dn.pid"

else

HADOOP_SECURE_DN_PID="$HADOOP_PID_DIR/hadoop_secure_dn.pid"

JSVC=$JSVC_HOME/jsvc

if[!-f$JSVC];then

echo"JSVC_HOMEisnotsetcorrectlysojsvccannotbefound.jsvcisrequiredtorunsecuredatanodes."

echo"Pleasedownloadandinstalljsvcfrom/dist/commons/daemon/binaries/"\

"andsetJSVC_HOMEtothedirectorycontainingthejsvcbinary."

exit

if[[!$JSVC_OUTFILE]];then

JSVC_OUTFILE="$HADOOP_LOG_DIR/jsvc.out"

if[[!$JSVC_ERRFILE]];then

JSVC_ERRFILE="$HADOOP_LOG_DIR/jsvc.err"

exec"$JSVC"\

-Dproc_$COMMAND-outfile"$JSVC_OUTFILE"\

-errfile"$JSVC_ERRFILE"\

-pidfile"$HADOOP_SECURE_DN_PID"\

-nodetach\

-user"$HADOOP_SECURE_DN_USER"\

-cp"$CLASSPATH"\

$JAVA_HEAP_MAX$HADOOP_OPTS\

org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"$@"

elif["$starting_privileged_nfs"="true"];then

if["$HADOOP_PID_DIR"=""];then

HADOOP_PRIVILEGED_NFS_PID="/tmp/hadoop_privileged_nfs3.pid"

else

HADOOP_PRIVILEGED_NFS_PID="$HADOOP_PID_DIR/h