diff options
Diffstat (limited to 'core/lib/hadoop-0.20.0/bin')
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/hadoop | 289 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/hadoop-config.sh | 68 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/hadoop-daemon.sh | 143 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/hadoop-daemons.sh | 34 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/rcc | 99 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/slaves.sh | 68 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/start-all.sh | 30 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/start-balancer.sh | 25 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/start-dfs.sh | 52 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/start-mapred.sh | 29 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/stop-all.sh | 27 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/stop-balancer.sh | 26 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/stop-dfs.sh | 29 | ||||
-rwxr-xr-x | core/lib/hadoop-0.20.0/bin/stop-mapred.sh | 28 |
14 files changed, 947 insertions, 0 deletions
diff --git a/core/lib/hadoop-0.20.0/bin/hadoop b/core/lib/hadoop-0.20.0/bin/hadoop new file mode 100755 index 0000000000..273549f138 --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/hadoop @@ -0,0 +1,289 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The Hadoop command script +# +# Environment Variables +# +# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +# +# HADOOP_CLASSPATH Extra Java CLASSPATH entries. +# +# HADOOP_HEAPSIZE The maximum amount of heap to use, in MB. +# Default is 1000. +# +# HADOOP_OPTS Extra Java runtime options. +# +# HADOOP_NAMENODE_OPTS These options are added to HADOOP_OPTS +# HADOOP_CLIENT_OPTS when the respective command is run. +# HADOOP_{COMMAND}_OPTS etc HADOOP_JT_OPTS applies to JobTracker +# for e.g. HADOOP_CLIENT_OPTS applies to +# more than one command (fs, dfs, fsck, +# dfsadmin etc) +# +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# +# HADOOP_ROOT_LOGGER The root appender. Default is INFO,console +# + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +cygwin=false +case "`uname`" in +CYGWIN*) cygwin=true;; +esac + +# if no args specified, show usage +if [ $# = 0 ]; then + echo "Usage: hadoop [--config confdir] COMMAND" + echo "where COMMAND is one of:" + echo " namenode -format format the DFS filesystem" + echo " secondarynamenode run the DFS secondary namenode" + echo " namenode run the DFS namenode" + echo " datanode run a DFS datanode" + echo " dfsadmin run a DFS admin client" + echo " mradmin run a Map-Reduce admin client" + echo " fsck run a DFS filesystem checking utility" + echo " fs run a generic filesystem user client" + echo " balancer run a cluster balancing utility" + echo " jobtracker run the MapReduce job Tracker node" + echo " pipes run a Pipes job" + echo " tasktracker run a MapReduce task Tracker node" + echo " job manipulate MapReduce jobs" + echo " queue get information regarding JobQueues" + echo " version print the version" + echo " jar <jar> run a jar file" + echo " distcp <srcurl> <desturl> copy file or directories recursively" + echo " archive -archiveName NAME <src>* <dest> create a hadoop archive" + echo " daemonlog get/set the log level for each daemon" + echo " or" + echo " CLASSNAME run the class named CLASSNAME" + echo "Most commands print help when invoked w/o parameters." + exit 1 +fi + +# get arguments +COMMAND=$1 +shift + +if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then + . "${HADOOP_CONF_DIR}/hadoop-env.sh" +fi + +# some Java parameters +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi + +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +# check envvars which might override default args +if [ "$HADOOP_HEAPSIZE" != "" ]; then + #echo "run with heapsize $HADOOP_HEAPSIZE" + JAVA_HEAP_MAX="-Xmx""$HADOOP_HEAPSIZE""m" + #echo $JAVA_HEAP_MAX +fi + +# CLASSPATH initially contains $HADOOP_CONF_DIR +CLASSPATH="${HADOOP_CONF_DIR}" +CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar + +# for developers, add Hadoop classes to CLASSPATH +if [ -d "$HADOOP_HOME/build/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/classes +fi +if [ -d "$HADOOP_HOME/build/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build +fi +if [ -d "$HADOOP_HOME/build/test/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/test/classes +fi +if [ -d "$HADOOP_HOME/build/tools" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/tools +fi + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + +# for releases, add core hadoop jar & webapps to CLASSPATH +if [ -d "$HADOOP_HOME/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME +fi +for f in $HADOOP_HOME/hadoop-*-core.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +# add libs to CLASSPATH +for f in $HADOOP_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +if [ -d "$HADOOP_HOME/build/ivy/lib/Hadoop/common" ]; then +for f in $HADOOP_HOME/build/ivy/lib/Hadoop/common/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done +fi + +for f in $HADOOP_HOME/lib/jsp-2.1/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +for f in $HADOOP_HOME/hadoop-*-tools.jar; do + TOOL_PATH=${TOOL_PATH}:$f; +done +for f in $HADOOP_HOME/build/hadoop-*-tools.jar; do + TOOL_PATH=${TOOL_PATH}:$f; +done + +# add user-specified CLASSPATH last +if [ "$HADOOP_CLASSPATH" != "" ]; then + CLASSPATH=${CLASSPATH}:${HADOOP_CLASSPATH} +fi + +# default log directory & file +if [ "$HADOOP_LOG_DIR" = "" ]; then + HADOOP_LOG_DIR="$HADOOP_HOME/logs" +fi +if [ "$HADOOP_LOGFILE" = "" ]; then + HADOOP_LOGFILE='hadoop.log' +fi + +# default policy file for service-level authorization +if [ "$HADOOP_POLICYFILE" = "" ]; then + HADOOP_POLICYFILE="hadoop-policy.xml" +fi + +# restore ordinary behaviour +unset IFS + +# figure out which class to run +if [ "$COMMAND" = "namenode" ] ; then + CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS" +elif [ "$COMMAND" = "secondarynamenode" ] ; then + CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode' + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS" +elif [ "$COMMAND" = "datanode" ] ; then + CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode' + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_DATANODE_OPTS" +elif [ "$COMMAND" = "fs" ] ; then + CLASS=org.apache.hadoop.fs.FsShell + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "dfs" ] ; then + CLASS=org.apache.hadoop.fs.FsShell + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "dfsadmin" ] ; then + CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "mradmin" ] ; then + CLASS=org.apache.hadoop.mapred.tools.MRAdmin + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "fsck" ] ; then + CLASS=org.apache.hadoop.hdfs.tools.DFSck + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "balancer" ] ; then + CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS" +elif [ "$COMMAND" = "jobtracker" ] ; then + CLASS=org.apache.hadoop.mapred.JobTracker + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOBTRACKER_OPTS" +elif [ "$COMMAND" = "tasktracker" ] ; then + CLASS=org.apache.hadoop.mapred.TaskTracker + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_TASKTRACKER_OPTS" +elif [ "$COMMAND" = "job" ] ; then + CLASS=org.apache.hadoop.mapred.JobClient +elif [ "$COMMAND" = "queue" ] ; then + CLASS=org.apache.hadoop.mapred.JobQueueClient +elif [ "$COMMAND" = "pipes" ] ; then + CLASS=org.apache.hadoop.mapred.pipes.Submitter + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "version" ] ; then + CLASS=org.apache.hadoop.util.VersionInfo + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "jar" ] ; then + CLASS=org.apache.hadoop.util.RunJar +elif [ "$COMMAND" = "distcp" ] ; then + CLASS=org.apache.hadoop.tools.DistCp + CLASSPATH=${CLASSPATH}:${TOOL_PATH} + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "daemonlog" ] ; then + CLASS=org.apache.hadoop.log.LogLevel + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "archive" ] ; then + CLASS=org.apache.hadoop.tools.HadoopArchives + CLASSPATH=${CLASSPATH}:${TOOL_PATH} + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +elif [ "$COMMAND" = "sampler" ] ; then + CLASS=org.apache.hadoop.mapred.lib.InputSampler + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" +else + CLASS=$COMMAND +fi + +# cygwin path translation +if $cygwin; then + CLASSPATH=`cygpath -p -w "$CLASSPATH"` + HADOOP_HOME=`cygpath -w "$HADOOP_HOME"` + HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"` + TOOL_PATH=`cygpath -p -w "$TOOL_PATH"` +fi +# setup 'java.library.path' for native-hadoop code if necessary +JAVA_LIBRARY_PATH='' +if [ -d "${HADOOP_HOME}/build/native" -o -d "${HADOOP_HOME}/lib/native" ]; then + JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} -Xmx32m org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` + + if [ -d "$HADOOP_HOME/build/native" ]; then + JAVA_LIBRARY_PATH=${HADOOP_HOME}/build/native/${JAVA_PLATFORM}/lib + fi + + if [ -d "${HADOOP_HOME}/lib/native" ]; then + if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_HOME}/lib/native/${JAVA_PLATFORM} + else + JAVA_LIBRARY_PATH=${HADOOP_HOME}/lib/native/${JAVA_PLATFORM} + fi + fi +fi + +# cygwin path translation +if $cygwin; then + JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` +fi + +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_HOME" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" +fi +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.policy.file=$HADOOP_POLICYFILE" + +# run it +exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@" diff --git a/core/lib/hadoop-0.20.0/bin/hadoop-config.sh b/core/lib/hadoop-0.20.0/bin/hadoop-config.sh new file mode 100755 index 0000000000..1f9d52da79 --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/hadoop-config.sh @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# included in all the hadoop scripts with source command +# should not be executable directly +# also should not be passed any arguments, since we need original $* + +# resolve links - $0 may be a softlink + +this="$0" +while [ -h "$this" ]; do + ls=`ls -ld "$this"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + this="$link" + else + this=`dirname "$this"`/"$link" + fi +done + +# convert relative path to absolute path +bin=`dirname "$this"` +script=`basename "$this"` +bin=`cd "$bin"; pwd` +this="$bin/$script" + +# the root of the Hadoop installation +export HADOOP_HOME=`dirname "$this"`/.. + +#check to see if the conf dir is given as an optional argument +if [ $# -gt 1 ] +then + if [ "--config" = "$1" ] + then + shift + confdir=$1 + shift + HADOOP_CONF_DIR=$confdir + fi +fi + +# Allow alternate conf dir location. +HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_HOME/conf}" + +#check to see it is specified whether to use the slaves or the +# masters file +if [ $# -gt 1 ] +then + if [ "--hosts" = "$1" ] + then + shift + slavesfile=$1 + shift + export HADOOP_SLAVES="${HADOOP_CONF_DIR}/$slavesfile" + fi +fi diff --git a/core/lib/hadoop-0.20.0/bin/hadoop-daemon.sh b/core/lib/hadoop-0.20.0/bin/hadoop-daemon.sh new file mode 100755 index 0000000000..e10390a9e2 --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/hadoop-daemon.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Runs a Hadoop command as a daemon. +# +# Environment Variables +# +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HADOOP_LOG_DIR Where log files are stored. PWD by default. +# HADOOP_MASTER host:path where hadoop code should be rsync'd from +# HADOOP_PID_DIR The pid files are stored. /tmp by default. +# HADOOP_IDENT_STRING A string representing this instance of hadoop. $USER by default +# HADOOP_NICENESS The scheduling priority for daemons. Defaults to 0. +## + +usage="Usage: hadoop-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <hadoop-command> <args...>" + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +# get arguments +startStop=$1 +shift +command=$1 +shift + +hadoop_rotate_log () +{ + log=$1; + num=5; + if [ -n "$2" ]; then + num=$2 + fi + if [ -f "$log" ]; then # rotate logs + while [ $num -gt 1 ]; do + prev=`expr $num - 1` + [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" + num=$prev + done + mv "$log" "$log.$num"; + fi +} + +if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then + . "${HADOOP_CONF_DIR}/hadoop-env.sh" +fi + +# get log directory +if [ "$HADOOP_LOG_DIR" = "" ]; then + export HADOOP_LOG_DIR="$HADOOP_HOME/logs" +fi +mkdir -p "$HADOOP_LOG_DIR" + +if [ "$HADOOP_PID_DIR" = "" ]; then + HADOOP_PID_DIR=/tmp +fi + +if [ "$HADOOP_IDENT_STRING" = "" ]; then + export HADOOP_IDENT_STRING="$USER" +fi + +# some variables +export HADOOP_LOGFILE=hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.log +export HADOOP_ROOT_LOGGER="INFO,DRFA" +log=$HADOOP_LOG_DIR/hadoop-$HADOOP_IDENT_STRING-$command-$HOSTNAME.out +pid=$HADOOP_PID_DIR/hadoop-$HADOOP_IDENT_STRING-$command.pid + +# Set default scheduling priority +if [ "$HADOOP_NICENESS" = "" ]; then + export HADOOP_NICENESS=0 +fi + +case $startStop in + + (start) + + mkdir -p "$HADOOP_PID_DIR" + + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo $command running as process `cat $pid`. Stop it first. + exit 1 + fi + fi + + if [ "$HADOOP_MASTER" != "" ]; then + echo rsync from $HADOOP_MASTER + rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $HADOOP_MASTER/ "$HADOOP_HOME" + fi + + hadoop_rotate_log $log + echo starting $command, logging to $log + cd "$HADOOP_HOME" + nohup nice -n $HADOOP_NICENESS "$HADOOP_HOME"/bin/hadoop --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null & + echo $! > $pid + sleep 1; head "$log" + ;; + + (stop) + + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo stopping $command + kill `cat $pid` + else + echo no $command to stop + fi + else + echo no $command to stop + fi + ;; + + (*) + echo $usage + exit 1 + ;; + +esac + + diff --git a/core/lib/hadoop-0.20.0/bin/hadoop-daemons.sh b/core/lib/hadoop-0.20.0/bin/hadoop-daemons.sh new file mode 100755 index 0000000000..894d8ab11a --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/hadoop-daemons.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Run a Hadoop command on all slave hosts. + +usage="Usage: hadoop-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..." + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. $bin/hadoop-config.sh + +exec "$bin/slaves.sh" --config $HADOOP_CONF_DIR cd "$HADOOP_HOME" \; "$bin/hadoop-daemon.sh" --config $HADOOP_CONF_DIR "$@" diff --git a/core/lib/hadoop-0.20.0/bin/rcc b/core/lib/hadoop-0.20.0/bin/rcc new file mode 100755 index 0000000000..a39745be6f --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/rcc @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# The Hadoop record compiler +# +# Environment Variables +# +# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +# +# HADOOP_OPTS Extra Java runtime options. +# +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then + . "${HADOOP_CONF_DIR}/hadoop-env.sh" +fi + +# some Java parameters +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi + +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +# CLASSPATH initially contains $HADOOP_CONF_DIR +CLASSPATH="${HADOOP_CONF_DIR}" +CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar + +# for developers, add Hadoop classes to CLASSPATH +if [ -d "$HADOOP_HOME/build/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/classes +fi +if [ -d "$HADOOP_HOME/build/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build +fi +if [ -d "$HADOOP_HOME/build/test/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME/build/test/classes +fi + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + +# for releases, add core hadoop jar & webapps to CLASSPATH +if [ -d "$HADOOP_HOME/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HOME +fi +for f in $HADOOP_HOME/hadoop-*-core.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +# add libs to CLASSPATH +for f in $HADOOP_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +for f in $HADOOP_HOME/lib/jetty-ext/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +# restore ordinary behaviour +unset IFS + +CLASS='org.apache.hadoop.record.compiler.generated.Rcc' + +# cygwin path translation +if expr `uname` : 'CYGWIN*' > /dev/null; then + CLASSPATH=`cygpath -p -w "$CLASSPATH"` +fi + +# run it +exec "$JAVA" $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@" diff --git a/core/lib/hadoop-0.20.0/bin/slaves.sh b/core/lib/hadoop-0.20.0/bin/slaves.sh new file mode 100755 index 0000000000..fc9f720be7 --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/slaves.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Run a shell command on all slave hosts. +# +# Environment Variables +# +# HADOOP_SLAVES File naming remote hosts. +# Default is ${HADOOP_CONF_DIR}/slaves. +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +## + +usage="Usage: slaves.sh [--config confdir] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +# If the slaves file is specified in the command line, +# then it takes precedence over the definition in +# hadoop-env.sh. Save it here. +HOSTLIST=$HADOOP_SLAVES + +if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then + . "${HADOOP_CONF_DIR}/hadoop-env.sh" +fi + +if [ "$HOSTLIST" = "" ]; then + if [ "$HADOOP_SLAVES" = "" ]; then + export HOSTLIST="${HADOOP_CONF_DIR}/slaves" + else + export HOSTLIST="${HADOOP_SLAVES}" + fi +fi + +for slave in `cat "$HOSTLIST"|sed "s/#.*$//;/^$/d"`; do + ssh $HADOOP_SSH_OPTS $slave $"${@// /\\ }" \ + 2>&1 | sed "s/^/$slave: /" & + if [ "$HADOOP_SLAVE_SLEEP" != "" ]; then + sleep $HADOOP_SLAVE_SLEEP + fi +done + +wait diff --git a/core/lib/hadoop-0.20.0/bin/start-all.sh b/core/lib/hadoop-0.20.0/bin/start-all.sh new file mode 100755 index 0000000000..b1eefc8fbe --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/start-all.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start all hadoop daemons. Run this on master node. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +# start dfs daemons +"$bin"/start-dfs.sh --config $HADOOP_CONF_DIR + +# start mapred daemons +"$bin"/start-mapred.sh --config $HADOOP_CONF_DIR diff --git a/core/lib/hadoop-0.20.0/bin/start-balancer.sh b/core/lib/hadoop-0.20.0/bin/start-balancer.sh new file mode 100755 index 0000000000..e8c93f90ca --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/start-balancer.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +# Start balancer daemon. + +"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start balancer $@ diff --git a/core/lib/hadoop-0.20.0/bin/start-dfs.sh b/core/lib/hadoop-0.20.0/bin/start-dfs.sh new file mode 100755 index 0000000000..bda2035a2b --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/start-dfs.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start hadoop dfs daemons. +# Optinally upgrade or rollback dfs state. +# Run this on master node. + +usage="Usage: start-dfs.sh [-upgrade|-rollback]" + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +# get arguments +if [ $# -ge 1 ]; then + nameStartOpt=$1 + shift + case $nameStartOpt in + (-upgrade) + ;; + (-rollback) + dataStartOpt=$nameStartOpt + ;; + (*) + echo $usage + exit 1 + ;; + esac +fi + +# start dfs daemons +# start namenode after datanodes, to minimize time namenode is up w/o data +# note: datanodes will log connection errors until namenode starts +"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start namenode $nameStartOpt +"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR start datanode $dataStartOpt +"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR --hosts masters start secondarynamenode diff --git a/core/lib/hadoop-0.20.0/bin/start-mapred.sh b/core/lib/hadoop-0.20.0/bin/start-mapred.sh new file mode 100755 index 0000000000..b64c8f51d8 --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/start-mapred.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Start hadoop map reduce daemons. Run this on master node. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +# start mapred daemons +# start jobtracker first to minimize connection errors at startup +"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR start jobtracker +"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR start tasktracker diff --git a/core/lib/hadoop-0.20.0/bin/stop-all.sh b/core/lib/hadoop-0.20.0/bin/stop-all.sh new file mode 100755 index 0000000000..033f2fe8d8 --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/stop-all.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Stop all hadoop daemons. Run this on master node. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +"$bin"/stop-mapred.sh --config $HADOOP_CONF_DIR +"$bin"/stop-dfs.sh --config $HADOOP_CONF_DIR diff --git a/core/lib/hadoop-0.20.0/bin/stop-balancer.sh b/core/lib/hadoop-0.20.0/bin/stop-balancer.sh new file mode 100755 index 0000000000..483a9c2549 --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/stop-balancer.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +# Stop balancer daemon. +# Run this on the machine where the balancer is running + +"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop balancer diff --git a/core/lib/hadoop-0.20.0/bin/stop-dfs.sh b/core/lib/hadoop-0.20.0/bin/stop-dfs.sh new file mode 100755 index 0000000000..14fe61d17e --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/stop-dfs.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Stop hadoop DFS daemons. Run this on master node. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop namenode +"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR stop datanode +"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR --hosts masters stop secondarynamenode + diff --git a/core/lib/hadoop-0.20.0/bin/stop-mapred.sh b/core/lib/hadoop-0.20.0/bin/stop-mapred.sh new file mode 100755 index 0000000000..aa51c1f87b --- /dev/null +++ b/core/lib/hadoop-0.20.0/bin/stop-mapred.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Stop hadoop map reduce daemons. Run this on master node. + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin"/hadoop-config.sh + +"$bin"/hadoop-daemon.sh --config $HADOOP_CONF_DIR stop jobtracker +"$bin"/hadoop-daemons.sh --config $HADOOP_CONF_DIR stop tasktracker + |