diff options
author | Denny <dennybritz@gmail.com> | 2012-08-01 13:17:31 -0700 |
---|---|---|
committer | Denny <dennybritz@gmail.com> | 2012-08-01 20:38:52 -0700 |
commit | 0ee44c225e38abbf3382be6e9555ab9a35424a54 (patch) | |
tree | 9fc106518bf55fc27083463843e9ee591c63ee59 /bin/slaves.sh | |
parent | 545165e8153bb516d4364f9b3df1440c6b44c01b (diff) | |
download | spark-0ee44c225e38abbf3382be6e9555ab9a35424a54.tar.gz spark-0ee44c225e38abbf3382be6e9555ab9a35424a54.tar.bz2 spark-0ee44c225e38abbf3382be6e9555ab9a35424a54.zip |
Spark standalone mode cluster scripts.
Heavily inspired by Hadoop cluster scripts ;-)
Diffstat (limited to 'bin/slaves.sh')
-rwxr-xr-x | bin/slaves.sh | 59 |
1 files changed, 59 insertions, 0 deletions
diff --git a/bin/slaves.sh b/bin/slaves.sh new file mode 100755 index 0000000000..e0dd883358 --- /dev/null +++ b/bin/slaves.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Run a shell command on all slave hosts. +# +# Environment Variables +# +# SPARK_SLAVES File naming remote hosts. +# Default is ${SPARK_CONF_DIR}/slaves. +# SPARK_CONF_DIR Alternate conf dir. Default is ${SPARK_HOME}/conf. +# SPARK_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# SPARK_SSH_OPTS Options passed to ssh when running remote commands. +## + +usage="Usage: slaves.sh [--config confdir] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +. "$bin/spark-config.sh" + +# If the slaves file is specified in the command line, +# then it takes precedence over the definition in +# spark-env.sh. Save it here. +HOSTLIST=$SPARK_SLAVES + +if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then + . "${SPARK_CONF_DIR}/spark-env.sh" +fi + +if [ "$HOSTLIST" = "" ]; then + if [ "$SPARK_SLAVES" = "" ]; then + export HOSTLIST="${SPARK_CONF_DIR}/slaves" + else + export HOSTLIST="${SPARK_SLAVES}" + fi +fi + +echo $"${@// /\\ }" + +# By default disable strict host key checking +if [ "$SPARK_SSH_OPTS" = "" ]; then + SPARK_SSH_OPTS="-o StrictHostKeyChecking=no" +fi + +for slave in `cat "$HOSTLIST"|sed "s/#.*$//;/^$/d"`; do + ssh $SPARK_SSH_OPTS $slave $"${@// /\\ }" \ + 2>&1 | sed "s/^/$slave: /" & + if [ "$SPARK_SLAVE_SLEEP" != "" ]; then + sleep $SPARK_SLAVE_SLEEP + fi +done + +wait |