diff options
author | shane-huang <shengsheng.huang@intel.com> | 2013-09-26 17:08:47 +0800 |
---|---|---|
committer | shane-huang <shengsheng.huang@intel.com> | 2013-09-26 17:08:47 +0800 |
commit | e8b1ee04fcb4cd7b666d3148d6d5ff148551ce72 (patch) | |
tree | f5b06502813cca286e2db98b6cf7df2355df0c2a /bin | |
parent | 14098037630196d2672431539503f27be67be480 (diff) | |
download | spark-e8b1ee04fcb4cd7b666d3148d6d5ff148551ce72.tar.gz spark-e8b1ee04fcb4cd7b666d3148d6d5ff148551ce72.tar.bz2 spark-e8b1ee04fcb4cd7b666d3148d6d5ff148551ce72.zip |
fix paths and change spark to use APP_MEM as application driver memory instead of SPARK_MEM, user should add application jars to SPARK_CLASSPATH
Signed-off-by: shane-huang <shengsheng.huang@intel.com>
Diffstat (limited to 'bin')
-rwxr-xr-x | bin/spark | 41 |
1 files changed, 8 insertions, 33 deletions
@@ -31,40 +31,11 @@ if [ -e $FWDIR/conf/spark-env.sh ] ; then fi if [ -z "$1" ]; then - echo "Usage: spark-class <class> [<args>]" >&2 + echo "Usage: spark <class> [<args>]" >&2 + echo "Usage: export SPARK_CLASSPATH before running the command" >&2 exit 1 fi -# If this is a standalone cluster daemon, reset SPARK_JAVA_OPTS and SPARK_MEM to reasonable -# values for that; it doesn't need a lot -if [ "$1" = "org.apache.spark.deploy.master.Master" -o "$1" = "org.apache.spark.deploy.worker.Worker" ]; then - SPARK_MEM=${SPARK_DAEMON_MEMORY:-512m} - SPARK_DAEMON_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS -Dspark.akka.logLifecycleEvents=true" - # Do not overwrite SPARK_JAVA_OPTS environment variable in this script - OUR_JAVA_OPTS="$SPARK_DAEMON_JAVA_OPTS" # Empty by default -else - OUR_JAVA_OPTS="$SPARK_JAVA_OPTS" -fi - - -# Add java opts for master, worker, executor. The opts maybe null -case "$1" in - 'org.apache.spark.deploy.master.Master') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_MASTER_OPTS" - ;; - 'org.apache.spark.deploy.worker.Worker') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_WORKER_OPTS" - ;; - 'org.apache.spark.executor.StandaloneExecutorBackend') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_EXECUTOR_OPTS" - ;; - 'org.apache.spark.executor.MesosExecutorBackend') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_EXECUTOR_OPTS" - ;; - 'org.apache.spark.repl.Main') - OUR_JAVA_OPTS="$OUR_JAVA_OPTS $SPARK_REPL_OPTS" - ;; -esac # Find the java binary if [ -n "${JAVA_HOME}" ]; then @@ -78,14 +49,18 @@ else fi fi -# Set SPARK_MEM if it isn't already set since we also use it for this process +# Set SPARK_MEM if it isn't already set SPARK_MEM=${SPARK_MEM:-512m} export SPARK_MEM +# Set APP_MEM if it isn't already set, we use this for this process as the app driver process may need +# as much memory as specified in SPARK_MEM +APP_MEM=${APP_MEM:-512m} + # Set JAVA_OPTS to be able to load native libraries and to set heap size JAVA_OPTS="$OUR_JAVA_OPTS" JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$SPARK_LIBRARY_PATH" -JAVA_OPTS="$JAVA_OPTS -Xms$SPARK_MEM -Xmx$SPARK_MEM" +JAVA_OPTS="$JAVA_OPTS -Xms$APP_MEM -Xmx$APP_MEM" # Load extra JAVA_OPTS from conf/java-opts, if it exists if [ -e $FWDIR/conf/java-opts ] ; then JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`" |