diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/compute-classpath.sh | 2 | ||||
| -rwxr-xr-x | bin/spark-class | 9 | ||||
| -rwxr-xr-x | bin/spark-shell | 11 | ||||
| -rwxr-xr-x | bin/spark-submit | 6 |
4 files changed, 14 insertions, 14 deletions
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index 3a59f599fd..b0218531e9 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -28,7 +28,7 @@ FWDIR="$(cd `dirname $0`/..; pwd)" . $FWDIR/bin/load-spark-env.sh # Build up classpath -CLASSPATH="$SPARK_CLASSPATH:$FWDIR/conf" +CLASSPATH="$SPARK_CLASSPATH:$SPARK_SUBMIT_CLASSPATH:$FWDIR/conf" ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SCALA_VERSION" diff --git a/bin/spark-class b/bin/spark-class index 6871e180c9..e8160c8af6 100755 --- a/bin/spark-class +++ b/bin/spark-class @@ -73,11 +73,13 @@ case "$1" in OUR_JAVA_MEM=${SPARK_EXECUTOR_MEMORY:-$DEFAULT_MEM} ;; - # All drivers use SPARK_JAVA_OPTS + SPARK_DRIVER_MEMORY. The repl also uses SPARK_REPL_OPTS. - 'org.apache.spark.repl.Main') - OUR_JAVA_OPTS="$SPARK_JAVA_OPTS $SPARK_REPL_OPTS" + # Spark submit uses SPARK_SUBMIT_OPTS and SPARK_JAVA_OPTS + 'org.apache.spark.deploy.SparkSubmit') + OUR_JAVA_OPTS="$SPARK_JAVA_OPTS $SPARK_SUBMIT_OPTS \ + -Djava.library.path=$SPARK_SUBMIT_LIBRARY_PATH" OUR_JAVA_MEM=${SPARK_DRIVER_MEMORY:-$DEFAULT_MEM} ;; + *) OUR_JAVA_OPTS="$SPARK_JAVA_OPTS" OUR_JAVA_MEM=${SPARK_DRIVER_MEMORY:-$DEFAULT_MEM} @@ -98,7 +100,6 @@ fi # Set JAVA_OPTS to be able to load native libraries and to set heap size JAVA_OPTS="$OUR_JAVA_OPTS" -JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$_SPARK_LIBRARY_PATH" JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM" # Load extra JAVA_OPTS from conf/java-opts, if it exists if [ -e "$FWDIR/conf/java-opts" ] ; then diff --git a/bin/spark-shell b/bin/spark-shell index f1f3c18877..7f03349c5e 100755 --- a/bin/spark-shell +++ b/bin/spark-shell @@ -20,7 +20,6 @@ # # Shell script for starting the Spark Shell REPL -args="$@" cygwin=false case "`uname`" in CYGWIN*) cygwin=true;; @@ -46,12 +45,12 @@ function main(){ # "Backspace sends ^H" setting in "Keys" section of the Mintty options # (see https://github.com/sbt/sbt/issues/562). stty -icanon min 1 -echo > /dev/null 2>&1 - export SPARK_REPL_OPTS="$SPARK_REPL_OPTS -Djline.terminal=unix" - $FWDIR/bin/spark-submit spark-internal "$args" --class org.apache.spark.repl.Main + export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Djline.terminal=unix" + $FWDIR/bin/spark-submit spark-internal "$@" --class org.apache.spark.repl.Main stty icanon echo > /dev/null 2>&1 else - export SPARK_REPL_OPTS - $FWDIR/bin/spark-submit spark-internal "$args" --class org.apache.spark.repl.Main + export SPARK_SUBMIT_OPTS + $FWDIR/bin/spark-submit spark-internal "$@" --class org.apache.spark.repl.Main fi } @@ -83,7 +82,7 @@ if [[ ! $? ]]; then saved_stty="" fi -main +main "$@" # record the exit status lest it be overwritten: # then reenable echo and propagate the code. diff --git a/bin/spark-submit b/bin/spark-submit index b2a1dca721..dd0d95d9d4 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -26,11 +26,11 @@ while (($#)); do elif [ "$1" = "--driver-memory" ]; then DRIVER_MEMORY=$2 elif [ "$1" = "--driver-library-path" ]; then - export _SPARK_LIBRARY_PATH=$2 + export SPARK_SUBMIT_LIBRARY_PATH=$2 elif [ "$1" = "--driver-class-path" ]; then - export SPARK_CLASSPATH="$SPARK_CLASSPATH:$2" + export SPARK_SUBMIT_CLASSPATH=$2 elif [ "$1" = "--driver-java-options" ]; then - export SPARK_JAVA_OPTS="$SPARK_JAVA_OPTS $2" + export SPARK_SUBMIT_OPTS=$2 fi shift done |