diff options
| -rwxr-xr-x | conf/spark-env.sh.template | 10 | ||||
| -rwxr-xr-x | make-distribution.sh | 5 | ||||
| -rwxr-xr-x | sbt/sbt | 15 |
3 files changed, 23 insertions, 7 deletions
diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index c978db00d9..eefd1f86c5 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -19,6 +19,14 @@ # - SPARK_JAVA_OPTS, to set the jvm options for executor backend. Note: This is # only for node-specific options, whereas app-specific options should be set # in the application. -# Examples of node-speicic options : -Dspark.local.dir, GC related options. +# Examples of node-specific options : -Dspark.local.dir, GC related options. # Examples of app-specific options : -Dspark.serializer +# Hadoop version to build against +# export SPARK_HADOOP_VERSION=1.0.4 + +# Uncomment this when using a Hadoop version with YARN +# export SPARK_WITH_YARN=true + +# Extra arguments to pass to `java` when building with SBT +# export EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS -Xmx8g" diff --git a/make-distribution.sh b/make-distribution.sh index 70aff418c7..846548617a 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -50,6 +50,11 @@ SPARK_HADOOP_VERSION=1.0.4 SPARK_WITH_YARN=false MAKE_TGZ=false +# Load configuration +if [ -f "$FWDIR/conf/spark-env.sh" ]; then + source "$FWDIR/conf/spark-env.sh" +fi + # Parse arguments while (( "$#" )); do case $1 in @@ -17,12 +17,15 @@ # limitations under the License. # -EXTRA_ARGS="" -if [ "$MESOS_HOME" != "" ]; then - EXTRA_ARGS="-Djava.library.path=$MESOS_HOME/lib/java" -fi - export SPARK_HOME=$(cd "$(dirname $0)/.." 2>&1 >/dev/null ; pwd) export SPARK_TESTING=1 # To put test classes on classpath -java -Xmx1200m -XX:MaxPermSize=350m -XX:ReservedCodeCacheSize=128m $EXTRA_ARGS -jar $SPARK_HOME/sbt/sbt-launch-*.jar "$@" +if [ -f "$SPARK_HOME/conf/spark-env.sh" ]; then + source "$SPARK_HOME/conf/spark-env.sh" +fi + +if [ "$MESOS_HOME" != "" ]; then + EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS -Djava.library.path=$MESOS_HOME/lib/java" +fi + +java -Xmx1200m -XX:MaxPermSize=350m -XX:ReservedCodeCacheSize=128m $EXTRA_BUILD_ARGS -jar $SPARK_HOME/sbt/sbt-launch-*.jar "$@" |