diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2012-10-24 23:21:00 -0700 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2012-10-24 23:21:00 -0700 |
commit | 863a55ae42c2b9c0583b77cf37ff13bd2459f82b (patch) | |
tree | f4b18ebe461343ffb864dabb6afefcdf88dfafaf /run | |
parent | ed71df46cddc9a4f1363b937c10bfa2a928e564c (diff) | |
parent | f63a40fd99bf907c03cd44585fd5979bf21b304d (diff) | |
download | spark-863a55ae42c2b9c0583b77cf37ff13bd2459f82b.tar.gz spark-863a55ae42c2b9c0583b77cf37ff13bd2459f82b.tar.bz2 spark-863a55ae42c2b9c0583b77cf37ff13bd2459f82b.zip |
Merge remote-tracking branch 'public/master' into dev
Conflicts:
core/src/main/scala/spark/BlockStoreShuffleFetcher.scala
core/src/main/scala/spark/KryoSerializer.scala
core/src/main/scala/spark/MapOutputTracker.scala
core/src/main/scala/spark/RDD.scala
core/src/main/scala/spark/SparkContext.scala
core/src/main/scala/spark/executor/Executor.scala
core/src/main/scala/spark/network/Connection.scala
core/src/main/scala/spark/network/ConnectionManagerTest.scala
core/src/main/scala/spark/rdd/BlockRDD.scala
core/src/main/scala/spark/rdd/NewHadoopRDD.scala
core/src/main/scala/spark/scheduler/ShuffleMapTask.scala
core/src/main/scala/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
core/src/main/scala/spark/storage/BlockManager.scala
core/src/main/scala/spark/storage/BlockMessage.scala
core/src/main/scala/spark/storage/BlockStore.scala
core/src/main/scala/spark/storage/StorageLevel.scala
core/src/main/scala/spark/util/AkkaUtils.scala
project/SparkBuild.scala
run
Diffstat (limited to 'run')
-rwxr-xr-x | run | 58 |
1 files changed, 34 insertions, 24 deletions
@@ -1,6 +1,6 @@ #!/bin/bash -SCALA_VERSION=2.9.1 +SCALA_VERSION=2.9.2 # Figure out where the Scala framework is installed FWDIR="$(cd `dirname $0`; pwd)" @@ -13,16 +13,33 @@ if [ -e $FWDIR/conf/spark-env.sh ] ; then . $FWDIR/conf/spark-env.sh fi -# Check that SCALA_HOME has been specified -if [ -z "$SCALA_HOME" ]; then - echo "SCALA_HOME is not set" >&2 - exit 1 -fi - -# If the user specifies a Mesos JAR, put it before our included one on the classpath -MESOS_CLASSPATH="" -if [ -z "$MESOS_JAR" ] ; then - MESOS_CLASSPATH="$MESOS_JAR" +if [ "$SPARK_LAUNCH_WITH_SCALA" == "1" ]; then + if [ `command -v scala` ]; then + RUNNER="scala" + else + if [ -z "$SCALA_HOME" ]; then + echo "SCALA_HOME is not set" >&2 + exit 1 + fi + RUNNER="${SCALA_HOME}/bin/scala" + fi +else + if [ `command -v java` ]; then + RUNNER="java" + else + if [ -z "$JAVA_HOME" ]; then + echo "JAVA_HOME is not set" >&2 + exit 1 + fi + RUNNER="${JAVA_HOME}/bin/java" + fi + if [ -z "$SCALA_LIBRARY_PATH" ]; then + if [ -z "$SCALA_HOME" ]; then + echo "SCALA_HOME is not set" >&2 + exit 1 + fi + SCALA_LIBRARY_PATH="$SCALA_HOME/lib" + fi fi # Figure out how much memory to use per executor and set it as an environment @@ -50,16 +67,15 @@ STREAMING_DIR="$FWDIR/streaming" # Build up classpath CLASSPATH="$SPARK_CLASSPATH" -CLASSPATH+=":$MESOS_CLASSPATH" CLASSPATH+=":$FWDIR/conf" CLASSPATH+=":$CORE_DIR/target/scala-$SCALA_VERSION/classes" +if [ -n "$SPARK_TESTING" ] ; then + CLASSPATH+=":$CORE_DIR/target/scala-$SCALA_VERSION/test-classes" +fi CLASSPATH+=":$CORE_DIR/src/main/resources" CLASSPATH+=":$REPL_DIR/target/scala-$SCALA_VERSION/classes" CLASSPATH+=":$EXAMPLES_DIR/target/scala-$SCALA_VERSION/classes" CLASSPATH+=":$STREAMING_DIR/target/scala-$SCALA_VERSION/classes" -for jar in `find $CORE_DIR/lib -name '*jar'`; do - CLASSPATH+=":$jar" -done for jar in `find $FWDIR/lib_managed/jars -name '*jar'`; do CLASSPATH+=":$jar" done @@ -79,17 +95,11 @@ export CLASSPATH # Needed for spark-shell # the Spark shell, the wrapper is necessary to properly reset the terminal # when we exit, so we allow it to set a variable to launch with scala. if [ "$SPARK_LAUNCH_WITH_SCALA" == "1" ]; then - RUNNER="${SCALA_HOME}/bin/scala" EXTRA_ARGS="" # Java options will be passed to scala as JAVA_OPTS else - CLASSPATH+=":$SCALA_HOME/lib/scala-library.jar" - CLASSPATH+=":$SCALA_HOME/lib/scala-compiler.jar" - CLASSPATH+=":$SCALA_HOME/lib/jline.jar" - if [ -n "$JAVA_HOME" ]; then - RUNNER="${JAVA_HOME}/bin/java" - else - RUNNER=java - fi + CLASSPATH+=":$SCALA_LIBRARY_PATH/scala-library.jar" + CLASSPATH+=":$SCALA_LIBRARY_PATH/scala-compiler.jar" + CLASSPATH+=":$SCALA_LIBRARY_PATH/jline.jar" # The JVM doesn't read JAVA_OPTS by default so we need to pass it in EXTRA_ARGS="$JAVA_OPTS" fi |