diff options
author | shane-huang <shengsheng.huang@intel.com> | 2013-09-27 09:28:33 +0800 |
---|---|---|
committer | shane-huang <shengsheng.huang@intel.com> | 2013-09-27 09:28:33 +0800 |
commit | 84849baf88d31cfaaeee158a947c4db1abe94ce6 (patch) | |
tree | 364ffa6e77f252264e1ecaf9e517ff387920b8ab | |
parent | 714fdabd99bbff3a0cdec5dcf06b021a3a3f2da8 (diff) | |
parent | 3a5aa920fc9839aa99ea1befc467cc1f60230f3d (diff) | |
download | spark-84849baf88d31cfaaeee158a947c4db1abe94ce6.tar.gz spark-84849baf88d31cfaaeee158a947c4db1abe94ce6.tar.bz2 spark-84849baf88d31cfaaeee158a947c4db1abe94ce6.zip |
Merge branch 'reorgscripts' into scripts-reorg
-rw-r--r-- | assembly/src/main/assembly/assembly.xml | 11 | ||||
-rwxr-xr-x | bin/pyspark (renamed from pyspark) | 2 | ||||
-rw-r--r-- | bin/pyspark.cmd (renamed from pyspark.cmd) | 0 | ||||
-rw-r--r-- | bin/pyspark2.cmd (renamed from pyspark2.cmd) | 2 | ||||
-rwxr-xr-x | bin/run-example (renamed from run-example) | 4 | ||||
-rw-r--r-- | bin/run-example.cmd (renamed from run-example.cmd) | 0 | ||||
-rw-r--r-- | bin/run-example2.cmd (renamed from run-example2.cmd) | 4 | ||||
-rwxr-xr-x | bin/spark | 92 | ||||
-rwxr-xr-x | bin/spark-shell (renamed from spark-shell) | 4 | ||||
-rw-r--r-- | bin/spark-shell.cmd (renamed from spark-shell.cmd) | 5 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala | 2 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala | 4 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala | 4 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala | 2 | ||||
-rw-r--r-- | core/src/test/scala/org/apache/spark/DriverSuite.scala | 2 | ||||
-rw-r--r-- | data/kmeans_data.txt (renamed from kmeans_data.txt) | 0 | ||||
-rw-r--r-- | data/lr_data.txt (renamed from lr_data.txt) | 0 | ||||
-rw-r--r-- | data/pagerank_data.txt (renamed from pagerank_data.txt) | 0 | ||||
-rw-r--r-- | docs/running-on-yarn.md | 4 | ||||
-rw-r--r-- | docs/spark-standalone.md | 14 | ||||
-rwxr-xr-x | make-distribution.sh | 5 | ||||
-rw-r--r-- | python/pyspark/java_gateway.py | 2 | ||||
-rwxr-xr-x | python/run-tests | 2 | ||||
-rw-r--r-- | sbin/compute-classpath.cmd (renamed from bin/compute-classpath.cmd) | 0 | ||||
-rwxr-xr-x | sbin/compute-classpath.sh (renamed from bin/compute-classpath.sh) | 0 | ||||
-rwxr-xr-x | sbin/slaves.sh (renamed from bin/slaves.sh) | 6 | ||||
-rwxr-xr-x | sbin/spark-class (renamed from spark-class) | 4 | ||||
-rw-r--r-- | sbin/spark-class.cmd (renamed from spark-class.cmd) | 0 | ||||
-rw-r--r-- | sbin/spark-class2.cmd (renamed from spark-class2.cmd) | 4 | ||||
-rwxr-xr-x | sbin/spark-config.sh (renamed from bin/spark-config.sh) | 0 | ||||
-rwxr-xr-x | sbin/spark-daemon.sh (renamed from bin/spark-daemon.sh) | 8 | ||||
-rwxr-xr-x | sbin/spark-daemons.sh (renamed from bin/spark-daemons.sh) | 8 | ||||
-rwxr-xr-x | sbin/spark-executor (renamed from spark-executor) | 5 | ||||
-rwxr-xr-x | sbin/start-all.sh (renamed from bin/start-all.sh) | 10 | ||||
-rwxr-xr-x | sbin/start-master.sh (renamed from bin/start-master.sh) | 8 | ||||
-rwxr-xr-x | sbin/start-slave.sh (renamed from bin/start-slave.sh) | 6 | ||||
-rwxr-xr-x | sbin/start-slaves.sh (renamed from bin/start-slaves.sh) | 10 | ||||
-rwxr-xr-x | sbin/stop-all.sh (renamed from bin/stop-all.sh) | 10 | ||||
-rwxr-xr-x | sbin/stop-master.sh (renamed from bin/stop-master.sh) | 8 | ||||
-rwxr-xr-x | sbin/stop-slaves.sh (renamed from bin/stop-slaves.sh) | 10 |
40 files changed, 175 insertions, 87 deletions
diff --git a/assembly/src/main/assembly/assembly.xml b/assembly/src/main/assembly/assembly.xml index 47d3fa93d0..6af383db65 100644 --- a/assembly/src/main/assembly/assembly.xml +++ b/assembly/src/main/assembly/assembly.xml @@ -39,23 +39,20 @@ </fileSet> <fileSet> <directory> - ${project.parent.basedir}/bin/ + ${project.parent.basedir}/sbin/ </directory> - <outputDirectory>/bin</outputDirectory> + <outputDirectory>/sbin</outputDirectory> <includes> <include>**/*</include> </includes> </fileSet> <fileSet> <directory> - ${project.parent.basedir} + ${project.parent.basedir}/bin/ </directory> <outputDirectory>/bin</outputDirectory> <includes> - <include>run-example*</include> - <include>spark-class*</include> - <include>spark-shell*</include> - <include>spark-executor*</include> + <include>**/*</include> </includes> </fileSet> </fileSets> @@ -18,7 +18,7 @@ # # Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; pwd)" +FWDIR="$(cd `dirname $0`/..; pwd)" # Export this as SPARK_HOME export SPARK_HOME="$FWDIR" diff --git a/pyspark.cmd b/bin/pyspark.cmd index 7c26fbbac2..7c26fbbac2 100644 --- a/pyspark.cmd +++ b/bin/pyspark.cmd diff --git a/pyspark2.cmd b/bin/pyspark2.cmd index f58e349643..bb8e624c2f 100644 --- a/pyspark2.cmd +++ b/bin/pyspark2.cmd @@ -20,7 +20,7 @@ rem set SCALA_VERSION=2.9.3 rem Figure out where the Spark framework is installed -set FWDIR=%~dp0 +set FWDIR=%~dp0..\ rem Export this as SPARK_HOME set SPARK_HOME=%FWDIR% diff --git a/run-example b/bin/run-example index 08ec717ca5..f29bb2c015 100755 --- a/run-example +++ b/bin/run-example @@ -20,7 +20,7 @@ SCALA_VERSION=2.9.3 # Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; pwd)" +FWDIR="$(cd `dirname $0`/..; pwd)" # Export this as SPARK_HOME export SPARK_HOME="$FWDIR" @@ -56,7 +56,7 @@ fi # Since the examples JAR ideally shouldn't include spark-core (that dependency should be # "provided"), also add our standard Spark classpath, built using compute-classpath.sh. -CLASSPATH=`$FWDIR/bin/compute-classpath.sh` +CLASSPATH=`$FWDIR/sbin/compute-classpath.sh` CLASSPATH="$SPARK_EXAMPLES_JAR:$CLASSPATH" # Find java binary diff --git a/run-example.cmd b/bin/run-example.cmd index 5b2d048d6e..5b2d048d6e 100644 --- a/run-example.cmd +++ b/bin/run-example.cmd diff --git a/run-example2.cmd b/bin/run-example2.cmd index dbb371ecfc..e250a92965 100644 --- a/run-example2.cmd +++ b/bin/run-example2.cmd @@ -20,7 +20,7 @@ rem set SCALA_VERSION=2.9.3 rem Figure out where the Spark framework is installed -set FWDIR=%~dp0 +set FWDIR=%~dp0..\ rem Export this as SPARK_HOME set SPARK_HOME=%FWDIR% @@ -49,7 +49,7 @@ if "x%SPARK_EXAMPLES_JAR%"=="x" ( rem Compute Spark classpath using external script set DONT_PRINT_CLASSPATH=1 -call "%FWDIR%bin\compute-classpath.cmd" +call "%FWDIR%sbin\compute-classpath.cmd" set DONT_PRINT_CLASSPATH=0 set CLASSPATH=%SPARK_EXAMPLES_JAR%;%CLASSPATH% diff --git a/bin/spark b/bin/spark new file mode 100755 index 0000000000..aa005a51f5 --- /dev/null +++ b/bin/spark @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SCALA_VERSION=2.9.3 + +# Figure out where the Scala framework is installed +FWDIR="$(cd `dirname $0`/..; pwd)" + +# Export this as SPARK_HOME +export SPARK_HOME="$FWDIR" + +# Load environment variables from conf/spark-env.sh, if it exists +if [ -e $FWDIR/conf/spark-env.sh ] ; then + . $FWDIR/conf/spark-env.sh +fi + +if [ -z "$1" ]; then + echo "Usage: spark <class> [<args>]" >&2 + echo "Usage: export SPARK_CLASSPATH before running the command" >&2 + exit 1 +fi + + +# Find the java binary +if [ -n "${JAVA_HOME}" ]; then + RUNNER="${JAVA_HOME}/bin/java" +else + if [ `command -v java` ]; then + RUNNER="java" + else + echo "JAVA_HOME is not set" >&2 + exit 1 + fi +fi + +# Set SPARK_MEM if it isn't already set +SPARK_MEM=${SPARK_MEM:-512m} +export SPARK_MEM + +# Set APP_MEM if it isn't already set, we use this for this process as the app driver process may need +# as much memory as specified in SPARK_MEM +APP_MEM=${APP_MEM:-512m} + +# Set JAVA_OPTS to be able to load native libraries and to set heap size +JAVA_OPTS="$OUR_JAVA_OPTS" +JAVA_OPTS="$JAVA_OPTS -Djava.library.path=$SPARK_LIBRARY_PATH" +JAVA_OPTS="$JAVA_OPTS -Xms$APP_MEM -Xmx$APP_MEM" +# Load extra JAVA_OPTS from conf/java-opts, if it exists +if [ -e $FWDIR/conf/java-opts ] ; then + JAVA_OPTS="$JAVA_OPTS `cat $FWDIR/conf/java-opts`" +fi +export JAVA_OPTS +# Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected in ExecutorRunner.scala! + +if [ ! -f "$FWDIR/RELEASE" ]; then + # Exit if the user hasn't compiled Spark + ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/spark-assembly*hadoop*.jar >& /dev/null + if [[ $? != 0 ]]; then + echo "Failed to find Spark assembly in $FWDIR/assembly/target" >&2 + echo "You need to build Spark with sbt/sbt assembly before running this program" >&2 + exit 1 + fi +fi + +# Compute classpath using external script +CLASSPATH=`$FWDIR/sbin/compute-classpath.sh` +export CLASSPATH + +if [ "$SPARK_PRINT_LAUNCH_COMMAND" == "1" ]; then + echo -n "Spark Command: " + echo "$RUNNER" -cp "$CLASSPATH" $JAVA_OPTS "$@" + echo "========================================" + echo +fi + +exec "$RUNNER" -cp "$CLASSPATH" $JAVA_OPTS "$@" diff --git a/spark-shell b/bin/spark-shell index 9608bd3f30..6717fe7049 100755 --- a/spark-shell +++ b/bin/spark-shell @@ -28,7 +28,7 @@ # Enter posix mode for bash set -o posix -FWDIR="`dirname $0`" +FWDIR="$(cd `dirname $0`/..; pwd)" for o in "$@"; do if [ "$1" = "-c" -o "$1" = "--cores" ]; then @@ -79,7 +79,7 @@ if [[ ! $? ]]; then saved_stty="" fi -$FWDIR/spark-class $OPTIONS org.apache.spark.repl.Main "$@" +$FWDIR/sbin/spark-class $OPTIONS org.apache.spark.repl.Main "$@" # record the exit status lest it be overwritten: # then reenable echo and propagate the code. diff --git a/spark-shell.cmd b/bin/spark-shell.cmd index 3e52bf835e..23973e3e3d 100644 --- a/spark-shell.cmd +++ b/bin/spark-shell.cmd @@ -17,6 +17,7 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem -set FWDIR=%~dp0 +rem Find the path of sbin +set SBIN=%~dp0..\sbin\ -cmd /V /E /C %FWDIR%spark-class2.cmd org.apache.spark.repl.Main %* +cmd /V /E /C %SBIN%spark-class2.cmd org.apache.spark.repl.Main %* diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala index e3dc30eefc..d48545ecc1 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala @@ -120,7 +120,7 @@ private[spark] class ExecutorRunner( // Figure out our classpath with the external compute-classpath script val ext = if (System.getProperty("os.name").startsWith("Windows")) ".cmd" else ".sh" val classPath = Utils.executeAndGetOutput( - Seq(sparkHome + "/bin/compute-classpath" + ext), + Seq(sparkHome + "/sbin/compute-classpath" + ext), extraEnvironment=appDesc.command.environment) Seq("-cp", classPath) ++ libraryOpts ++ workerLocalOpts ++ userOpts ++ memoryOpts diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala index 8f2eef9a53..15b3397a3a 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/CoarseMesosSchedulerBackend.scala @@ -125,7 +125,7 @@ private[spark] class CoarseMesosSchedulerBackend( StandaloneSchedulerBackend.ACTOR_NAME) val uri = System.getProperty("spark.executor.uri") if (uri == null) { - val runScript = new File(sparkHome, "spark-class").getCanonicalPath + val runScript = new File(sparkHome, "./sbin/spark-class").getCanonicalPath command.setValue( "\"%s\" org.apache.spark.executor.StandaloneExecutorBackend %s %s %s %d".format( runScript, driverUrl, offer.getSlaveId.getValue, offer.getHostname, numCores)) @@ -134,7 +134,7 @@ private[spark] class CoarseMesosSchedulerBackend( // glob the directory "correctly". val basename = uri.split('/').last.split('.').head command.setValue( - "cd %s*; ./spark-class org.apache.spark.executor.StandaloneExecutorBackend %s %s %s %d".format( + "cd %s*; ./sbin/spark-class org.apache.spark.executor.StandaloneExecutorBackend %s %s %s %d".format( basename, driverUrl, offer.getSlaveId.getValue, offer.getHostname, numCores)) command.addUris(CommandInfo.URI.newBuilder().setValue(uri)) } diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index 50cbc2ca92..7e9c05cd66 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -103,12 +103,12 @@ private[spark] class MesosSchedulerBackend( .setEnvironment(environment) val uri = System.getProperty("spark.executor.uri") if (uri == null) { - command.setValue(new File(sparkHome, "spark-executor").getCanonicalPath) + command.setValue(new File(sparkHome, "/sbin/spark-executor").getCanonicalPath) } else { // Grab everything to the first '.'. We'll use that and '*' to // glob the directory "correctly". val basename = uri.split('/').last.split('.').head - command.setValue("cd %s*; ./spark-executor".format(basename)) + command.setValue("cd %s*; ./sbin/spark-executor".format(basename)) command.addUris(CommandInfo.URI.newBuilder().setValue(uri)) } val memory = Resource.newBuilder() diff --git a/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala b/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala index 453394dfda..b4128b71e7 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIWorkloadGenerator.scala @@ -35,7 +35,7 @@ private[spark] object UIWorkloadGenerator { def main(args: Array[String]) { if (args.length < 2) { - println("usage: ./spark-class spark.ui.UIWorkloadGenerator [master] [FIFO|FAIR]") + println("usage: ./sbin/spark-class spark.ui.UIWorkloadGenerator [master] [FIFO|FAIR]") System.exit(1) } val master = args(0) diff --git a/core/src/test/scala/org/apache/spark/DriverSuite.scala b/core/src/test/scala/org/apache/spark/DriverSuite.scala index 01a72d8401..5e5ece0033 100644 --- a/core/src/test/scala/org/apache/spark/DriverSuite.scala +++ b/core/src/test/scala/org/apache/spark/DriverSuite.scala @@ -35,7 +35,7 @@ class DriverSuite extends FunSuite with Timeouts { val masters = Table(("master"), ("local"), ("local-cluster[2,1,512]")) forAll(masters) { (master: String) => failAfter(30 seconds) { - Utils.execute(Seq("./spark-class", "org.apache.spark.DriverWithoutCleanup", master), + Utils.execute(Seq("./sbin/spark-class", "org.apache.spark.DriverWithoutCleanup", master), new File(System.getenv("SPARK_HOME"))) } } diff --git a/kmeans_data.txt b/data/kmeans_data.txt index 338664f78d..338664f78d 100644 --- a/kmeans_data.txt +++ b/data/kmeans_data.txt diff --git a/lr_data.txt b/data/lr_data.txt index d4df0634e0..d4df0634e0 100644 --- a/lr_data.txt +++ b/data/lr_data.txt diff --git a/pagerank_data.txt b/data/pagerank_data.txt index 95755ab8f5..95755ab8f5 100644 --- a/pagerank_data.txt +++ b/data/pagerank_data.txt diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index c611db0af4..767eb5cdac 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -42,7 +42,7 @@ This would be used to connect to the cluster, write to the dfs and submit jobs t The command to launch the YARN Client is as follows: - SPARK_JAR=<SPARK_ASSEMBLY_JAR_FILE> ./spark-class org.apache.spark.deploy.yarn.Client \ + SPARK_JAR=<SPARK_ASSEMBLY_JAR_FILE> ./sbin/spark-class org.apache.spark.deploy.yarn.Client \ --jar <YOUR_APP_JAR_FILE> \ --class <APP_MAIN_CLASS> \ --args <APP_MAIN_ARGUMENTS> \ @@ -62,7 +62,7 @@ For example: # Submit Spark's ApplicationMaster to YARN's ResourceManager, and instruct Spark to run the SparkPi example $ SPARK_JAR=./assembly/target/scala-{{site.SCALA_VERSION}}/spark-assembly-{{site.SPARK_VERSION}}-hadoop2.0.5-alpha.jar \ - ./spark-class org.apache.spark.deploy.yarn.Client \ + ./sbin/spark-class org.apache.spark.deploy.yarn.Client \ --jar examples/target/scala-{{site.SCALA_VERSION}}/spark-examples-assembly-{{site.SPARK_VERSION}}.jar \ --class org.apache.spark.examples.SparkPi \ --args yarn-standalone \ diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md index 81cdbefd0c..b3f9160673 100644 --- a/docs/spark-standalone.md +++ b/docs/spark-standalone.md @@ -25,7 +25,7 @@ the master's web UI, which is [http://localhost:8080](http://localhost:8080) by Similarly, you can start one or more workers and connect them to the master via: - ./spark-class org.apache.spark.deploy.worker.Worker spark://IP:PORT + ./sbin/spark-class org.apache.spark.deploy.worker.Worker spark://IP:PORT Once you have started a worker, look at the master's web UI ([http://localhost:8080](http://localhost:8080) by default). You should see the new node listed there, along with its number of CPUs and memory (minus one gigabyte left for the OS). @@ -67,12 +67,12 @@ To launch a Spark standalone cluster with the launch scripts, you need to create Once you've set up this file, you can launch or stop your cluster with the following shell scripts, based on Hadoop's deploy scripts, and available in `SPARK_HOME/bin`: -- `bin/start-master.sh` - Starts a master instance on the machine the script is executed on. -- `bin/start-slaves.sh` - Starts a slave instance on each machine specified in the `conf/slaves` file. -- `bin/start-all.sh` - Starts both a master and a number of slaves as described above. -- `bin/stop-master.sh` - Stops the master that was started via the `bin/start-master.sh` script. -- `bin/stop-slaves.sh` - Stops the slave instances that were started via `bin/start-slaves.sh`. -- `bin/stop-all.sh` - Stops both the master and the slaves as described above. +- `sbin/start-master.sh` - Starts a master instance on the machine the script is executed on. +- `sbin/start-slaves.sh` - Starts a slave instance on each machine specified in the `conf/slaves` file. +- `sbin/start-all.sh` - Starts both a master and a number of slaves as described above. +- `sbin/stop-master.sh` - Stops the master that was started via the `bin/start-master.sh` script. +- `sbin/stop-slaves.sh` - Stops the slave instances that were started via `bin/start-slaves.sh`. +- `sbin/stop-all.sh` - Stops both the master and the slaves as described above. Note that these scripts must be executed on the machine you want to run the Spark master on, not your local machine. diff --git a/make-distribution.sh b/make-distribution.sh index bffb19843c..13843bb6c2 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -98,10 +98,7 @@ mkdir "$DISTDIR"/conf cp "$FWDIR/conf/*.template" "$DISTDIR"/conf cp -r "$FWDIR/bin" "$DISTDIR" cp -r "$FWDIR/python" "$DISTDIR" -cp "$FWDIR/spark-class" "$DISTDIR" -cp "$FWDIR/spark-shell" "$DISTDIR" -cp "$FWDIR/spark-executor" "$DISTDIR" -cp "$FWDIR/pyspark" "$DISTDIR" +cp -r "$FWDIR/sbin" "$DISTDIR" if [ "$MAKE_TGZ" == "true" ]; then diff --git a/python/pyspark/java_gateway.py b/python/pyspark/java_gateway.py index e615c1e9b6..b872ae61d5 100644 --- a/python/pyspark/java_gateway.py +++ b/python/pyspark/java_gateway.py @@ -31,7 +31,7 @@ def launch_gateway(): # Launch the Py4j gateway using Spark's run command so that we pick up the # proper classpath and SPARK_MEM settings from spark-env.sh on_windows = platform.system() == "Windows" - script = "spark-class.cmd" if on_windows else "spark-class" + script = "./sbin/spark-class.cmd" if on_windows else "./sbin/spark-class" command = [os.path.join(SPARK_HOME, script), "py4j.GatewayServer", "--die-on-broken-pipe", "0"] if not on_windows: diff --git a/python/run-tests b/python/run-tests index cbc554ea9d..8a08ae3df9 100755 --- a/python/run-tests +++ b/python/run-tests @@ -29,7 +29,7 @@ FAILED=0 rm -f unit-tests.log function run_test() { - $FWDIR/pyspark $1 2>&1 | tee -a unit-tests.log + $FWDIR/bin/pyspark $1 2>&1 | tee -a unit-tests.log FAILED=$((PIPESTATUS[0]||$FAILED)) } diff --git a/bin/compute-classpath.cmd b/sbin/compute-classpath.cmd index cf38188c4b..cf38188c4b 100644 --- a/bin/compute-classpath.cmd +++ b/sbin/compute-classpath.cmd diff --git a/bin/compute-classpath.sh b/sbin/compute-classpath.sh index c7819d4932..c7819d4932 100755 --- a/bin/compute-classpath.sh +++ b/sbin/compute-classpath.sh diff --git a/bin/slaves.sh b/sbin/slaves.sh index 752565b759..68408bcad8 100755 --- a/bin/slaves.sh +++ b/sbin/slaves.sh @@ -36,10 +36,10 @@ if [ $# -le 0 ]; then exit 1 fi -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" # If the slaves file is specified in the command line, # then it takes precedence over the definition in diff --git a/spark-class b/sbin/spark-class index e111ef6da7..7f25fe1050 100755 --- a/spark-class +++ b/sbin/spark-class @@ -20,7 +20,7 @@ SCALA_VERSION=2.9.3 # Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; pwd)" +FWDIR="$(cd `dirname $0`/..; pwd)" # Export this as SPARK_HOME export SPARK_HOME="$FWDIR" @@ -104,7 +104,7 @@ if [ ! -f "$FWDIR/RELEASE" ]; then fi # Compute classpath using external script -CLASSPATH=`$FWDIR/bin/compute-classpath.sh` +CLASSPATH=`$FWDIR/sbin/compute-classpath.sh` export CLASSPATH if [ "$SPARK_PRINT_LAUNCH_COMMAND" == "1" ]; then diff --git a/spark-class.cmd b/sbin/spark-class.cmd index 19850db9e1..19850db9e1 100644 --- a/spark-class.cmd +++ b/sbin/spark-class.cmd diff --git a/spark-class2.cmd b/sbin/spark-class2.cmd index d4d853e8ad..5e00bd39ea 100644 --- a/spark-class2.cmd +++ b/sbin/spark-class2.cmd @@ -20,7 +20,7 @@ rem set SCALA_VERSION=2.9.3 rem Figure out where the Spark framework is installed -set FWDIR=%~dp0 +set FWDIR=%~dp0..\ rem Export this as SPARK_HOME set SPARK_HOME=%FWDIR% @@ -67,7 +67,7 @@ if "%FOUND_JAR%"=="0" ( rem Compute classpath using external script set DONT_PRINT_CLASSPATH=1 -call "%FWDIR%bin\compute-classpath.cmd" +call "%FWDIR%sbin\compute-classpath.cmd" set DONT_PRINT_CLASSPATH=0 rem Figure out where java is. diff --git a/bin/spark-config.sh b/sbin/spark-config.sh index cd2c7b7b0d..cd2c7b7b0d 100755 --- a/bin/spark-config.sh +++ b/sbin/spark-config.sh diff --git a/bin/spark-daemon.sh b/sbin/spark-daemon.sh index 5bfe967fbf..ae82349cc6 100755 --- a/bin/spark-daemon.sh +++ b/sbin/spark-daemon.sh @@ -37,10 +37,10 @@ if [ $# -le 1 ]; then exit 1 fi -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" # get arguments startStop=$1 @@ -128,7 +128,7 @@ case $startStop in spark_rotate_log "$log" echo starting $command, logging to $log cd "$SPARK_PREFIX" - nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & + nohup nice -n $SPARK_NICENESS "$SPARK_PREFIX"/sbin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null & newpid=$! echo $newpid > $pid sleep 2 diff --git a/bin/spark-daemons.sh b/sbin/spark-daemons.sh index 354eb905a1..d91254b690 100755 --- a/bin/spark-daemons.sh +++ b/sbin/spark-daemons.sh @@ -27,9 +27,9 @@ if [ $# -le 1 ]; then exit 1 fi -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" -exec "$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/spark-daemon.sh" "$@" +exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/spark-daemon.sh" "$@" diff --git a/spark-executor b/sbin/spark-executor index 2c07c54843..214e00f6f8 100755 --- a/spark-executor +++ b/sbin/spark-executor @@ -17,6 +17,7 @@ # limitations under the License. # -FWDIR="`dirname $0`" +FWDIR="$(cd `dirname $0`/..; pwd)" + echo "Running spark-executor with framework dir = $FWDIR" -exec $FWDIR/spark-class org.apache.spark.executor.MesosExecutorBackend +exec $FWDIR/sbin/spark-class org.apache.spark.executor.MesosExecutorBackend diff --git a/bin/start-all.sh b/sbin/start-all.sh index 0182f1ab24..2daf49db35 100755 --- a/bin/start-all.sh +++ b/sbin/start-all.sh @@ -21,14 +21,14 @@ # Starts the master on this node. # Starts a worker on each node specified in conf/slaves -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` # Load the Spark configuration -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" # Start Master -"$bin"/start-master.sh +"$sbin"/start-master.sh # Start Workers -"$bin"/start-slaves.sh +"$sbin"/start-slaves.sh diff --git a/bin/start-master.sh b/sbin/start-master.sh index 648c7ae75f..3dcf7cc348 100755 --- a/bin/start-master.sh +++ b/sbin/start-master.sh @@ -19,10 +19,10 @@ # Starts the master on the machine this script is executed on. -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then . "${SPARK_CONF_DIR}/spark-env.sh" @@ -49,4 +49,4 @@ if [ "$SPARK_PUBLIC_DNS" = "" ]; then fi fi -"$bin"/spark-daemon.sh start org.apache.spark.deploy.master.Master 1 --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT +"$sbin"/spark-daemon.sh start org.apache.spark.deploy.master.Master 1 --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT diff --git a/bin/start-slave.sh b/sbin/start-slave.sh index 4eefa20944..524be38c62 100755 --- a/bin/start-slave.sh +++ b/sbin/start-slave.sh @@ -20,8 +20,8 @@ # Usage: start-slave.sh <worker#> <master-spark-URL> # where <master-spark-URL> is like "spark://localhost:7077" -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` # Set SPARK_PUBLIC_DNS so slaves can be linked in master web UI if [ "$SPARK_PUBLIC_DNS" = "" ]; then @@ -32,4 +32,4 @@ if [ "$SPARK_PUBLIC_DNS" = "" ]; then fi fi -"$bin"/spark-daemon.sh start org.apache.spark.deploy.worker.Worker "$@" +"$sbin"/spark-daemon.sh start org.apache.spark.deploy.worker.Worker "$@" diff --git a/bin/start-slaves.sh b/sbin/start-slaves.sh index 00dc4888b2..fd5cdeb1e6 100755 --- a/bin/start-slaves.sh +++ b/sbin/start-slaves.sh @@ -17,10 +17,10 @@ # limitations under the License. # -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then . "${SPARK_CONF_DIR}/spark-env.sh" @@ -37,12 +37,12 @@ fi # Launch the slaves if [ "$SPARK_WORKER_INSTANCES" = "" ]; then - exec "$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT + exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT else if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then SPARK_WORKER_WEBUI_PORT=8081 fi for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do - "$bin/slaves.sh" cd "$SPARK_HOME" \; "$bin/start-slave.sh" $(( $i + 1 )) spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT --webui-port $(( $SPARK_WORKER_WEBUI_PORT + $i )) + "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" $(( $i + 1 )) spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT --webui-port $(( $SPARK_WORKER_WEBUI_PORT + $i )) done fi diff --git a/bin/stop-all.sh b/sbin/stop-all.sh index b6c83a7ba4..60b358d374 100755 --- a/bin/stop-all.sh +++ b/sbin/stop-all.sh @@ -21,12 +21,12 @@ # Run this on the master nde -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` # Load the Spark configuration -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" # Stop the slaves, then the master -"$bin"/stop-slaves.sh -"$bin"/stop-master.sh +"$sbin"/stop-slaves.sh +"$sbin"/stop-master.sh diff --git a/bin/stop-master.sh b/sbin/stop-master.sh index 310e33bedc..2adabd4265 100755 --- a/bin/stop-master.sh +++ b/sbin/stop-master.sh @@ -19,9 +19,9 @@ # Starts the master on the machine this script is executed on. -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" -"$bin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 +"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1 diff --git a/bin/stop-slaves.sh b/sbin/stop-slaves.sh index 03e416a132..63802e6df5 100755 --- a/bin/stop-slaves.sh +++ b/sbin/stop-slaves.sh @@ -19,19 +19,19 @@ # Starts the master on the machine this script is executed on. -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +sbin=`dirname "$0"` +sbin=`cd "$sbin"; pwd` -. "$bin/spark-config.sh" +. "$sbin/spark-config.sh" if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then . "${SPARK_CONF_DIR}/spark-env.sh" fi if [ "$SPARK_WORKER_INSTANCES" = "" ]; then - "$bin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker 1 + "$sbin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker 1 else for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do - "$bin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker $(( $i + 1 )) + "$sbin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker $(( $i + 1 )) done fi |