diff options
author | Denny <dennybritz@gmail.com> | 2012-08-02 15:59:39 -0700 |
---|---|---|
committer | Denny <dennybritz@gmail.com> | 2012-08-02 15:59:39 -0700 |
commit | 53008c2d8a8b28f4204eaafa89f0e8087dc11466 (patch) | |
tree | 1f1ee1568a80a0e42043cc0b130753dd791fa670 | |
parent | aaed039e36b6b29b59e650ef075f66a6e4162449 (diff) | |
download | spark-53008c2d8a8b28f4204eaafa89f0e8087dc11466.tar.gz spark-53008c2d8a8b28f4204eaafa89f0e8087dc11466.tar.bz2 spark-53008c2d8a8b28f4204eaafa89f0e8087dc11466.zip |
Settings variables and bugfix for stop script.
-rwxr-xr-x | bin/stop-master.sh | 2 | ||||
-rwxr-xr-x | bin/stop-slaves.sh | 2 | ||||
-rwxr-xr-x | conf/spark-env.sh.template | 10 | ||||
-rw-r--r-- | core/src/main/scala/spark/deploy/master/MasterArguments.scala | 13 | ||||
-rw-r--r-- | core/src/main/scala/spark/deploy/worker/WorkerArguments.scala | 16 |
5 files changed, 38 insertions, 5 deletions
diff --git a/bin/stop-master.sh b/bin/stop-master.sh index f33f71664e..f75167dd2c 100755 --- a/bin/stop-master.sh +++ b/bin/stop-master.sh @@ -7,4 +7,4 @@ bin=`cd "$bin"; pwd` . "$bin/spark-config.sh" -"$bin"/spark-daemon.sh stop spark.deploy.worker.Worker
\ No newline at end of file +"$bin"/spark-daemon.sh stop spark.deploy.master.Master
\ No newline at end of file diff --git a/bin/stop-slaves.sh b/bin/stop-slaves.sh index f75167dd2c..21c9ebf324 100755 --- a/bin/stop-slaves.sh +++ b/bin/stop-slaves.sh @@ -7,4 +7,4 @@ bin=`cd "$bin"; pwd` . "$bin/spark-config.sh" -"$bin"/spark-daemon.sh stop spark.deploy.master.Master
\ No newline at end of file +"$bin"/spark-daemons.sh stop spark.deploy.worker.Worker
\ No newline at end of file diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index c09af42717..64eacce8a2 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -9,5 +9,13 @@ # - SPARK_MEM, to change the amount of memory used per node (this should # be in the same format as the JVM's -Xmx option, e.g. 300m or 1g). # - SPARK_LIBRARY_PATH, to add extra search paths for native libraries. -# - SPARK_MASTER_PORT, to start the spark master on a different port (standalone mode only) +# Settings used by the scripts in the bin/ directory, apply to standalone mode only. +# Note that the same worker settings apply to all of the workers. +# - SPARK_MASTER_IP, to bind the master to a different ip address, for example a public one (Default: local ip address) +# - SPARK_MASTER_PORT, to start the spark master on a different port (Default: 7077) +# - SPARK_MASTER_WEBUI_PORT, to specify a different port for the Master WebUI (Default: 8080) +# - SPARK_WORKER_PORT, to start the spark worker on a specific port (Default: random) +# - SPARK_WORKER_CORES, to specify the number of cores to use (Default: all available cores) +# - SPARK_WORKER_MEMORY, to specify how much memory to use, e.g. 1000M, 2G (Default: MAX(Available - 1024MB, 512MB)) +# - SPARK_WORKER_WEBUI_PORT, to specify a different port for the Worker WebUI (Default: 8081)
\ No newline at end of file diff --git a/core/src/main/scala/spark/deploy/master/MasterArguments.scala b/core/src/main/scala/spark/deploy/master/MasterArguments.scala index 5d975cd546..0f7a92bdd0 100644 --- a/core/src/main/scala/spark/deploy/master/MasterArguments.scala +++ b/core/src/main/scala/spark/deploy/master/MasterArguments.scala @@ -10,7 +10,18 @@ class MasterArguments(args: Array[String]) { var ip = Utils.localIpAddress() var port = 7077 var webUiPort = 8080 - + + // Check for settings in environment variables + if (System.getenv("SPARK_MASTER_IP") != null) { + ip = System.getenv("SPARK_MASTER_IP") + } + if (System.getenv("SPARK_MASTER_PORT") != null) { + port = System.getenv("SPARK_MASTER_PORT").toInt + } + if (System.getenv("SPARK_MASTER_WEBUI_PORT") != null) { + webUiPort = System.getenv("SPARK_MASTER_WEBUI_PORT").toInt + } + parse(args.toList) def parse(args: List[String]): Unit = args match { diff --git a/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala b/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala index 3248d03697..1efe8304ea 100644 --- a/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala +++ b/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala @@ -15,7 +15,21 @@ class WorkerArguments(args: Array[String]) { var cores = inferDefaultCores() var memory = inferDefaultMemory() var master: String = null - + + // Check for settings in environment variables + if (System.getenv("SPARK_WORKER_PORT") != null) { + port = System.getenv("SPARK_WORKER_PORT").toInt + } + if (System.getenv("SPARK_WORKER_CORES") != null) { + cores = System.getenv("SPARK_WORKER_CORES").toInt + } + if (System.getenv("SPARK_WORKER_MEMORY") != null) { + memory = Utils.memoryStringToMb(System.getenv("SPARK_WORKER_MEMORY")) + } + if (System.getenv("SPARK_WORKER_WEBUI_PORT") != null) { + webUiPort = System.getenv("SPARK_WORKER_WEBUI_PORT").toInt + } + parse(args.toList) def parse(args: List[String]): Unit = args match { |