diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2012-09-26 22:54:39 -0700 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2012-09-26 22:54:39 -0700 |
commit | ea05fc130b64ce356ab7524a3d5bd1e022cf51b5 (patch) | |
tree | 551ac8546cb21aa750a0967ef115e16639b0ef64 /core | |
parent | 1ef4f0fbd27e54803f14fed1df541fb341daced8 (diff) | |
download | spark-ea05fc130b64ce356ab7524a3d5bd1e022cf51b5.tar.gz spark-ea05fc130b64ce356ab7524a3d5bd1e022cf51b5.tar.bz2 spark-ea05fc130b64ce356ab7524a3d5bd1e022cf51b5.zip |
Updates to standalone cluster, web UI and deploy docs.
Diffstat (limited to 'core')
17 files changed, 42 insertions, 19 deletions
diff --git a/core/src/main/scala/spark/deploy/master/MasterArguments.scala b/core/src/main/scala/spark/deploy/master/MasterArguments.scala index 0f7a92bdd0..d712e3d5b3 100644 --- a/core/src/main/scala/spark/deploy/master/MasterArguments.scala +++ b/core/src/main/scala/spark/deploy/master/MasterArguments.scala @@ -51,7 +51,7 @@ class MasterArguments(args: Array[String]) { */ def printUsageAndExit(exitCode: Int) { System.err.println( - "Usage: spark-master [options]\n" + + "Usage: Master [options]\n" + "\n" + "Options:\n" + " -i IP, --ip IP IP address or DNS name to listen on\n" + diff --git a/core/src/main/scala/spark/deploy/master/MasterWebUI.scala b/core/src/main/scala/spark/deploy/master/MasterWebUI.scala index f03c0a0229..9f9994e4ba 100644 --- a/core/src/main/scala/spark/deploy/master/MasterWebUI.scala +++ b/core/src/main/scala/spark/deploy/master/MasterWebUI.scala @@ -22,7 +22,7 @@ class MasterWebUI(val actorSystem: ActorSystem, master: ActorRef) extends Direct completeWith { val future = master ? RequestMasterState future.map { - masterState => masterui.html.index.render(masterState.asInstanceOf[MasterState]) + masterState => spark.deploy.master.html.index.render(masterState.asInstanceOf[MasterState]) } } } ~ @@ -36,7 +36,7 @@ class MasterWebUI(val actorSystem: ActorSystem, master: ActorRef) extends Direct // A bit ugly an inefficient, but we won't have a number of jobs // so large that it will make a significant difference. (masterState.activeJobs ::: masterState.completedJobs).find(_.id == jobId) match { - case Some(job) => masterui.html.job_details.render(job) + case Some(job) => spark.deploy.master.html.job_details.render(job) case _ => null } } diff --git a/core/src/main/scala/spark/deploy/worker/Worker.scala b/core/src/main/scala/spark/deploy/worker/Worker.scala index 175464d40d..645613257d 100644 --- a/core/src/main/scala/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/spark/deploy/worker/Worker.scala @@ -16,7 +16,14 @@ import spark.deploy.RegisterWorkerFailed import akka.actor.Terminated import java.io.File -class Worker(ip: String, port: Int, webUiPort: Int, cores: Int, memory: Int, masterUrl: String) +class Worker( + ip: String, + port: Int, + webUiPort: Int, + cores: Int, + memory: Int, + masterUrl: String, + workDirPath: String = null) extends Actor with Logging { val DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss") // For worker and executor IDs @@ -37,7 +44,11 @@ class Worker(ip: String, port: Int, webUiPort: Int, cores: Int, memory: Int, mas def memoryFree: Int = memory - memoryUsed def createWorkDir() { - workDir = new File(sparkHome, "work") + workDir = if (workDirPath != null) { + new File(workDirPath) + } else { + new File(sparkHome, "work") + } try { if (!workDir.exists() && !workDir.mkdirs()) { logError("Failed to create work directory " + workDir) @@ -164,7 +175,8 @@ object Worker { val args = new WorkerArguments(argStrings) val (actorSystem, boundPort) = AkkaUtils.createActorSystem("spark", args.ip, args.port) val actor = actorSystem.actorOf( - Props(new Worker(args.ip, boundPort, args.webUiPort, args.cores, args.memory, args.master)), + Props(new Worker(args.ip, boundPort, args.webUiPort, args.cores, args.memory, + args.master, args.workDir)), name = "Worker") actorSystem.awaitTermination() } diff --git a/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala b/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala index 1efe8304ea..059c40da9f 100644 --- a/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala +++ b/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala @@ -15,6 +15,7 @@ class WorkerArguments(args: Array[String]) { var cores = inferDefaultCores() var memory = inferDefaultMemory() var master: String = null + var workDir: String = null // Check for settings in environment variables if (System.getenv("SPARK_WORKER_PORT") != null) { @@ -29,6 +30,9 @@ class WorkerArguments(args: Array[String]) { if (System.getenv("SPARK_WORKER_WEBUI_PORT") != null) { webUiPort = System.getenv("SPARK_WORKER_WEBUI_PORT").toInt } + if (System.getenv("SPARK_WORKER_DIR") != null) { + workDir = System.getenv("SPARK_WORKER_DIR") + } parse(args.toList) @@ -49,6 +53,10 @@ class WorkerArguments(args: Array[String]) { memory = value parse(tail) + case ("--work-dir" | "-d") :: value :: tail => + workDir = value + parse(tail) + case "--webui-port" :: IntParam(value) :: tail => webUiPort = value parse(tail) @@ -77,13 +85,14 @@ class WorkerArguments(args: Array[String]) { */ def printUsageAndExit(exitCode: Int) { System.err.println( - "Usage: spark-worker [options] <master>\n" + + "Usage: Worker [options] <master>\n" + "\n" + "Master must be a URL of the form spark://hostname:port\n" + "\n" + "Options:\n" + " -c CORES, --cores CORES Number of cores to use\n" + " -m MEM, --memory MEM Amount of memory to use (e.g. 1000M, 2G)\n" + + " -d DIR, --work-dir DIR Directory to run jobs in (default: SPARK_HOME/work)\n" + " -i IP, --ip IP IP address or DNS name to listen on\n" + " -p PORT, --port PORT Port to listen on (default: random)\n" + " --webui-port PORT Port for web UI (default: 8081)") diff --git a/core/src/main/scala/spark/deploy/worker/WorkerWebUI.scala b/core/src/main/scala/spark/deploy/worker/WorkerWebUI.scala index 58a05e1a38..f84b92b63e 100644 --- a/core/src/main/scala/spark/deploy/worker/WorkerWebUI.scala +++ b/core/src/main/scala/spark/deploy/worker/WorkerWebUI.scala @@ -21,7 +21,7 @@ class WorkerWebUI(val actorSystem: ActorSystem, worker: ActorRef) extends Direct completeWith{ val future = worker ? RequestWorkerState future.map { workerState => - workerui.html.index(workerState.asInstanceOf[WorkerState]) + spark.deploy.worker.html.index(workerState.asInstanceOf[WorkerState]) } } } ~ diff --git a/core/src/main/twirl/common/layout.scala.html b/core/src/main/twirl/spark/deploy/common/layout.scala.html index b9192060aa..b9192060aa 100644 --- a/core/src/main/twirl/common/layout.scala.html +++ b/core/src/main/twirl/spark/deploy/common/layout.scala.html diff --git a/core/src/main/twirl/masterui/executor_row.scala.html b/core/src/main/twirl/spark/deploy/master/executor_row.scala.html index 784d692fc2..784d692fc2 100644 --- a/core/src/main/twirl/masterui/executor_row.scala.html +++ b/core/src/main/twirl/spark/deploy/master/executor_row.scala.html diff --git a/core/src/main/twirl/masterui/executors_table.scala.html b/core/src/main/twirl/spark/deploy/master/executors_table.scala.html index cafc42c80e..cafc42c80e 100644 --- a/core/src/main/twirl/masterui/executors_table.scala.html +++ b/core/src/main/twirl/spark/deploy/master/executors_table.scala.html diff --git a/core/src/main/twirl/masterui/index.scala.html b/core/src/main/twirl/spark/deploy/master/index.scala.html index 31ca8f4132..7562076b00 100644 --- a/core/src/main/twirl/masterui/index.scala.html +++ b/core/src/main/twirl/spark/deploy/master/index.scala.html @@ -1,13 +1,13 @@ @(state: spark.deploy.MasterState) @import spark.deploy.master._ -@common.html.layout(title = "Spark Master on " + state.uri) { +@spark.deploy.common.html.layout(title = "Spark Master on " + state.uri) { <!-- Cluster Details --> <div class="row"> <div class="span12"> <ul class="unstyled"> - <li><strong>URI:</strong> spark://@(state.uri)</li> + <li><strong>URL:</strong> spark://@(state.uri)</li> <li><strong>Number of Workers:</strong> @state.workers.size </li> <li><strong>Cores:</strong> @state.workers.map(_.cores).sum Total, @state.workers.map(_.coresUsed).sum Used</li> <li><strong>Memory:</strong> @state.workers.map(_.memory).sum Total, @state.workers.map(_.memoryUsed).sum Used</li> @@ -47,4 +47,4 @@ </div> </div> -}
\ No newline at end of file +} diff --git a/core/src/main/twirl/masterui/job_details.scala.html b/core/src/main/twirl/spark/deploy/master/job_details.scala.html index 73cefb8269..dcf41c28f2 100644 --- a/core/src/main/twirl/masterui/job_details.scala.html +++ b/core/src/main/twirl/spark/deploy/master/job_details.scala.html @@ -1,6 +1,6 @@ @(job: spark.deploy.master.JobInfo) -@common.html.layout(title = "Job Details") { +@spark.deploy.common.html.layout(title = "Job Details") { <!-- Job Details --> <div class="row"> @@ -37,4 +37,4 @@ </div> </div> -}
\ No newline at end of file +} diff --git a/core/src/main/twirl/masterui/job_row.scala.html b/core/src/main/twirl/spark/deploy/master/job_row.scala.html index 7c4865bb6e..7c4865bb6e 100644 --- a/core/src/main/twirl/masterui/job_row.scala.html +++ b/core/src/main/twirl/spark/deploy/master/job_row.scala.html diff --git a/core/src/main/twirl/masterui/job_table.scala.html b/core/src/main/twirl/spark/deploy/master/job_table.scala.html index 52bad6c4b8..52bad6c4b8 100644 --- a/core/src/main/twirl/masterui/job_table.scala.html +++ b/core/src/main/twirl/spark/deploy/master/job_table.scala.html diff --git a/core/src/main/twirl/masterui/worker_row.scala.html b/core/src/main/twirl/spark/deploy/master/worker_row.scala.html index b21bd9c977..017cc4859e 100644 --- a/core/src/main/twirl/masterui/worker_row.scala.html +++ b/core/src/main/twirl/spark/deploy/master/worker_row.scala.html @@ -4,7 +4,8 @@ <td> <a href="http://@worker.host:@worker.webUiPort">@worker.id</href> </td> - <td>@worker.host:@worker.port</td> + <td>@{worker.host}:@{worker.port}</td> <td>@worker.cores (@worker.coresUsed Used)</td> - <td>@worker.memory (@worker.memoryUsed Used)</td> -</tr>
\ No newline at end of file + <td>@{spark.Utils.memoryMegabytesToString(worker.memory)} + (@{spark.Utils.memoryMegabytesToString(worker.memoryUsed)} Used)</td> +</tr> diff --git a/core/src/main/twirl/masterui/worker_table.scala.html b/core/src/main/twirl/spark/deploy/master/worker_table.scala.html index 2028842297..2028842297 100644 --- a/core/src/main/twirl/masterui/worker_table.scala.html +++ b/core/src/main/twirl/spark/deploy/master/worker_table.scala.html diff --git a/core/src/main/twirl/workerui/executor_row.scala.html b/core/src/main/twirl/spark/deploy/worker/executor_row.scala.html index c3842dbf85..c3842dbf85 100644 --- a/core/src/main/twirl/workerui/executor_row.scala.html +++ b/core/src/main/twirl/spark/deploy/worker/executor_row.scala.html diff --git a/core/src/main/twirl/workerui/executors_table.scala.html b/core/src/main/twirl/spark/deploy/worker/executors_table.scala.html index 327a2399c7..327a2399c7 100644 --- a/core/src/main/twirl/workerui/executors_table.scala.html +++ b/core/src/main/twirl/spark/deploy/worker/executors_table.scala.html diff --git a/core/src/main/twirl/workerui/index.scala.html b/core/src/main/twirl/spark/deploy/worker/index.scala.html index edd82e02f2..69746ed02c 100644 --- a/core/src/main/twirl/workerui/index.scala.html +++ b/core/src/main/twirl/spark/deploy/worker/index.scala.html @@ -1,6 +1,6 @@ @(worker: spark.deploy.WorkerState) -@common.html.layout(title = "Spark Worker on " + worker.uri) { +@spark.deploy.common.html.layout(title = "Spark Worker on " + worker.uri) { <!-- Worker Details --> <div class="row"> @@ -12,7 +12,8 @@ (WebUI at <a href="@worker.masterWebUiUrl">@worker.masterWebUiUrl</a>) </li> <li><strong>Cores:</strong> @worker.cores (@worker.coresUsed Used)</li> - <li><strong>Memory:</strong> @worker.memory (@worker.memoryUsed Used)</li> + <li><strong>Memory:</strong> @{spark.Utils.memoryMegabytesToString(worker.memory)} + (@{spark.Utils.memoryMegabytesToString(worker.memoryUsed)} Used)</li> </ul> </div> </div> @@ -39,4 +40,4 @@ </div> </div> -}
\ No newline at end of file +} |