diff options
author | Patrick Wendell <pwendell@gmail.com> | 2013-07-13 16:05:21 -0700 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2013-07-13 16:14:40 -0700 |
commit | ddb97f0fdf16442afaa9cab656376267e4044510 (patch) | |
tree | b35b99b5f94e00ae0e0e8a6e22d1dbbeeb601829 /core/src/main | |
parent | 37abe84212c6dad6fb87a3b47666d6a3c14c1f66 (diff) | |
download | spark-ddb97f0fdf16442afaa9cab656376267e4044510.tar.gz spark-ddb97f0fdf16442afaa9cab656376267e4044510.tar.bz2 spark-ddb97f0fdf16442afaa9cab656376267e4044510.zip |
Add `Environment` tab to SparkUI.
This adds a tab which displays system property and classpath information. This
can be useful in debugging various types of issues such as:
1. Extra/incorrect Hadoop jars being included in the classpath
2. Spark launching with a different JRE version than intended
3. Spark system properties not being set to intended values
4. User added jars that conflict with Spark jars
Diffstat (limited to 'core/src/main')
-rw-r--r-- | core/src/main/scala/spark/ui/Page.scala | 2 | ||||
-rw-r--r-- | core/src/main/scala/spark/ui/SparkUI.scala | 7 | ||||
-rw-r--r-- | core/src/main/scala/spark/ui/UIUtils.scala | 5 | ||||
-rw-r--r-- | core/src/main/scala/spark/ui/env/EnvironmentUI.scala | 58 |
4 files changed, 67 insertions, 5 deletions
diff --git a/core/src/main/scala/spark/ui/Page.scala b/core/src/main/scala/spark/ui/Page.scala index c853b44b76..ed8f91842c 100644 --- a/core/src/main/scala/spark/ui/Page.scala +++ b/core/src/main/scala/spark/ui/Page.scala @@ -1,3 +1,3 @@ package spark.ui -private[spark] object Page extends Enumeration { val Storage, Jobs = Value }
\ No newline at end of file +private[spark] object Page extends Enumeration { val Storage, Jobs, Environment = Value }
\ No newline at end of file diff --git a/core/src/main/scala/spark/ui/SparkUI.scala b/core/src/main/scala/spark/ui/SparkUI.scala index b3bdc2c490..874e5ba8ec 100644 --- a/core/src/main/scala/spark/ui/SparkUI.scala +++ b/core/src/main/scala/spark/ui/SparkUI.scala @@ -5,15 +5,13 @@ import javax.servlet.http.HttpServletRequest import org.eclipse.jetty.server.{Handler, Server} import spark.{Logging, SparkContext, Utils} +import spark.ui.env.EnvironmentUI import spark.ui.storage.BlockManagerUI import spark.ui.jobs.JobProgressUI -import spark.ui.UIUtils._ import spark.ui.JettyUtils._ /** Top level user interface for Spark */ private[spark] class SparkUI(sc: SparkContext) extends Logging { - // TODO(pwendell): It would be nice to add a view that prints out environment information - val host = Utils.localHostName() val port = Option(System.getProperty("spark.ui.port")).getOrElse(SparkUI.DEFAULT_PORT).toInt var boundPort: Option[Int] = None @@ -25,7 +23,8 @@ private[spark] class SparkUI(sc: SparkContext) extends Logging { ) val storage = new BlockManagerUI(sc) val jobs = new JobProgressUI(sc) - val allHandlers = storage.getHandlers ++ jobs.getHandlers ++ handlers + val env = new EnvironmentUI(sc) + val allHandlers = storage.getHandlers ++ jobs.getHandlers ++ env.getHandlers ++ handlers /** Bind the HTTP server which backs this web interface */ def bind() { diff --git a/core/src/main/scala/spark/ui/UIUtils.scala b/core/src/main/scala/spark/ui/UIUtils.scala index 7b79290d1b..36d9c47245 100644 --- a/core/src/main/scala/spark/ui/UIUtils.scala +++ b/core/src/main/scala/spark/ui/UIUtils.scala @@ -19,6 +19,10 @@ private[spark] object UIUtils { case Jobs => <li class="active"><a href="/stages">Jobs</a></li> case _ => <li><a href="/stages">Jobs</a></li> } + val environment = page match { + case Environment => <li class="active"><a href="/environment">Environment</a></li> + case _ => <li><a href="/environment">Environment</a></li> + } <html> <head> @@ -44,6 +48,7 @@ private[spark] object UIUtils { <ul class="nav"> {storage} {jobs} + {environment} </ul> <ul id="infolist"> <li>Application: <strong>{sc.appName}</strong></li> diff --git a/core/src/main/scala/spark/ui/env/EnvironmentUI.scala b/core/src/main/scala/spark/ui/env/EnvironmentUI.scala new file mode 100644 index 0000000000..8a3b330405 --- /dev/null +++ b/core/src/main/scala/spark/ui/env/EnvironmentUI.scala @@ -0,0 +1,58 @@ +package spark.ui.env + +import javax.servlet.http.HttpServletRequest + +import org.eclipse.jetty.server.Handler + +import scala.collection.JavaConversions._ + +import spark.ui.JettyUtils._ +import spark.ui.UIUtils.headerSparkPage +import spark.ui.Page.Environment +import spark.SparkContext +import spark.ui.UIUtils + +import scala.xml.Node + +private[spark] class EnvironmentUI(sc: SparkContext) { + + def getHandlers = Seq[(String, Handler)]( + ("/environment", (request: HttpServletRequest) => envDetails(request)) + ) + + def envDetails(request: HttpServletRequest): Seq[Node] = { + val properties = System.getProperties.iterator.toSeq + + val classPathProperty = properties + .filter{case (k, v) => k.contains("java.class.path")} + .headOption + .getOrElse("", "") + val sparkProperties = properties.filter(_._1.contains("spark")) + val otherProperties = properties.diff(sparkProperties :+ classPathProperty) + + val propertyHeaders = Seq("Name", "Value") + def propertyRow(kv: (String, String)) = <tr><td>{kv._1}</td><td>{kv._2}</td></tr> + val propertyTable = UIUtils.listingTable( + propertyHeaders, propertyRow, sparkProperties ++ otherProperties) + + val classPathEntries = classPathProperty._2 + .split(System.getProperty("path.separator", ":")) + .filterNot(e => e.isEmpty) + .map(e => (e, "System Classpath")) + val addedJars = sc.addedJars.iterator.toSeq.map{case (path, time) => (path, "Added By User")} + val addedFiles = sc.addedFiles.iterator.toSeq.map{case (path, time) => (path, "Added By User")} + val classPath = addedJars ++ addedFiles ++ classPathEntries + + val classPathHeaders = Seq("Resource", "Source") + def classPathRow(data: (String, String)) = <tr><td>{data._1}</td><td>{data._2}</td></tr> + val classPathTable = UIUtils.listingTable(classPathHeaders, classPathRow, classPath) + + val content = + <span> + <h2>System Properties</h2> {propertyTable} + <h2>Classpath Entries</h2> {classPathTable} + </span> + + headerSparkPage(content, sc, "Environment", Environment) + } +} |