SPARK-1399: show stage failure reason in UI

https://issues.apache.org/jira/browse/SPARK-1399

refactor StageTable a bit to support additional column for failed stage

Author: CodingCat <zhunansjtu@gmail.com>
Author: Nan Zhu <CodingCat@users.noreply.github.com>

Closes #421 from CodingCat/SPARK-1399 and squashes the following commits:

2caba36 [CodingCat] remove dummy tag
77cf305 [CodingCat] create dummy element to wrap columns
3989ce2 [CodingCat] address Aaron's comments
18fc09f [Nan Zhu] fix compile error
00ea30a [Nan Zhu] address Kay's comments
16ac83d [CodingCat] set a default value of failureReason
35df3df [CodingCat] address andrew's comments
06d21a4 [CodingCat] address andrew's comments
25a6db6 [CodingCat] style fix
dc8856d [CodingCat] show stage failure reason in UI

3 files changed, 61 insertions, 39 deletions

diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala
index 34ff2ac34a..0da6289211 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala
@@ -41,10 +41,12 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("")
       val now = System.currentTimeMillis
 
       val activeStagesTable =
-        new StageTable(activeStages.sortBy(_.submissionTime).reverse, parent, parent.killEnabled)
+        new StageTableBase(activeStages.sortBy(_.submissionTime).reverse,
+          parent, parent.killEnabled)
       val completedStagesTable =
-        new StageTable(completedStages.sortBy(_.submissionTime).reverse, parent)
-      val failedStagesTable = new StageTable(failedStages.sortBy(_.submissionTime).reverse, parent)
+        new StageTableBase(completedStages.sortBy(_.submissionTime).reverse, parent)
+      val failedStagesTable =
+        new FailedStageTable(failedStages.sortBy(_.submissionTime).reverse, parent)
 
       // For now, pool information is only accessible in live UIs
       val pools = if (live) sc.getAllPools else Seq[Schedulable]()
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala
index fd83d37583..0a2bf31833 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala
@@ -40,7 +40,8 @@ private[ui] class PoolPage(parent: JobProgressTab) extends WebUIPage("pool") {
         case Some(s) => s.values.toSeq
         case None => Seq[StageInfo]()
       }
-      val activeStagesTable = new StageTable(activeStages.sortBy(_.submissionTime).reverse, parent)
+      val activeStagesTable =
+        new StageTableBase(activeStages.sortBy(_.submissionTime).reverse, parent)
 
       // For now, pool information is only accessible in live UIs
       val pools = if (live) Seq(sc.getPoolForName(poolName).get) else Seq[Schedulable]()
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
index 8c5b1f55fd..2eb8c7a3a3 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@ -27,34 +27,37 @@ import org.apache.spark.ui.UIUtils
 import org.apache.spark.util.Utils
 
 /** Page showing list of all ongoing and recently finished stages */
-private[ui] class StageTable(
+private[ui] class StageTableBase(
     stages: Seq[StageInfo],
     parent: JobProgressTab,
     killEnabled: Boolean = false) {
 
   private val basePath = parent.basePath
   private val listener = parent.listener
-  private lazy val isFairScheduler = parent.isFairScheduler
+  protected def isFairScheduler = parent.isFairScheduler
+
+  protected def columns: Seq[Node] = {
+    // create dummy element to wrap the columns
+    <th>Stage Id</th> ++
+    {if (isFairScheduler) {<th>Pool Name</th>} else Seq.empty} ++
+    <th>Description</th>
+    <th>Submitted</th>
+    <th>Duration</th>
+    <th>Tasks: Succeeded/Total</th>
+    <th>Shuffle Read</th>
+    <th>Shuffle Write</th>
+  }
 
   def toNodeSeq: Seq[Node] = {
     listener.synchronized {
-      stageTable(stageRow, stages)
+      stageTable(renderStageRow, stages)
     }
   }
 
   /** Special table that merges two header cells. */
-  private def stageTable[T](makeRow: T => Seq[Node], rows: Seq[T]): Seq[Node] = {
+  protected def stageTable[T](makeRow: T => Seq[Node], rows: Seq[T]): Seq[Node] = {
     <table class="table table-bordered table-striped table-condensed sortable">
-      <thead>
-        <th>Stage Id</th>
-        {if (isFairScheduler) {<th>Pool Name</th>} else {}}
-        <th>Description</th>
-        <th>Submitted</th>
-        <th>Duration</th>
-        <th>Tasks: Succeeded/Total</th>
-        <th>Shuffle Read</th>
-        <th>Shuffle Write</th>
-      </thead>
+      <thead>{columns}</thead>
       <tbody>
         {rows.map(r => makeRow(r))}
       </tbody>
@@ -94,8 +97,7 @@ private[ui] class StageTable(
       .getOrElse(<div> {killLink}{nameLink}</div>)
   }
 
-  /** Render an HTML row that represents a stage */
-  private def stageRow(s: StageInfo): Seq[Node] = {
+  protected def stageRow(s: StageInfo): Seq[Node] = {
     val poolName = listener.stageIdToPool.get(s.stageId)
     val submissionTime = s.submissionTime match {
       case Some(t) => UIUtils.formatDate(new Date(t))
@@ -124,25 +126,42 @@ private[ui] class StageTable(
       case 0 => ""
       case b => Utils.bytesToString(b)
     }
-
-    <tr>
-      <td>{s.stageId}</td>
-      {if (isFairScheduler) {
-        <td>
-          <a href={"%s/stages/pool?poolname=%s"
-            .format(UIUtils.prependBaseUri(basePath), poolName.get)}>
-            {poolName.get}
-          </a>
-        </td>
-      }}
-      <td>{makeDescription(s)}</td>
-      <td valign="middle">{submissionTime}</td>
-      <td sorttable_customkey={duration.getOrElse(-1).toString}>{formattedDuration}</td>
-      <td class="progress-cell">
-        {makeProgressBar(startedTasks, completedTasks, failedTasks, totalTasks)}
+    <td>{s.stageId}</td> ++
+    {if (isFairScheduler) {
+      <td>
+        <a href={"%s/stages/pool?poolname=%s"
+          .format(UIUtils.prependBaseUri(basePath), poolName.get)}>
+          {poolName.get}
+        </a>
       </td>
-      <td sorttable_customekey={shuffleReadSortable.toString}>{shuffleRead}</td>
-      <td sorttable_customekey={shuffleWriteSortable.toString}>{shuffleWrite}</td>
-    </tr>
+    } else {
+      Seq.empty
+    }} ++
+    <td>{makeDescription(s)}</td>
+    <td valign="middle">{submissionTime}</td>
+    <td sorttable_customkey={duration.getOrElse(-1).toString}>{formattedDuration}</td>
+    <td class="progress-cell">
+      {makeProgressBar(startedTasks, completedTasks, failedTasks, totalTasks)}
+    </td>
+    <td sorttable_customekey={shuffleReadSortable.toString}>{shuffleRead}</td>
+    <td sorttable_customekey={shuffleWriteSortable.toString}>{shuffleWrite}</td>
+  }
+
+  /** Render an HTML row that represents a stage */
+  private def renderStageRow(s: StageInfo): Seq[Node] = <tr>{stageRow(s)}</tr>
+}
+
+private[ui] class FailedStageTable(
+    stages: Seq[StageInfo],
+    parent: JobProgressTab,
+    killEnabled: Boolean = false)
+  extends StageTableBase(stages, parent, killEnabled) {
+
+  override protected def columns: Seq[Node] = super.columns ++ <th>Failure Reason</th>
+
+  override protected def stageRow(s: StageInfo): Seq[Node] = {
+    val basicColumns = super.stageRow(s)
+    val failureReason = <td valign="middle">{s.failureReason.getOrElse("")}</td>
+    basicColumns ++ failureReason
   }
 }