diff options
| author | Patrick Wendell <pwendell@gmail.com> | 2013-08-12 14:45:19 -0700 |
|---|---|---|
| committer | Patrick Wendell <pwendell@gmail.com> | 2013-08-13 16:25:55 -0700 |
| commit | f0382007dc7268c603db3f0227e2ca01dc5e7b71 (patch) | |
| tree | 911af55f7e0245b96fa0db65a34ba47d1a826a0f /core | |
| parent | 622f83ce1ce522ea0058665cbf43c64a73b44439 (diff) | |
| download | spark-f0382007dc7268c603db3f0227e2ca01dc5e7b71.tar.gz spark-f0382007dc7268c603db3f0227e2ca01dc5e7b71.tar.bz2 spark-f0382007dc7268c603db3f0227e2ca01dc5e7b71.zip | |
Bug fix for display of shuffle read/write metrics.
This fixes an error where empty cells are missing if a given task
has no shuffle read/write.
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/spark/ui/jobs/StagePage.scala | 17 |
1 files changed, 11 insertions, 6 deletions
diff --git a/core/src/main/scala/spark/ui/jobs/StagePage.scala b/core/src/main/scala/spark/ui/jobs/StagePage.scala index 1b071a91e5..884c065dee 100644 --- a/core/src/main/scala/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/spark/ui/jobs/StagePage.scala @@ -87,7 +87,7 @@ private[spark] class StagePage(parent: JobProgressUI) { {if (hasShuffleWrite) Seq("Shuffle Write") else Nil} ++ Seq("Details") - val taskTable = listingTable(taskHeaders, taskRow, tasks) + val taskTable = listingTable(taskHeaders, taskRow(hasShuffleRead, hasShuffleWrite), tasks) // Excludes tasks which failed and have incomplete metrics val validTasks = tasks.filter(t => t._1.status == "SUCCESS" && (t._2.isDefined)) @@ -135,7 +135,8 @@ private[spark] class StagePage(parent: JobProgressUI) { } - def taskRow(taskData: (TaskInfo, Option[TaskMetrics], Option[ExceptionFailure])): Seq[Node] = { + def taskRow(shuffleRead: Boolean, shuffleWrite: Boolean) + (taskData: (TaskInfo, Option[TaskMetrics], Option[ExceptionFailure])): Seq[Node] = { def fmtStackTrace(trace: Seq[StackTraceElement]): Seq[Node] = trace.map(e => <span style="display:block;">{e.toString}</span>) val (info, metrics, exception) = taskData @@ -154,10 +155,14 @@ private[spark] class StagePage(parent: JobProgressUI) { <td>{info.taskLocality}</td> <td>{info.hostPort}</td> <td>{dateFmt.format(new Date(info.launchTime))}</td> - {metrics.flatMap{m => m.shuffleReadMetrics}.map{s => - <td>{Utils.memoryBytesToString(s.remoteBytesRead)}</td>}.getOrElse("")} - {metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => - <td>{Utils.memoryBytesToString(s.shuffleBytesWritten)}</td>}.getOrElse("")} + {if (shuffleRead) { + <td>{metrics.flatMap{m => m.shuffleReadMetrics}.map{s => + Utils.memoryBytesToString(s.remoteBytesRead)}.getOrElse("")}</td> + }} + {if (shuffleWrite) { + <td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => + Utils.memoryBytesToString(s.shuffleBytesWritten)}.getOrElse("")}</td> + }} <td>{exception.map(e => <span> {e.className} ({e.description})<br/> |