diff options
| author | Tor Myklebust <tmyklebu@csclub.uwaterloo.ca> | 2013-12-16 13:27:03 -0500 |
|---|---|---|
| committer | Tor Myklebust <tmyklebu@csclub.uwaterloo.ca> | 2013-12-16 13:27:03 -0500 |
| commit | 882d544856c61573cdd6124e921f700d580d170d (patch) | |
| tree | 84f63e24a17c00514d8cd4355c61eb0fc53762e4 | |
| parent | 8a397a959bf0b68f7d10fa57665225e0c2b5d03a (diff) | |
| download | spark-882d544856c61573cdd6124e921f700d580d170d.tar.gz spark-882d544856c61573cdd6124e921f700d580d170d.tar.bz2 spark-882d544856c61573cdd6124e921f700d580d170d.zip | |
UI to display serialisation time of a stage.
| -rw-r--r-- | core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala | 6 |
1 files changed, 6 insertions, 0 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 69f9446bab..81651bdd20 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -101,6 +101,11 @@ private[spark] class StagePage(parent: JobProgressUI) { None } else { + val serializationTimes = validTasks.map{case (info, metrics, exception) => + metrics.get.serializationTime.toDouble} + val serializationQuantiles = "Serialization Time" +: Distribution(serializationTimes).get.getQuantiles().map( + ms => parent.formatDuration(ms.toLong)) + val serviceTimes = validTasks.map{case (info, metrics, exception) => metrics.get.executorRunTime.toDouble} val serviceQuantiles = "Duration" +: Distribution(serviceTimes).get.getQuantiles().map( @@ -149,6 +154,7 @@ private[spark] class StagePage(parent: JobProgressUI) { val shuffleWriteQuantiles = "Shuffle Write" +: getQuantileCols(shuffleWriteSizes) val listings: Seq[Seq[String]] = Seq( + serializationQuantiles, serviceQuantiles, gettingResultQuantiles, schedulerDelayQuantiles, |