diff options
author | Derek Dagit <derekd@yahoo-inc.com> | 2015-11-18 15:56:54 -0800 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2015-11-18 15:56:54 -0800 |
commit | c07a50b86254578625be777b1890ff95e832ac6e (patch) | |
tree | 4410eedd8e357cf6b225fb95b918eec428695237 /core/src | |
parent | 3a9851936ddfe5bcb6a7f364d535fac977551f5d (diff) | |
download | spark-c07a50b86254578625be777b1890ff95e832ac6e.tar.gz spark-c07a50b86254578625be777b1890ff95e832ac6e.tar.bz2 spark-c07a50b86254578625be777b1890ff95e832ac6e.zip |
[SPARK-10930] History "Stages" page "duration" can be confusing
Author: Derek Dagit <derekd@yahoo-inc.com>
Closes #9051 from d2r/spark-10930-ui-max-task-dur.
Diffstat (limited to 'core/src')
-rw-r--r-- | core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala | 19 |
1 files changed, 16 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala index ea806d09b6..2a1c3c1a50 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala @@ -145,9 +145,22 @@ private[ui] class StageTableBase( case None => "Unknown" } val finishTime = s.completionTime.getOrElse(System.currentTimeMillis) - val duration = s.submissionTime.map { t => - if (finishTime > t) finishTime - t else System.currentTimeMillis - t - } + + // The submission time for a stage is misleading because it counts the time + // the stage waits to be launched. (SPARK-10930) + val taskLaunchTimes = + stageData.taskData.values.map(_.taskInfo.launchTime).filter(_ > 0) + val duration: Option[Long] = + if (taskLaunchTimes.nonEmpty) { + val startTime = taskLaunchTimes.min + if (finishTime > startTime) { + Some(finishTime - startTime) + } else { + Some(System.currentTimeMillis() - startTime) + } + } else { + None + } val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown") val inputRead = stageData.inputBytes |