diff options
| author | Carson Wang <carson.wang@intel.com> | 2015-07-20 18:08:59 -0700 |
|---|---|---|
| committer | Kay Ousterhout <kayousterhout@gmail.com> | 2015-07-20 18:08:59 -0700 |
| commit | 66bb8003b949860b8652542e1232bc48665448c2 (patch) | |
| tree | 5c5377be0495c7e79b0a021b4a0c7172bee9298e /core | |
| parent | ff3c72dbafa16c6158fc36619f3c38344c452ba0 (diff) | |
| download | spark-66bb8003b949860b8652542e1232bc48665448c2.tar.gz spark-66bb8003b949860b8652542e1232bc48665448c2.tar.bz2 spark-66bb8003b949860b8652542e1232bc48665448c2.zip | |
[SPARK-9187] [WEBUI] Timeline view may show negative value for running tasks
For running tasks, the executorRunTime metrics is 0 which causes negative executorComputingTime in the timeline. It also causes an incorrect SchedulerDelay time.
Author: Carson Wang <carson.wang@intel.com>
Closes #7526 from carsonwang/timeline-negValue and squashes the following commits:
7b17db2 [Carson Wang] Fix negative value in timeline view
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala | 23 |
1 files changed, 15 insertions, 8 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala index 27b82aaddd..6e077bf3e7 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala @@ -537,20 +537,27 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") { (metricsOpt.flatMap(_.shuffleWriteMetrics .map(_.shuffleWriteTime)).getOrElse(0L) / 1e6).toLong val shuffleWriteTimeProportion = toProportion(shuffleWriteTime) - val executorComputingTime = metricsOpt.map(_.executorRunTime).getOrElse(0L) - - shuffleReadTime - shuffleWriteTime - val executorComputingTimeProportion = toProportion(executorComputingTime) + val serializationTime = metricsOpt.map(_.resultSerializationTime).getOrElse(0L) val serializationTimeProportion = toProportion(serializationTime) val deserializationTime = metricsOpt.map(_.executorDeserializeTime).getOrElse(0L) val deserializationTimeProportion = toProportion(deserializationTime) val gettingResultTime = getGettingResultTime(taskUIData.taskInfo, currentTime) val gettingResultTimeProportion = toProportion(gettingResultTime) - val schedulerDelay = totalExecutionTime - - (executorComputingTime + shuffleReadTime + shuffleWriteTime + - serializationTime + deserializationTime + gettingResultTime) - val schedulerDelayProportion = - (100 - executorComputingTimeProportion - shuffleReadTimeProportion - + val schedulerDelay = + metricsOpt.map(getSchedulerDelay(taskInfo, _, currentTime)).getOrElse(0L) + val schedulerDelayProportion = toProportion(schedulerDelay) + + val executorOverhead = serializationTime + deserializationTime + val executorRunTime = if (taskInfo.running) { + totalExecutionTime - executorOverhead - gettingResultTime + } else { + metricsOpt.map(_.executorRunTime).getOrElse( + totalExecutionTime - executorOverhead - gettingResultTime) + } + val executorComputingTime = executorRunTime - shuffleReadTime - shuffleWriteTime + val executorComputingTimeProportion = + (100 - schedulerDelayProportion - shuffleReadTimeProportion - shuffleWriteTimeProportion - serializationTimeProportion - deserializationTimeProportion - gettingResultTimeProportion) |