diff options
| author | xutingjun <xutingjun@huawei.com> | 2015-06-30 13:56:59 -0700 |
|---|---|---|
| committer | Andrew Or <andrew@databricks.com> | 2015-06-30 13:57:27 -0700 |
| commit | 79f0b371a36560a009c1b0943c928adc5a1bdd8f (patch) | |
| tree | 07f6f90e15a224e77cf9953d36e28bd2d1853444 /core | |
| parent | 61d7b533dd50bfac2162b4edcea94724bbd8fcb1 (diff) | |
| download | spark-79f0b371a36560a009c1b0943c928adc5a1bdd8f.tar.gz spark-79f0b371a36560a009c1b0943c928adc5a1bdd8f.tar.bz2 spark-79f0b371a36560a009c1b0943c928adc5a1bdd8f.zip | |
[SPARK-8560] [UI] The Executors page will have negative if having resubmitted tasks
when the ```taskEnd.reason``` is ```Resubmitted```, it shouldn't do statistics. Because this tasks has a ```SUCCESS``` taskEnd before.
Author: xutingjun <xutingjun@huawei.com>
Closes #6950 from XuTingjun/pageError and squashes the following commits:
af35dc3 [xutingjun] When taskEnd is Resubmitted, don't do statistics
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala | 13 |
1 files changed, 10 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala index 39583af143..a88fc4c37d 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala @@ -19,7 +19,7 @@ package org.apache.spark.ui.exec import scala.collection.mutable.HashMap -import org.apache.spark.{ExceptionFailure, SparkContext} +import org.apache.spark.{Resubmitted, ExceptionFailure, SparkContext} import org.apache.spark.annotation.DeveloperApi import org.apache.spark.scheduler._ import org.apache.spark.storage.{StorageStatus, StorageStatusListener} @@ -92,15 +92,22 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener) extends Sp val info = taskEnd.taskInfo if (info != null) { val eid = info.executorId - executorToTasksActive(eid) = executorToTasksActive.getOrElse(eid, 1) - 1 - executorToDuration(eid) = executorToDuration.getOrElse(eid, 0L) + info.duration taskEnd.reason match { + case Resubmitted => + // Note: For resubmitted tasks, we continue to use the metrics that belong to the + // first attempt of this task. This may not be 100% accurate because the first attempt + // could have failed half-way through. The correct fix would be to keep track of the + // metrics added by each attempt, but this is much more complicated. + return case e: ExceptionFailure => executorToTasksFailed(eid) = executorToTasksFailed.getOrElse(eid, 0) + 1 case _ => executorToTasksComplete(eid) = executorToTasksComplete.getOrElse(eid, 0) + 1 } + executorToTasksActive(eid) = executorToTasksActive.getOrElse(eid, 1) - 1 + executorToDuration(eid) = executorToDuration.getOrElse(eid, 0L) + info.duration + // Update shuffle read/write val metrics = taskEnd.taskMetrics if (metrics != null) { |