diff options
author | Zhang, Liye <liye.zhang@intel.com> | 2014-11-03 18:17:32 -0800 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2014-11-03 18:17:32 -0800 |
commit | 97a466eca0a629f17e9662ca2b59eeca99142c54 (patch) | |
tree | b9663459ad1b41fa479fff927d1113279efbf131 /core | |
parent | 15b58a2234ab7ba30c9c0cbb536177a3c725e350 (diff) | |
download | spark-97a466eca0a629f17e9662ca2b59eeca99142c54.tar.gz spark-97a466eca0a629f17e9662ca2b59eeca99142c54.tar.bz2 spark-97a466eca0a629f17e9662ca2b59eeca99142c54.zip |
[SPARK-4168][WebUI] web statges number should show correctly when stages are more than 1000
The number of completed stages and failed stages showed on webUI will always be less than 1000. This is really misleading when there are already thousands of stages completed or failed. The number should be correct even when only partial stages listed on the webUI (stage info will be removed if the number is too large).
Author: Zhang, Liye <liye.zhang@intel.com>
Closes #3035 from liyezhang556520/webStageNum and squashes the following commits:
d9e29fb [Zhang, Liye] add detailed comments for variables
4ea8fd1 [Zhang, Liye] change variable name accroding to comments
f4c404d [Zhang, Liye] [SPARK-4168][WebUI] web statges number should show correctly when stages are more than 1000
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala | 9 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala | 10 |
2 files changed, 15 insertions, 4 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index b520736051..e3223403c1 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -59,6 +59,13 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { val failedStages = ListBuffer[StageInfo]() val stageIdToData = new HashMap[(StageId, StageAttemptId), StageUIData] val stageIdToInfo = new HashMap[StageId, StageInfo] + + // Number of completed and failed stages, may not actually equal to completedStages.size and + // failedStages.size respectively due to completedStage and failedStages only maintain the latest + // part of the stages, the earlier ones will be removed when there are too many stages for + // memory sake. + var numCompletedStages = 0 + var numFailedStages = 0 // Map from pool name to a hash map (map from stage id to StageInfo). val poolToActiveStages = HashMap[String, HashMap[Int, StageInfo]]() @@ -110,9 +117,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging { activeStages.remove(stage.stageId) if (stage.failureReason.isEmpty) { completedStages += stage + numCompletedStages += 1 trimIfNecessary(completedStages) } else { failedStages += stage + numFailedStages += 1 trimIfNecessary(failedStages) } } diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala index 6e718eecdd..83a7898071 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala @@ -34,7 +34,9 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("") listener.synchronized { val activeStages = listener.activeStages.values.toSeq val completedStages = listener.completedStages.reverse.toSeq + val numCompletedStages = listener.numCompletedStages val failedStages = listener.failedStages.reverse.toSeq + val numFailedStages = listener.numFailedStages val now = System.currentTimeMillis val activeStagesTable = @@ -69,11 +71,11 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("") </li> <li> <a href="#completed"><strong>Completed Stages:</strong></a> - {completedStages.size} + {numCompletedStages} </li> <li> <a href="#failed"><strong>Failed Stages:</strong></a> - {failedStages.size} + {numFailedStages} </li> </ul> </div> @@ -86,9 +88,9 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("") }} ++ <h4 id="active">Active Stages ({activeStages.size})</h4> ++ activeStagesTable.toNodeSeq ++ - <h4 id="completed">Completed Stages ({completedStages.size})</h4> ++ + <h4 id="completed">Completed Stages ({numCompletedStages})</h4> ++ completedStagesTable.toNodeSeq ++ - <h4 id ="failed">Failed Stages ({failedStages.size})</h4> ++ + <h4 id ="failed">Failed Stages ({numFailedStages})</h4> ++ failedStagesTable.toNodeSeq UIUtils.headerSparkPage("Spark Stages", content, parent) |