aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndrew Or <andrew@databricks.com>2015-02-09 21:18:48 -0800
committerAndrew Or <andrew@databricks.com>2015-02-09 21:18:48 -0800
commita95ed52157473fb0e42e910ee15270e7f0edf943 (patch)
tree6a1d11f6de1a780d48bf207a3cd4333f8d3a8acc
parent20a6013106b56a1a1cc3e8cda092330ffbe77cc3 (diff)
downloadspark-a95ed52157473fb0e42e910ee15270e7f0edf943.tar.gz
spark-a95ed52157473fb0e42e910ee15270e7f0edf943.tar.bz2
spark-a95ed52157473fb0e42e910ee15270e7f0edf943.zip
[SPARK-5703] AllJobsPage throws empty.max exception
If you have a `SparkListenerJobEnd` event without the corresponding `SparkListenerJobStart` event, then `JobProgressListener` will create an empty `JobUIData` with an empty `stageIds` list. However, later in `AllJobsPage` we call `stageIds.max`. If this is empty, it will throw an exception. This crashed my history server. Author: Andrew Or <andrew@databricks.com> Closes #4490 from andrewor14/jobs-page-max and squashes the following commits: 21797d3 [Andrew Or] Check nonEmpty before calling max
-rw-r--r--core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala4
1 files changed, 3 insertions, 1 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
index 045c69da06..bd923d78a8 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
@@ -42,7 +42,9 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
}
def makeRow(job: JobUIData): Seq[Node] = {
- val lastStageInfo = listener.stageIdToInfo.get(job.stageIds.max)
+ val lastStageInfo = Option(job.stageIds)
+ .filter(_.nonEmpty)
+ .flatMap { ids => listener.stageIdToInfo.get(ids.max) }
val lastStageData = lastStageInfo.flatMap { s =>
listener.stageIdToData.get((s.stageId, s.attemptId))
}