diff options
author | Sundeep Narravula <sundeepn@yahoo-inc.com> | 2013-12-01 00:50:34 -0800 |
---|---|---|
committer | Sundeep Narravula <sundeepn@yahoo-inc.com> | 2013-12-01 00:50:34 -0800 |
commit | be3ea2394fa2e626fb6b5f2cd46e7156016c9b3f (patch) | |
tree | 92b32b70d4ac898fecca9b8d9ad5a688603fa8ee | |
parent | 4d53830eb79174cfd9641f6342727bc980d5c3e0 (diff) | |
download | spark-be3ea2394fa2e626fb6b5f2cd46e7156016c9b3f.tar.gz spark-be3ea2394fa2e626fb6b5f2cd46e7156016c9b3f.tar.bz2 spark-be3ea2394fa2e626fb6b5f2cd46e7156016c9b3f.zip |
Log exception in scheduler in addition to passing it to the caller.
Code Styling changes.
-rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index f6a4482679..915918630b 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -384,12 +384,14 @@ class DAGScheduler( private[scheduler] def processEvent(event: DAGSchedulerEvent): Boolean = { event match { case JobSubmitted(jobId, rdd, func, partitions, allowLocal, callSite, listener, properties) => - var finalStage:Stage = null + var finalStage: Stage = null try { + // New stage creation at times and if its not protected, the scheduler thread is killed. + // e.g. it can fail when jobs are run on HadoopRDD whose underlying hdfs files have been deleted finalStage = newStage(rdd, partitions.size, None, jobId, Some(callSite)) } catch { case e: Exception => - logWarning("Creating new stage failed due to exception - job: " + jobId ) + logWarning("Creating new stage failed due to exception - job: " + jobId, e) listener.jobFailed(e) return false } |