diff options
| author | Reynold Xin <rxin@apache.org> | 2014-10-01 00:29:14 -0700 |
|---|---|---|
| committer | Reynold Xin <rxin@apache.org> | 2014-10-01 00:30:18 -0700 |
| commit | a7d2df42bb98fab045be653283b1eae731aeccea (patch) | |
| tree | 3a4817bb9af5ca8c6eab50062ae7039389fce7b7 /core | |
| parent | 06b96d4a3d6af64ade3e76d8cbee783e8a6e3fad (diff) | |
| download | spark-a7d2df42bb98fab045be653283b1eae731aeccea.tar.gz spark-a7d2df42bb98fab045be653283b1eae731aeccea.tar.bz2 spark-a7d2df42bb98fab045be653283b1eae731aeccea.zip | |
[SPARK-3747] TaskResultGetter could incorrectly abort a stage if it cannot get result for a specific task
Author: Reynold Xin <rxin@apache.org>
Closes #2599 from rxin/SPARK-3747 and squashes the following commits:
a74c04d [Reynold Xin] Added a line of comment explaining NonFatal
0e8d44c [Reynold Xin] [SPARK-3747] TaskResultGetter could incorrectly abort a stage if it cannot get result for a specific task
(cherry picked from commit eb43043f411b87b7b412ee31e858246bd93fdd04)
Signed-off-by: Reynold Xin <rxin@apache.org>
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala index df59f444b7..3f345ceeaa 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala @@ -19,6 +19,8 @@ package org.apache.spark.scheduler import java.nio.ByteBuffer +import scala.util.control.NonFatal + import org.apache.spark._ import org.apache.spark.TaskState.TaskState import org.apache.spark.serializer.SerializerInstance @@ -32,7 +34,7 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul private val THREADS = sparkEnv.conf.getInt("spark.resultGetter.threads", 4) private val getTaskResultExecutor = Utils.newDaemonFixedThreadPool( - THREADS, "Result resolver thread") + THREADS, "task-result-getter") protected val serializer = new ThreadLocal[SerializerInstance] { override def initialValue(): SerializerInstance = { @@ -70,7 +72,8 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul case cnf: ClassNotFoundException => val loader = Thread.currentThread.getContextClassLoader taskSetManager.abort("ClassNotFound with classloader: " + loader) - case ex: Exception => + // Matching NonFatal so we don't catch the ControlThrowable from the "return" above. + case NonFatal(ex) => logError("Exception while getting task result", ex) taskSetManager.abort("Exception while getting task result: %s".format(ex)) } |