diff options
author | Reynold Xin <rxin@apache.org> | 2014-10-01 00:29:14 -0700 |
---|---|---|
committer | Reynold Xin <rxin@apache.org> | 2014-10-01 00:29:14 -0700 |
commit | eb43043f411b87b7b412ee31e858246bd93fdd04 (patch) | |
tree | eaa82edd4c1049a02cc69c6d40a1501bfe1c4c28 | |
parent | c5414b681868a0a11cc5a94184116e66e8d3e9c0 (diff) | |
download | spark-eb43043f411b87b7b412ee31e858246bd93fdd04.tar.gz spark-eb43043f411b87b7b412ee31e858246bd93fdd04.tar.bz2 spark-eb43043f411b87b7b412ee31e858246bd93fdd04.zip |
[SPARK-3747] TaskResultGetter could incorrectly abort a stage if it cannot get result for a specific task
Author: Reynold Xin <rxin@apache.org>
Closes #2599 from rxin/SPARK-3747 and squashes the following commits:
a74c04d [Reynold Xin] Added a line of comment explaining NonFatal
0e8d44c [Reynold Xin] [SPARK-3747] TaskResultGetter could incorrectly abort a stage if it cannot get result for a specific task
-rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala index df59f444b7..3f345ceeaa 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskResultGetter.scala @@ -19,6 +19,8 @@ package org.apache.spark.scheduler import java.nio.ByteBuffer +import scala.util.control.NonFatal + import org.apache.spark._ import org.apache.spark.TaskState.TaskState import org.apache.spark.serializer.SerializerInstance @@ -32,7 +34,7 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul private val THREADS = sparkEnv.conf.getInt("spark.resultGetter.threads", 4) private val getTaskResultExecutor = Utils.newDaemonFixedThreadPool( - THREADS, "Result resolver thread") + THREADS, "task-result-getter") protected val serializer = new ThreadLocal[SerializerInstance] { override def initialValue(): SerializerInstance = { @@ -70,7 +72,8 @@ private[spark] class TaskResultGetter(sparkEnv: SparkEnv, scheduler: TaskSchedul case cnf: ClassNotFoundException => val loader = Thread.currentThread.getContextClassLoader taskSetManager.abort("ClassNotFound with classloader: " + loader) - case ex: Exception => + // Matching NonFatal so we don't catch the ControlThrowable from the "return" above. + case NonFatal(ex) => logError("Exception while getting task result", ex) taskSetManager.abort("Exception while getting task result: %s".format(ex)) } |