diff options
author | Charles Reiss <charles@eecs.berkeley.edu> | 2013-02-19 11:52:35 -0800 |
---|---|---|
committer | Charles Reiss <charles@eecs.berkeley.edu> | 2013-02-19 13:03:02 -0800 |
commit | 687581c3ec2b6b8310bd5be9f2d15b25b9051aac (patch) | |
tree | 127eaa2c705b276ec1886ddfd66134522b659d74 | |
parent | 03d847999e8c54684128573b94973544026081b2 (diff) | |
download | spark-687581c3ec2b6b8310bd5be9f2d15b25b9051aac.tar.gz spark-687581c3ec2b6b8310bd5be9f2d15b25b9051aac.tar.bz2 spark-687581c3ec2b6b8310bd5be9f2d15b25b9051aac.zip |
Paranoid uncaught exception handling for exceptions during shutdown
-rw-r--r-- | core/src/main/scala/spark/executor/Executor.scala | 29 |
1 files changed, 23 insertions, 6 deletions
diff --git a/core/src/main/scala/spark/executor/Executor.scala b/core/src/main/scala/spark/executor/Executor.scala index bd21ba719a..b63bec11ad 100644 --- a/core/src/main/scala/spark/executor/Executor.scala +++ b/core/src/main/scala/spark/executor/Executor.scala @@ -50,14 +50,31 @@ private[spark] class Executor extends Logging { override def uncaughtException(thread: Thread, exception: Throwable) { try { logError("Uncaught exception in thread " + thread, exception) - if (exception.isInstanceOf[OutOfMemoryError]) { - System.exit(ExecutorExitCode.OOM) - } else { - System.exit(ExecutorExitCode.UNCAUGHT_EXCEPTION) + + // We may have been called from a shutdown hook. If so, we must not call System.exit(). + // (If we do, we will deadlock.) Runtime#addShutdownHook should fail if we are shutting + // down, which would either occur if we were called from a shutdown hook or if + // a System.exit() occured concurrently. + var shuttingDown = false + try { + val hook = new Thread { + override def run() {} + } + Runtime.getRuntime.addShutdownHook(hook) + Runtime.getRuntime.removeShutdownHook(hook) + } catch { + case ise: IllegalStateException => shuttingDown = true + } + if (!shuttingDown) { + if (exception.isInstanceOf[OutOfMemoryError]) { + System.exit(ExecutorExitCode.OOM) + } else { + System.exit(ExecutorExitCode.UNCAUGHT_EXCEPTION) + } } } catch { - case oom: OutOfMemoryError => System.exit(ExecutorExitCode.OOM) - case t: Throwable => System.exit(ExecutorExitCode.UNCAUGHT_EXCEPTION_TWICE) + case oom: OutOfMemoryError => Runtime.getRuntime.halt(ExecutorExitCode.OOM) + case t: Throwable => Runtime.getRuntime.halt(ExecutorExitCode.UNCAUGHT_EXCEPTION_TWICE) } } } |