diff options
author | Alexander Shorin <kxepal@apache.org> | 2016-11-28 18:28:24 -0800 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-11-28 18:28:24 -0800 |
commit | 71352c94ad2a60d1695bd7ac0f4452539270e10c (patch) | |
tree | cad04f573688b2339271c328f4ebf2a87910a40e /python/pyspark/context.py | |
parent | 2e809903d459b5b5aa6fd882b5c4a0c915af4d43 (diff) | |
download | spark-71352c94ad2a60d1695bd7ac0f4452539270e10c.tar.gz spark-71352c94ad2a60d1695bd7ac0f4452539270e10c.tar.bz2 spark-71352c94ad2a60d1695bd7ac0f4452539270e10c.zip |
[SPARK-18523][PYSPARK] Make SparkContext.stop more reliable
## What changes were proposed in this pull request?
This PR fixes SparkContext broken state in which it may fall if spark driver get crashed or killed by OOM.
## How was this patch tested?
1. Start SparkContext;
2. Find Spark driver process and `kill -9` it;
3. Call `sc.stop()`;
4. Create new SparkContext after that;
Without this patch you will crash on step 3 and won't be able to do step 4 without manual reset private attibutes or IPython notebook / shell restart.
Author: Alexander Shorin <kxepal@apache.org>
Closes #15961 from kxepal/18523-make-spark-context-stop-more-reliable.
Diffstat (limited to 'python/pyspark/context.py')
-rw-r--r-- | python/pyspark/context.py | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/python/pyspark/context.py b/python/pyspark/context.py index 2fd3aee01d..5c4e79cb04 100644 --- a/python/pyspark/context.py +++ b/python/pyspark/context.py @@ -26,6 +26,8 @@ import warnings from threading import RLock from tempfile import NamedTemporaryFile +from py4j.protocol import Py4JError + from pyspark import accumulators from pyspark.accumulators import Accumulator from pyspark.broadcast import Broadcast @@ -373,8 +375,19 @@ class SparkContext(object): Shut down the SparkContext. """ if getattr(self, "_jsc", None): - self._jsc.stop() - self._jsc = None + try: + self._jsc.stop() + except Py4JError: + # Case: SPARK-18523 + warnings.warn( + 'Unable to cleanly shutdown Spark JVM process.' + ' It is possible that the process has crashed,' + ' been killed or may also be in a zombie state.', + RuntimeWarning + ) + pass + finally: + self._jsc = None if getattr(self, "_accumulatorServer", None): self._accumulatorServer.shutdown() self._accumulatorServer = None |