diff options
| author | Holden Karau <holden@us.ibm.com> | 2016-04-10 02:34:54 +0100 |
|---|---|---|
| committer | Sean Owen <sowen@cloudera.com> | 2016-04-10 02:34:54 +0100 |
| commit | 00288ea2a463180e91fd16c8e2b627e69566e1f0 (patch) | |
| tree | 3826c842acc64ee83225cff9a467c9583d6716f1 /python/pyspark/tests.py | |
| parent | 5989c85b535f7f623392d6456d8b37052487f24b (diff) | |
| download | spark-00288ea2a463180e91fd16c8e2b627e69566e1f0.tar.gz spark-00288ea2a463180e91fd16c8e2b627e69566e1f0.tar.bz2 spark-00288ea2a463180e91fd16c8e2b627e69566e1f0.zip | |
[SPARK-13687][PYTHON] Cleanup PySpark parallelize temporary files
## What changes were proposed in this pull request?
Eagerly cleanup PySpark's temporary parallelize cleanup files rather than waiting for shut down.
## How was this patch tested?
Unit tests
Author: Holden Karau <holden@us.ibm.com>
Closes #12233 from holdenk/SPARK-13687-cleanup-pyspark-temporary-files.
Diffstat (limited to 'python/pyspark/tests.py')
| -rw-r--r-- | python/pyspark/tests.py | 7 |
1 files changed, 7 insertions, 0 deletions
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 15c87e22f9..97ea39dde0 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -1914,6 +1914,13 @@ class ContextTests(unittest.TestCase): with SparkContext.getOrCreate() as sc: self.assertTrue(SparkContext.getOrCreate() is sc) + def test_parallelize_eager_cleanup(self): + with SparkContext() as sc: + temp_files = os.listdir(sc._temp_dir) + rdd = sc.parallelize([0, 1, 2]) + post_parallalize_temp_files = os.listdir(sc._temp_dir) + self.assertEqual(temp_files, post_parallalize_temp_files) + def test_stop(self): sc = SparkContext() self.assertNotEqual(SparkContext._active_spark_context, None) |