Merge pull request #504 from JoshRosen/SPARK-1025

Fix PySpark hang when input files are deleted (SPARK-1025)

This pull request addresses [SPARK-1025](https://spark-project.atlassian.net/browse/SPARK-1025), an issue where PySpark could hang if its input files were deleted.

1 files changed, 11 insertions, 0 deletions

diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
index acd1ca5676..527104587f 100644
--- a/python/pyspark/tests.py
+++ b/python/pyspark/tests.py
@@ -168,6 +168,17 @@ class TestRDDFunctions(PySparkTestCase):
         self.assertEqual("Hello World!", x.strip())
         self.assertEqual("Hello World!", y.strip())
 
+    def test_deleting_input_files(self):
+        # Regression test for SPARK-1025
+        tempFile = NamedTemporaryFile(delete=False)
+        tempFile.write("Hello World!")
+        tempFile.close()
+        data = self.sc.textFile(tempFile.name)
+        filtered_data = data.filter(lambda x: True)
+        self.assertEqual(1, filtered_data.count())
+        os.unlink(tempFile.name)
+        self.assertRaises(Exception, lambda: filtered_data.count())
+
 
 class TestIO(PySparkTestCase):