From f83068497ba42c5ea5c636efebca81f684e96177 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 23 Jan 2014 18:10:16 -0800 Subject: Fix for SPARK-1025: PySpark hang on missing files. --- python/pyspark/tests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'python/pyspark/tests.py') diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index acd1ca5676..527104587f 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -168,6 +168,17 @@ class TestRDDFunctions(PySparkTestCase): self.assertEqual("Hello World!", x.strip()) self.assertEqual("Hello World!", y.strip()) + def test_deleting_input_files(self): + # Regression test for SPARK-1025 + tempFile = NamedTemporaryFile(delete=False) + tempFile.write("Hello World!") + tempFile.close() + data = self.sc.textFile(tempFile.name) + filtered_data = data.filter(lambda x: True) + self.assertEqual(1, filtered_data.count()) + os.unlink(tempFile.name) + self.assertRaises(Exception, lambda: filtered_data.count()) + class TestIO(PySparkTestCase): -- cgit v1.2.3