diff options
author | Josh Rosen <joshrosen@apache.org> | 2014-09-26 14:47:14 -0700 |
---|---|---|
committer | Josh Rosen <joshrosen@apache.org> | 2014-09-26 14:47:14 -0700 |
commit | f872e4fb80b8429800daa9c44c0cac620c1ff303 (patch) | |
tree | d43c6d4ebb15b4fca42f50e241bfc4ff94bb13d6 /python/pyspark/tests.py | |
parent | 7364fa5a176da69e425bca0e3e137ee73275c78c (diff) | |
download | spark-f872e4fb80b8429800daa9c44c0cac620c1ff303.tar.gz spark-f872e4fb80b8429800daa9c44c0cac620c1ff303.tar.bz2 spark-f872e4fb80b8429800daa9c44c0cac620c1ff303.zip |
Revert "[SPARK-3478] [PySpark] Profile the Python tasks"
This reverts commit 1aa549ba9839565274a12c52fa1075b424f138a6.
Diffstat (limited to 'python/pyspark/tests.py')
-rw-r--r-- | python/pyspark/tests.py | 30 |
1 files changed, 0 insertions, 30 deletions
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index e6002afa9c..d1bb2033b7 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -632,36 +632,6 @@ class TestRDDFunctions(PySparkTestCase): self.assertEquals(result.count(), 3) -class TestProfiler(PySparkTestCase): - - def setUp(self): - self._old_sys_path = list(sys.path) - class_name = self.__class__.__name__ - conf = SparkConf().set("spark.python.profile", "true") - self.sc = SparkContext('local[4]', class_name, batchSize=2, conf=conf) - - def test_profiler(self): - - def heavy_foo(x): - for i in range(1 << 20): - x = 1 - rdd = self.sc.parallelize(range(100)) - rdd.foreach(heavy_foo) - profiles = self.sc._profile_stats - self.assertEqual(1, len(profiles)) - id, acc, _ = profiles[0] - stats = acc.value - self.assertTrue(stats is not None) - width, stat_list = stats.get_print_list([]) - func_names = [func_name for fname, n, func_name in stat_list] - self.assertTrue("heavy_foo" in func_names) - - self.sc.show_profiles() - d = tempfile.gettempdir() - self.sc.dump_profiles(d) - self.assertTrue("rdd_%d.pstats" % id in os.listdir(d)) - - class TestSQL(PySparkTestCase): def setUp(self): |