diff options
| author | Josh Rosen <joshrosen@apache.org> | 2014-09-26 14:47:14 -0700 |
|---|---|---|
| committer | Josh Rosen <joshrosen@apache.org> | 2014-09-26 14:47:14 -0700 |
| commit | f872e4fb80b8429800daa9c44c0cac620c1ff303 (patch) | |
| tree | d43c6d4ebb15b4fca42f50e241bfc4ff94bb13d6 /python/pyspark/rdd.py | |
| parent | 7364fa5a176da69e425bca0e3e137ee73275c78c (diff) | |
| download | spark-f872e4fb80b8429800daa9c44c0cac620c1ff303.tar.gz spark-f872e4fb80b8429800daa9c44c0cac620c1ff303.tar.bz2 spark-f872e4fb80b8429800daa9c44c0cac620c1ff303.zip | |
Revert "[SPARK-3478] [PySpark] Profile the Python tasks"
This reverts commit 1aa549ba9839565274a12c52fa1075b424f138a6.
Diffstat (limited to 'python/pyspark/rdd.py')
| -rw-r--r-- | python/pyspark/rdd.py | 10 |
1 files changed, 2 insertions, 8 deletions
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py index 8ed89e2f97..680140d72d 100644 --- a/python/pyspark/rdd.py +++ b/python/pyspark/rdd.py @@ -15,6 +15,7 @@ # limitations under the License. # +from base64 import standard_b64encode as b64enc import copy from collections import defaultdict from itertools import chain, ifilter, imap @@ -31,7 +32,6 @@ import bisect from random import Random from math import sqrt, log, isinf, isnan -from pyspark.accumulators import PStatsParam from pyspark.serializers import NoOpSerializer, CartesianDeserializer, \ BatchedSerializer, CloudPickleSerializer, PairDeserializer, \ PickleSerializer, pack_long, AutoBatchedSerializer @@ -2080,9 +2080,7 @@ class PipelinedRDD(RDD): return self._jrdd_val if self._bypass_serializer: self._jrdd_deserializer = NoOpSerializer() - enable_profile = self.ctx._conf.get("spark.python.profile", "false") == "true" - profileStats = self.ctx.accumulator(None, PStatsParam) if enable_profile else None - command = (self.func, profileStats, self._prev_jrdd_deserializer, + command = (self.func, self._prev_jrdd_deserializer, self._jrdd_deserializer) # the serialized command will be compressed by broadcast ser = CloudPickleSerializer() @@ -2104,10 +2102,6 @@ class PipelinedRDD(RDD): self.ctx.pythonExec, broadcast_vars, self.ctx._javaAccumulator) self._jrdd_val = python_rdd.asJavaRDD() - - if enable_profile: - self._id = self._jrdd_val.id() - self.ctx._add_profile(self._id, profileStats) return self._jrdd_val def id(self): |