diff options
author | Jason White <jason.white@shopify.com> | 2016-10-03 14:12:03 -0700 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2016-10-03 14:12:03 -0700 |
commit | 1f31bdaef670dd43999613deae3620f4ddcd1fbf (patch) | |
tree | 0814b539f497beddf89f2421e90aa93d0bf83c6b /python/pyspark/rdd.py | |
parent | 1dd68d3827133d203e85294405400b04904879e0 (diff) | |
download | spark-1f31bdaef670dd43999613deae3620f4ddcd1fbf.tar.gz spark-1f31bdaef670dd43999613deae3620f4ddcd1fbf.tar.bz2 spark-1f31bdaef670dd43999613deae3620f4ddcd1fbf.zip |
[SPARK-17679] [PYSPARK] remove unnecessary Py4J ListConverter patch
## What changes were proposed in this pull request?
This PR removes a patch on ListConverter from https://github.com/apache/spark/pull/5570, as it is no longer necessary. The underlying issue in Py4J https://github.com/bartdag/py4j/issues/160 was patched in https://github.com/bartdag/py4j/commit/224b94b6665e56a93a064073886e1d803a4969d2 and is present in 0.10.3, the version currently in use in Spark.
## How was this patch tested?
The original test added in https://github.com/apache/spark/pull/5570 remains.
Author: Jason White <jason.white@shopify.com>
Closes #15254 from JasonMWhite/remove_listconverter_patch.
Diffstat (limited to 'python/pyspark/rdd.py')
-rw-r--r-- | python/pyspark/rdd.py | 13 |
1 files changed, 2 insertions, 11 deletions
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py index 5fb10f86f4..ed81eb16df 100644 --- a/python/pyspark/rdd.py +++ b/python/pyspark/rdd.py @@ -52,8 +52,6 @@ from pyspark.shuffle import Aggregator, ExternalMerger, \ get_used_memory, ExternalSorter, ExternalGroupBy from pyspark.traceback_utils import SCCallSiteSync -from py4j.java_collections import ListConverter, MapConverter - __all__ = ["RDD"] @@ -2317,16 +2315,9 @@ def _prepare_for_python_RDD(sc, command): # The broadcast will have same life cycle as created PythonRDD broadcast = sc.broadcast(pickled_command) pickled_command = ser.dumps(broadcast) - # There is a bug in py4j.java_gateway.JavaClass with auto_convert - # https://github.com/bartdag/py4j/issues/161 - # TODO: use auto_convert once py4j fix the bug - broadcast_vars = ListConverter().convert( - [x._jbroadcast for x in sc._pickled_broadcast_vars], - sc._gateway._gateway_client) + broadcast_vars = [x._jbroadcast for x in sc._pickled_broadcast_vars] sc._pickled_broadcast_vars.clear() - env = MapConverter().convert(sc.environment, sc._gateway._gateway_client) - includes = ListConverter().convert(sc._python_includes, sc._gateway._gateway_client) - return pickled_command, broadcast_vars, env, includes + return pickled_command, broadcast_vars, sc.environment, sc._python_includes def _wrap_function(sc, func, deserializer, serializer, profiler=None): |