diff options
| author | Davies Liu <davies@databricks.com> | 2015-04-10 14:04:53 -0700 |
|---|---|---|
| committer | Josh Rosen <joshrosen@databricks.com> | 2015-04-10 14:04:53 -0700 |
| commit | 4740d6a158cb4d35408a95265c5b950b9e9628a3 (patch) | |
| tree | 82ddbf26be5bc78e074c5b4034eaa3e7f0655f59 /python/pyspark/rdd.py | |
| parent | 0375134f42197f2e29aa865a513cda381f0a1445 (diff) | |
| download | spark-4740d6a158cb4d35408a95265c5b950b9e9628a3.tar.gz spark-4740d6a158cb4d35408a95265c5b950b9e9628a3.tar.bz2 spark-4740d6a158cb4d35408a95265c5b950b9e9628a3.zip | |
[SPARK-6216] [PySpark] check the python version in worker
Author: Davies Liu <davies@databricks.com>
Closes #5404 from davies/check_version and squashes the following commits:
e559248 [Davies Liu] add tests
ec33b5f [Davies Liu] check the python version in worker
Diffstat (limited to 'python/pyspark/rdd.py')
| -rw-r--r-- | python/pyspark/rdd.py | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py index c8e54ed5c6..c9ac95d117 100644 --- a/python/pyspark/rdd.py +++ b/python/pyspark/rdd.py @@ -2233,7 +2233,7 @@ class RDD(object): def _prepare_for_python_RDD(sc, command, obj=None): # the serialized command will be compressed by broadcast ser = CloudPickleSerializer() - pickled_command = ser.dumps(command) + pickled_command = ser.dumps((command, sys.version_info[:2])) if len(pickled_command) > (1 << 20): # 1M broadcast = sc.broadcast(pickled_command) pickled_command = ser.dumps(broadcast) |