aboutsummaryrefslogtreecommitdiff
path: root/python/pyspark/worker.py
diff options
context:
space:
mode:
authorDavies Liu <davies@databricks.com>2015-04-10 14:04:53 -0700
committerJosh Rosen <joshrosen@databricks.com>2015-04-10 14:04:53 -0700
commit4740d6a158cb4d35408a95265c5b950b9e9628a3 (patch)
tree82ddbf26be5bc78e074c5b4034eaa3e7f0655f59 /python/pyspark/worker.py
parent0375134f42197f2e29aa865a513cda381f0a1445 (diff)
downloadspark-4740d6a158cb4d35408a95265c5b950b9e9628a3.tar.gz
spark-4740d6a158cb4d35408a95265c5b950b9e9628a3.tar.bz2
spark-4740d6a158cb4d35408a95265c5b950b9e9628a3.zip
[SPARK-6216] [PySpark] check the python version in worker
Author: Davies Liu <davies@databricks.com> Closes #5404 from davies/check_version and squashes the following commits: e559248 [Davies Liu] add tests ec33b5f [Davies Liu] check the python version in worker
Diffstat (limited to 'python/pyspark/worker.py')
-rw-r--r--python/pyspark/worker.py6
1 files changed, 5 insertions, 1 deletions
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index 8a93c320ec..452d6fabdc 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -88,7 +88,11 @@ def main(infile, outfile):
command = pickleSer._read_with_length(infile)
if isinstance(command, Broadcast):
command = pickleSer.loads(command.value)
- (func, profiler, deserializer, serializer) = command
+ (func, profiler, deserializer, serializer), version = command
+ if version != sys.version_info[:2]:
+ raise Exception(("Python in worker has different version %s than that in " +
+ "driver %s, PySpark cannot run with different minor versions") %
+ (sys.version_info[:2], version))
init_time = time.time()
def process():