diff options
| author | Davies Liu <davies@databricks.com> | 2015-05-18 12:55:13 -0700 |
|---|---|---|
| committer | Josh Rosen <joshrosen@databricks.com> | 2015-05-18 12:55:13 -0700 |
| commit | 32fbd297dd651ba3ce4ce52aeb0488233149cdf9 (patch) | |
| tree | c18970db228448c856f8ce9538d708511e9e95fd /python/pyspark/sql | |
| parent | 9dadf019b93038e1e18336ccd06c5eecb4bae32f (diff) | |
| download | spark-32fbd297dd651ba3ce4ce52aeb0488233149cdf9.tar.gz spark-32fbd297dd651ba3ce4ce52aeb0488233149cdf9.tar.bz2 spark-32fbd297dd651ba3ce4ce52aeb0488233149cdf9.zip | |
[SPARK-6216] [PYSPARK] check python version of worker with driver
This PR revert #5404, change to pass the version of python in driver into JVM, check it in worker before deserializing closure, then it can works with different major version of Python.
Author: Davies Liu <davies@databricks.com>
Closes #6203 from davies/py_version and squashes the following commits:
b8fb76e [Davies Liu] fix test
6ce5096 [Davies Liu] use string for version
47c6278 [Davies Liu] check python version of worker with driver
Diffstat (limited to 'python/pyspark/sql')
| -rw-r--r-- | python/pyspark/sql/context.py | 1 | ||||
| -rw-r--r-- | python/pyspark/sql/functions.py | 4 |
2 files changed, 3 insertions, 2 deletions
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py index f6f107ca32..0bde719124 100644 --- a/python/pyspark/sql/context.py +++ b/python/pyspark/sql/context.py @@ -157,6 +157,7 @@ class SQLContext(object): env, includes, self._sc.pythonExec, + self._sc.pythonVer, bvars, self._sc._javaAccumulator, returnType.json()) diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 8d0e766ecd..fbe9bf5b52 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -353,8 +353,8 @@ class UserDefinedFunction(object): ssql_ctx = sc._jvm.SQLContext(sc._jsc.sc()) jdt = ssql_ctx.parseDataType(self.returnType.json()) fname = f.__name__ if hasattr(f, '__name__') else f.__class__.__name__ - judf = sc._jvm.UserDefinedPythonFunction(fname, bytearray(pickled_command), env, - includes, sc.pythonExec, broadcast_vars, + judf = sc._jvm.UserDefinedPythonFunction(fname, bytearray(pickled_command), env, includes, + sc.pythonExec, sc.pythonVer, broadcast_vars, sc._javaAccumulator, jdt) return judf |