diff options
| author | Davies Liu <davies.liu@gmail.com> | 2014-10-16 14:56:50 -0700 |
|---|---|---|
| committer | Xiangrui Meng <meng@databricks.com> | 2014-10-16 14:56:50 -0700 |
| commit | 091d32c52e9d73da95896016c1d920e89858abfa (patch) | |
| tree | 904edd29e64b57fa1ab72d3ca37ed2996aa9d1e4 /python/pyspark/mllib/regression.py | |
| parent | 4c589cac4496c6a4bb8485a340bd0641dca13847 (diff) | |
| download | spark-091d32c52e9d73da95896016c1d920e89858abfa.tar.gz spark-091d32c52e9d73da95896016c1d920e89858abfa.tar.bz2 spark-091d32c52e9d73da95896016c1d920e89858abfa.zip | |
[SPARK-3971] [MLLib] [PySpark] hotfix: Customized pickler should work in cluster mode
Customized pickler should be registered before unpickling, but in executor, there is no way to register the picklers before run the tasks.
So, we need to register the picklers in the tasks itself, duplicate the javaToPython() and pythonToJava() in MLlib, call SerDe.initialize() before pickling or unpickling.
Author: Davies Liu <davies.liu@gmail.com>
Closes #2830 from davies/fix_pickle and squashes the following commits:
0c85fb9 [Davies Liu] revert the privacy change
6b94e15 [Davies Liu] use JavaConverters instead of JavaConversions
0f02050 [Davies Liu] hotfix: Customized pickler does not work in cluster
Diffstat (limited to 'python/pyspark/mllib/regression.py')
| -rw-r--r-- | python/pyspark/mllib/regression.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/python/pyspark/mllib/regression.py b/python/pyspark/mllib/regression.py index 12b322aaae..93e17faf5c 100644 --- a/python/pyspark/mllib/regression.py +++ b/python/pyspark/mllib/regression.py @@ -19,8 +19,8 @@ import numpy as np from numpy import array from pyspark import SparkContext -from pyspark.mllib.linalg import SparseVector, _convert_to_vector from pyspark.serializers import PickleSerializer, AutoBatchedSerializer +from pyspark.mllib.linalg import SparseVector, _convert_to_vector, _to_java_object_rdd __all__ = ['LabeledPoint', 'LinearModel', 'LinearRegressionModel', 'RidgeRegressionModel', 'LinearRegressionWithSGD', 'LassoWithSGD', 'RidgeRegressionWithSGD'] @@ -131,7 +131,7 @@ def _regression_train_wrapper(sc, train_func, modelClass, data, initial_weights) # use AutoBatchedSerializer before cache to reduce the memory # overhead in JVM cached = data._reserialize(AutoBatchedSerializer(ser)).cache() - ans = train_func(cached._to_java_object_rdd(), initial_bytes) + ans = train_func(_to_java_object_rdd(cached), initial_bytes) assert len(ans) == 2, "JVM call result had unexpected length" weights = ser.loads(str(ans[0])) return modelClass(weights, ans[1]) |