diff options
| author | freeman <the.freeman.lab@gmail.com> | 2015-01-05 13:10:59 -0800 |
|---|---|---|
| committer | Xiangrui Meng <meng@databricks.com> | 2015-01-05 13:11:47 -0800 |
| commit | cf55a2b0e14649295b79d0bed365fb87df844361 (patch) | |
| tree | 67a1533f07b25df0847107991bdf482855a2b94b | |
| parent | f979205c1ca87eb7834a7a81381bd32ee0e3095a (diff) | |
| download | spark-cf55a2b0e14649295b79d0bed365fb87df844361.tar.gz spark-cf55a2b0e14649295b79d0bed365fb87df844361.tar.bz2 spark-cf55a2b0e14649295b79d0bed365fb87df844361.zip | |
[SPARK-5089][PYSPARK][MLLIB] Fix vector convert
This is a small change addressing a potentially significant bug in how PySpark + MLlib handles non-float64 numpy arrays. The automatic conversion to `DenseVector` that occurs when passing RDDs to MLlib algorithms in PySpark should automatically upcast to float64s, but currently this wasn't actually happening. As a result, non-float64 would be silently parsed inappropriately during SerDe, yielding erroneous results when running, for example, KMeans.
The PR includes the fix, as well as a new test for the correct conversion behavior.
davies
Author: freeman <the.freeman.lab@gmail.com>
Closes #3902 from freeman-lab/fix-vector-convert and squashes the following commits:
764db47 [freeman] Add a test for proper conversion behavior
704f97e [freeman] Return array after changing type
(cherry picked from commit 6c6f32574023b8e43a24f2081ff17e6e446de2f3)
Signed-off-by: Xiangrui Meng <meng@databricks.com>
| -rw-r--r-- | python/pyspark/mllib/linalg.py | 2 | ||||
| -rw-r--r-- | python/pyspark/mllib/tests.py | 10 |
2 files changed, 11 insertions, 1 deletions
diff --git a/python/pyspark/mllib/linalg.py b/python/pyspark/mllib/linalg.py index f7aa2b0cb0..4f8491f43e 100644 --- a/python/pyspark/mllib/linalg.py +++ b/python/pyspark/mllib/linalg.py @@ -178,7 +178,7 @@ class DenseVector(Vector): elif not isinstance(ar, np.ndarray): ar = np.array(ar, dtype=np.float64) if ar.dtype != np.float64: - ar.astype(np.float64) + ar = ar.astype(np.float64) self.array = ar def __reduce__(self): diff --git a/python/pyspark/mllib/tests.py b/python/pyspark/mllib/tests.py index 8332f8e061..bc2ee5af49 100644 --- a/python/pyspark/mllib/tests.py +++ b/python/pyspark/mllib/tests.py @@ -109,6 +109,16 @@ class VectorTests(PySparkTestCase): self.assertEquals(0.0, _squared_distance(dv, dv)) self.assertEquals(0.0, _squared_distance(lst, lst)) + def test_conversion(self): + # numpy arrays should be automatically upcast to float64 + # tests for fix of [SPARK-5089] + v = array([1, 2, 3, 4], dtype='float64') + dv = DenseVector(v) + self.assertTrue(dv.array.dtype == 'float64') + v = array([1, 2, 3, 4], dtype='float32') + dv = DenseVector(v) + self.assertTrue(dv.array.dtype == 'float64') + class ListTests(PySparkTestCase): |