diff options
author | Xusen Yin <yinxusen@gmail.com> | 2016-07-08 14:23:57 +0100 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2016-07-08 14:23:57 +0100 |
commit | 255d74fe4a0db2cc842177ec735bbde07c7c8732 (patch) | |
tree | 930bcb6dbf736faec173c3d3291dafd8f2c034b2 /mllib/src/test | |
parent | a54438cb23c80f7c7fc35da273677c39317cb1a5 (diff) | |
download | spark-255d74fe4a0db2cc842177ec735bbde07c7c8732.tar.gz spark-255d74fe4a0db2cc842177ec735bbde07c7c8732.tar.bz2 spark-255d74fe4a0db2cc842177ec735bbde07c7c8732.zip |
[SPARK-16369][MLLIB] tallSkinnyQR of RowMatrix should aware of empty partition
## What changes were proposed in this pull request?
tallSkinnyQR of RowMatrix should aware of empty partition, which could cause exception from Breeze qr decomposition.
See the [archived dev mail](https://mail-archives.apache.org/mod_mbox/spark-dev/201510.mbox/%3CCAF7ADNrycvPL3qX-VZJhq4OYmiUUhoscut_tkOm63Cm18iK1tQmail.gmail.com%3E) for more details.
## How was this patch tested?
Scala unit test.
Author: Xusen Yin <yinxusen@gmail.com>
Closes #14049 from yinxusen/SPARK-16369.
Diffstat (limited to 'mllib/src/test')
-rw-r--r-- | mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala | 17 |
1 files changed, 17 insertions, 0 deletions
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala index 7c4c6d8409..7c9e14f8ce 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.SparkFunSuite import org.apache.spark.mllib.linalg.{Matrices, Vector, Vectors} import org.apache.spark.mllib.random.RandomRDDs import org.apache.spark.mllib.util.{LocalClusterSparkContext, MLlibTestSparkContext} +import org.apache.spark.mllib.util.TestingUtils._ class RowMatrixSuite extends SparkFunSuite with MLlibTestSparkContext { @@ -281,6 +282,22 @@ class RowMatrixSuite extends SparkFunSuite with MLlibTestSparkContext { assert(cov(i, j) === cov(j, i)) } } + + test("QR decomposition should aware of empty partition (SPARK-16369)") { + val mat: RowMatrix = new RowMatrix(sc.parallelize(denseData, 1)) + val qrResult = mat.tallSkinnyQR(true) + + val matWithEmptyPartition = new RowMatrix(sc.parallelize(denseData, 8)) + val qrResult2 = matWithEmptyPartition.tallSkinnyQR(true) + + assert(qrResult.Q.numCols() === qrResult2.Q.numCols(), "Q matrix ncol not match") + assert(qrResult.Q.numRows() === qrResult2.Q.numRows(), "Q matrix nrow not match") + qrResult.Q.rows.collect().zip(qrResult2.Q.rows.collect()) + .foreach(x => assert(x._1 ~== x._2 relTol 1E-8, "Q matrix not match")) + + qrResult.R.toArray.zip(qrResult2.R.toArray) + .foreach(x => assert(x._1 ~== x._2 relTol 1E-8, "R matrix not match")) + } } class RowMatrixClusterSuite extends SparkFunSuite with LocalClusterSparkContext { |