aboutsummaryrefslogtreecommitdiff
path: root/mllib-local/src/test/scala/org/apache
diff options
context:
space:
mode:
Diffstat (limited to 'mllib-local/src/test/scala/org/apache')
-rw-r--r--mllib-local/src/test/scala/org/apache/spark/ml/SparkMLFunSuite.scala (renamed from mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala)16
-rw-r--r--mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala408
-rw-r--r--mllib-local/src/test/scala/org/apache/spark/ml/linalg/BreezeMatrixConversionSuite.scala71
-rw-r--r--mllib-local/src/test/scala/org/apache/spark/ml/linalg/BreezeVectorConversionSuite.scala67
-rw-r--r--mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala511
-rw-r--r--mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala358
-rw-r--r--mllib-local/src/test/scala/org/apache/spark/ml/util/TestingUtils.scala236
-rw-r--r--mllib-local/src/test/scala/org/apache/spark/ml/util/TestingUtilsSuite.scala187
8 files changed, 1847 insertions, 7 deletions
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/SparkMLFunSuite.scala
index 51b7c2409f..cb3b56bba8 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/SparkMLFunSuite.scala
@@ -17,12 +17,14 @@
package org.apache.spark.ml
-import org.scalatest.FunSuite // scalastyle:ignore funsuite
+// scalastyle:off
+import org.scalatest.{BeforeAndAfterAll, FunSuite}
-// This is testing if the new build works. To be removed soon.
-class DummyTestingSuite extends FunSuite { // scalastyle:ignore funsuite
-
- test("This is testing if the new build works.") {
- assert(DummyTesting.add10(15) === 25)
- }
+/**
+ * Base abstract class for all unit tests in Spark for handling common functionality.
+ */
+private[spark] abstract class SparkMLFunSuite
+ extends FunSuite
+ with BeforeAndAfterAll {
+ // scalastyle:on
}
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
new file mode 100644
index 0000000000..8a9f49792c
--- /dev/null
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
@@ -0,0 +1,408 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.linalg
+
+import org.apache.spark.ml.SparkMLFunSuite
+import org.apache.spark.ml.linalg.BLAS._
+import org.apache.spark.ml.util.TestingUtils._
+
+class BLASSuite extends SparkMLFunSuite {
+
+ test("copy") {
+ val sx = Vectors.sparse(4, Array(0, 2), Array(1.0, -2.0))
+ val dx = Vectors.dense(1.0, 0.0, -2.0, 0.0)
+ val sy = Vectors.sparse(4, Array(0, 1, 3), Array(2.0, 1.0, 1.0))
+ val dy = Array(2.0, 1.0, 0.0, 1.0)
+
+ val dy1 = Vectors.dense(dy.clone())
+ copy(sx, dy1)
+ assert(dy1 ~== dx absTol 1e-15)
+
+ val dy2 = Vectors.dense(dy.clone())
+ copy(dx, dy2)
+ assert(dy2 ~== dx absTol 1e-15)
+
+ intercept[IllegalArgumentException] {
+ copy(sx, sy)
+ }
+
+ intercept[IllegalArgumentException] {
+ copy(dx, sy)
+ }
+
+ withClue("vector sizes must match") {
+ intercept[Exception] {
+ copy(sx, Vectors.dense(0.0, 1.0, 2.0))
+ }
+ }
+ }
+
+ test("scal") {
+ val a = 0.1
+ val sx = Vectors.sparse(3, Array(0, 2), Array(1.0, -2.0))
+ val dx = Vectors.dense(1.0, 0.0, -2.0)
+
+ scal(a, sx)
+ assert(sx ~== Vectors.sparse(3, Array(0, 2), Array(0.1, -0.2)) absTol 1e-15)
+
+ scal(a, dx)
+ assert(dx ~== Vectors.dense(0.1, 0.0, -0.2) absTol 1e-15)
+ }
+
+ test("axpy") {
+ val alpha = 0.1
+ val sx = Vectors.sparse(3, Array(0, 2), Array(1.0, -2.0))
+ val dx = Vectors.dense(1.0, 0.0, -2.0)
+ val dy = Array(2.0, 1.0, 0.0)
+ val expected = Vectors.dense(2.1, 1.0, -0.2)
+
+ val dy1 = Vectors.dense(dy.clone())
+ axpy(alpha, sx, dy1)
+ assert(dy1 ~== expected absTol 1e-15)
+
+ val dy2 = Vectors.dense(dy.clone())
+ axpy(alpha, dx, dy2)
+ assert(dy2 ~== expected absTol 1e-15)
+
+ val sy = Vectors.sparse(4, Array(0, 1), Array(2.0, 1.0))
+
+ intercept[IllegalArgumentException] {
+ axpy(alpha, sx, sy)
+ }
+
+ intercept[IllegalArgumentException] {
+ axpy(alpha, dx, sy)
+ }
+
+ withClue("vector sizes must match") {
+ intercept[Exception] {
+ axpy(alpha, sx, Vectors.dense(1.0, 2.0))
+ }
+ }
+ }
+
+ test("dot") {
+ val sx = Vectors.sparse(3, Array(0, 2), Array(1.0, -2.0))
+ val dx = Vectors.dense(1.0, 0.0, -2.0)
+ val sy = Vectors.sparse(3, Array(0, 1), Array(2.0, 1.0))
+ val dy = Vectors.dense(2.0, 1.0, 0.0)
+
+ assert(dot(sx, sy) ~== 2.0 absTol 1e-15)
+ assert(dot(sy, sx) ~== 2.0 absTol 1e-15)
+ assert(dot(sx, dy) ~== 2.0 absTol 1e-15)
+ assert(dot(dy, sx) ~== 2.0 absTol 1e-15)
+ assert(dot(dx, dy) ~== 2.0 absTol 1e-15)
+ assert(dot(dy, dx) ~== 2.0 absTol 1e-15)
+
+ assert(dot(sx, sx) ~== 5.0 absTol 1e-15)
+ assert(dot(dx, dx) ~== 5.0 absTol 1e-15)
+ assert(dot(sx, dx) ~== 5.0 absTol 1e-15)
+ assert(dot(dx, sx) ~== 5.0 absTol 1e-15)
+
+ val sx1 = Vectors.sparse(10, Array(0, 3, 5, 7, 8), Array(1.0, 2.0, 3.0, 4.0, 5.0))
+ val sx2 = Vectors.sparse(10, Array(1, 3, 6, 7, 9), Array(1.0, 2.0, 3.0, 4.0, 5.0))
+ assert(dot(sx1, sx2) ~== 20.0 absTol 1e-15)
+ assert(dot(sx2, sx1) ~== 20.0 absTol 1e-15)
+
+ withClue("vector sizes must match") {
+ intercept[Exception] {
+ dot(sx, Vectors.dense(2.0, 1.0))
+ }
+ }
+ }
+
+ test("spr") {
+ // test dense vector
+ val alpha = 0.1
+ val x = new DenseVector(Array(1.0, 2, 2.1, 4))
+ val U = new DenseVector(Array(1.0, 2, 2, 3, 3, 3, 4, 4, 4, 4))
+ val expected = new DenseVector(Array(1.1, 2.2, 2.4, 3.21, 3.42, 3.441, 4.4, 4.8, 4.84, 5.6))
+
+ spr(alpha, x, U)
+ assert(U ~== expected absTol 1e-9)
+
+ val matrix33 = new DenseVector(Array(1.0, 2, 3, 4, 5))
+ withClue("Size of vector must match the rank of matrix") {
+ intercept[Exception] {
+ spr(alpha, x, matrix33)
+ }
+ }
+
+ // test sparse vector
+ val sv = new SparseVector(4, Array(0, 3), Array(1.0, 2))
+ val U2 = new DenseVector(Array(1.0, 2, 2, 3, 3, 3, 4, 4, 4, 4))
+ spr(0.1, sv, U2)
+ val expectedSparse = new DenseVector(Array(1.1, 2.0, 2.0, 3.0, 3.0, 3.0, 4.2, 4.0, 4.0, 4.4))
+ assert(U2 ~== expectedSparse absTol 1e-15)
+ }
+
+ test("syr") {
+ val dA = new DenseMatrix(4, 4,
+ Array(0.0, 1.2, 2.2, 3.1, 1.2, 3.2, 5.3, 4.6, 2.2, 5.3, 1.8, 3.0, 3.1, 4.6, 3.0, 0.8))
+ val x = new DenseVector(Array(0.0, 2.7, 3.5, 2.1))
+ val alpha = 0.15
+
+ val expected = new DenseMatrix(4, 4,
+ Array(0.0, 1.2, 2.2, 3.1, 1.2, 4.2935, 6.7175, 5.4505, 2.2, 6.7175, 3.6375, 4.1025, 3.1,
+ 5.4505, 4.1025, 1.4615))
+
+ syr(alpha, x, dA)
+
+ assert(dA ~== expected absTol 1e-15)
+
+ val dB =
+ new DenseMatrix(3, 4, Array(0.0, 1.2, 2.2, 3.1, 1.2, 3.2, 5.3, 4.6, 2.2, 5.3, 1.8, 3.0))
+
+ withClue("Matrix A must be a symmetric Matrix") {
+ intercept[Exception] {
+ syr(alpha, x, dB)
+ }
+ }
+
+ val dC =
+ new DenseMatrix(3, 3, Array(0.0, 1.2, 2.2, 1.2, 3.2, 5.3, 2.2, 5.3, 1.8))
+
+ withClue("Size of vector must match the rank of matrix") {
+ intercept[Exception] {
+ syr(alpha, x, dC)
+ }
+ }
+
+ val y = new DenseVector(Array(0.0, 2.7, 3.5, 2.1, 1.5))
+
+ withClue("Size of vector must match the rank of matrix") {
+ intercept[Exception] {
+ syr(alpha, y, dA)
+ }
+ }
+
+ val xSparse = new SparseVector(4, Array(0, 2, 3), Array(1.0, 3.0, 4.0))
+ val dD = new DenseMatrix(4, 4,
+ Array(0.0, 1.2, 2.2, 3.1, 1.2, 3.2, 5.3, 4.6, 2.2, 5.3, 1.8, 3.0, 3.1, 4.6, 3.0, 0.8))
+ syr(0.1, xSparse, dD)
+ val expectedSparse = new DenseMatrix(4, 4,
+ Array(0.1, 1.2, 2.5, 3.5, 1.2, 3.2, 5.3, 4.6, 2.5, 5.3, 2.7, 4.2, 3.5, 4.6, 4.2, 2.4))
+ assert(dD ~== expectedSparse absTol 1e-15)
+ }
+
+ test("gemm") {
+ val dA =
+ new DenseMatrix(4, 3, Array(0.0, 1.0, 0.0, 0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 3.0))
+ val sA = new SparseMatrix(4, 3, Array(0, 1, 3, 4), Array(1, 0, 2, 3), Array(1.0, 2.0, 1.0, 3.0))
+
+ val B = new DenseMatrix(3, 2, Array(1.0, 0.0, 0.0, 0.0, 2.0, 1.0))
+ val expected = new DenseMatrix(4, 2, Array(0.0, 1.0, 0.0, 0.0, 4.0, 0.0, 2.0, 3.0))
+ val BTman = new DenseMatrix(2, 3, Array(1.0, 0.0, 0.0, 2.0, 0.0, 1.0))
+ val BT = B.transpose
+
+ assert(dA.multiply(B) ~== expected absTol 1e-15)
+ assert(sA.multiply(B) ~== expected absTol 1e-15)
+
+ val C1 = new DenseMatrix(4, 2, Array(1.0, 0.0, 2.0, 1.0, 0.0, 0.0, 1.0, 0.0))
+ val C2 = C1.copy
+ val C3 = C1.copy
+ val C4 = C1.copy
+ val C5 = C1.copy
+ val C6 = C1.copy
+ val C7 = C1.copy
+ val C8 = C1.copy
+ val C9 = C1.copy
+ val C10 = C1.copy
+ val C11 = C1.copy
+ val C12 = C1.copy
+ val C13 = C1.copy
+ val C14 = C1.copy
+ val C15 = C1.copy
+ val C16 = C1.copy
+ val C17 = C1.copy
+ val expected2 = new DenseMatrix(4, 2, Array(2.0, 1.0, 4.0, 2.0, 4.0, 0.0, 4.0, 3.0))
+ val expected3 = new DenseMatrix(4, 2, Array(2.0, 2.0, 4.0, 2.0, 8.0, 0.0, 6.0, 6.0))
+ val expected4 = new DenseMatrix(4, 2, Array(5.0, 0.0, 10.0, 5.0, 0.0, 0.0, 5.0, 0.0))
+ val expected5 = C1.copy
+
+ gemm(1.0, dA, B, 2.0, C1)
+ gemm(1.0, sA, B, 2.0, C2)
+ gemm(2.0, dA, B, 2.0, C3)
+ gemm(2.0, sA, B, 2.0, C4)
+ assert(C1 ~== expected2 absTol 1e-15)
+ assert(C2 ~== expected2 absTol 1e-15)
+ assert(C3 ~== expected3 absTol 1e-15)
+ assert(C4 ~== expected3 absTol 1e-15)
+ gemm(1.0, dA, B, 0.0, C17)
+ assert(C17 ~== expected absTol 1e-15)
+ gemm(1.0, sA, B, 0.0, C17)
+ assert(C17 ~== expected absTol 1e-15)
+
+ withClue("columns of A don't match the rows of B") {
+ intercept[Exception] {
+ gemm(1.0, dA.transpose, B, 2.0, C1)
+ }
+ }
+
+ val dATman =
+ new DenseMatrix(3, 4, Array(0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 3.0))
+ val sATman =
+ new SparseMatrix(3, 4, Array(0, 1, 2, 3, 4), Array(1, 0, 1, 2), Array(2.0, 1.0, 1.0, 3.0))
+
+ val dATT = dATman.transpose
+ val sATT = sATman.transpose
+ val BTT = BTman.transpose.asInstanceOf[DenseMatrix]
+
+ assert(dATT.multiply(B) ~== expected absTol 1e-15)
+ assert(sATT.multiply(B) ~== expected absTol 1e-15)
+ assert(dATT.multiply(BTT) ~== expected absTol 1e-15)
+ assert(sATT.multiply(BTT) ~== expected absTol 1e-15)
+
+ gemm(1.0, dATT, BTT, 2.0, C5)
+ gemm(1.0, sATT, BTT, 2.0, C6)
+ gemm(2.0, dATT, BTT, 2.0, C7)
+ gemm(2.0, sATT, BTT, 2.0, C8)
+ gemm(1.0, dA, BTT, 2.0, C9)
+ gemm(1.0, sA, BTT, 2.0, C10)
+ gemm(2.0, dA, BTT, 2.0, C11)
+ gemm(2.0, sA, BTT, 2.0, C12)
+ assert(C5 ~== expected2 absTol 1e-15)
+ assert(C6 ~== expected2 absTol 1e-15)
+ assert(C7 ~== expected3 absTol 1e-15)
+ assert(C8 ~== expected3 absTol 1e-15)
+ assert(C9 ~== expected2 absTol 1e-15)
+ assert(C10 ~== expected2 absTol 1e-15)
+ assert(C11 ~== expected3 absTol 1e-15)
+ assert(C12 ~== expected3 absTol 1e-15)
+
+ gemm(0, dA, B, 5, C13)
+ gemm(0, sA, B, 5, C14)
+ gemm(0, dA, B, 1, C15)
+ gemm(0, sA, B, 1, C16)
+ assert(C13 ~== expected4 absTol 1e-15)
+ assert(C14 ~== expected4 absTol 1e-15)
+ assert(C15 ~== expected5 absTol 1e-15)
+ assert(C16 ~== expected5 absTol 1e-15)
+
+ }
+
+ test("gemv") {
+
+ val dA =
+ new DenseMatrix(4, 3, Array(0.0, 1.0, 0.0, 0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 3.0))
+ val sA = new SparseMatrix(4, 3, Array(0, 1, 3, 4), Array(1, 0, 2, 3), Array(1.0, 2.0, 1.0, 3.0))
+
+ val dA2 =
+ new DenseMatrix(4, 3, Array(0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 3.0), true)
+ val sA2 =
+ new SparseMatrix(4, 3, Array(0, 1, 2, 3, 4), Array(1, 0, 1, 2), Array(2.0, 1.0, 1.0, 3.0),
+ true)
+
+ val dx = new DenseVector(Array(1.0, 2.0, 3.0))
+ val sx = dx.toSparse
+ val expected = new DenseVector(Array(4.0, 1.0, 2.0, 9.0))
+
+ assert(dA.multiply(dx) ~== expected absTol 1e-15)
+ assert(sA.multiply(dx) ~== expected absTol 1e-15)
+ assert(dA.multiply(sx) ~== expected absTol 1e-15)
+ assert(sA.multiply(sx) ~== expected absTol 1e-15)
+
+ val y1 = new DenseVector(Array(1.0, 3.0, 1.0, 0.0))
+ val y2 = y1.copy
+ val y3 = y1.copy
+ val y4 = y1.copy
+ val y5 = y1.copy
+ val y6 = y1.copy
+ val y7 = y1.copy
+ val y8 = y1.copy
+ val y9 = y1.copy
+ val y10 = y1.copy
+ val y11 = y1.copy
+ val y12 = y1.copy
+ val y13 = y1.copy
+ val y14 = y1.copy
+ val y15 = y1.copy
+ val y16 = y1.copy
+
+ val expected2 = new DenseVector(Array(6.0, 7.0, 4.0, 9.0))
+ val expected3 = new DenseVector(Array(10.0, 8.0, 6.0, 18.0))
+
+ gemv(1.0, dA, dx, 2.0, y1)
+ gemv(1.0, sA, dx, 2.0, y2)
+ gemv(1.0, dA, sx, 2.0, y3)
+ gemv(1.0, sA, sx, 2.0, y4)
+
+ gemv(1.0, dA2, dx, 2.0, y5)
+ gemv(1.0, sA2, dx, 2.0, y6)
+ gemv(1.0, dA2, sx, 2.0, y7)
+ gemv(1.0, sA2, sx, 2.0, y8)
+
+ gemv(2.0, dA, dx, 2.0, y9)
+ gemv(2.0, sA, dx, 2.0, y10)
+ gemv(2.0, dA, sx, 2.0, y11)
+ gemv(2.0, sA, sx, 2.0, y12)
+
+ gemv(2.0, dA2, dx, 2.0, y13)
+ gemv(2.0, sA2, dx, 2.0, y14)
+ gemv(2.0, dA2, sx, 2.0, y15)
+ gemv(2.0, sA2, sx, 2.0, y16)
+
+ assert(y1 ~== expected2 absTol 1e-15)
+ assert(y2 ~== expected2 absTol 1e-15)
+ assert(y3 ~== expected2 absTol 1e-15)
+ assert(y4 ~== expected2 absTol 1e-15)
+
+ assert(y5 ~== expected2 absTol 1e-15)
+ assert(y6 ~== expected2 absTol 1e-15)
+ assert(y7 ~== expected2 absTol 1e-15)
+ assert(y8 ~== expected2 absTol 1e-15)
+
+ assert(y9 ~== expected3 absTol 1e-15)
+ assert(y10 ~== expected3 absTol 1e-15)
+ assert(y11 ~== expected3 absTol 1e-15)
+ assert(y12 ~== expected3 absTol 1e-15)
+
+ assert(y13 ~== expected3 absTol 1e-15)
+ assert(y14 ~== expected3 absTol 1e-15)
+ assert(y15 ~== expected3 absTol 1e-15)
+ assert(y16 ~== expected3 absTol 1e-15)
+
+ withClue("columns of A don't match the rows of B") {
+ intercept[Exception] {
+ gemv(1.0, dA.transpose, dx, 2.0, y1)
+ }
+ intercept[Exception] {
+ gemv(1.0, sA.transpose, dx, 2.0, y1)
+ }
+ intercept[Exception] {
+ gemv(1.0, dA.transpose, sx, 2.0, y1)
+ }
+ intercept[Exception] {
+ gemv(1.0, sA.transpose, sx, 2.0, y1)
+ }
+ }
+
+ val dAT =
+ new DenseMatrix(3, 4, Array(0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 3.0))
+ val sAT =
+ new SparseMatrix(3, 4, Array(0, 1, 2, 3, 4), Array(1, 0, 1, 2), Array(2.0, 1.0, 1.0, 3.0))
+
+ val dATT = dAT.transpose
+ val sATT = sAT.transpose
+
+ assert(dATT.multiply(dx) ~== expected absTol 1e-15)
+ assert(sATT.multiply(dx) ~== expected absTol 1e-15)
+ assert(dATT.multiply(sx) ~== expected absTol 1e-15)
+ assert(sATT.multiply(sx) ~== expected absTol 1e-15)
+ }
+}
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BreezeMatrixConversionSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BreezeMatrixConversionSuite.scala
new file mode 100644
index 0000000000..70a21e41bf
--- /dev/null
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BreezeMatrixConversionSuite.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.linalg
+
+import breeze.linalg.{CSCMatrix => BSM, DenseMatrix => BDM}
+
+import org.apache.spark.ml.SparkMLFunSuite
+
+class BreezeMatrixConversionSuite extends SparkMLFunSuite {
+ test("dense matrix to breeze") {
+ val mat = Matrices.dense(3, 2, Array(0.0, 1.0, 2.0, 3.0, 4.0, 5.0))
+ val breeze = mat.toBreeze.asInstanceOf[BDM[Double]]
+ assert(breeze.rows === mat.numRows)
+ assert(breeze.cols === mat.numCols)
+ assert(breeze.data.eq(mat.asInstanceOf[DenseMatrix].values), "should not copy data")
+ }
+
+ test("dense breeze matrix to matrix") {
+ val breeze = new BDM[Double](3, 2, Array(0.0, 1.0, 2.0, 3.0, 4.0, 5.0))
+ val mat = Matrices.fromBreeze(breeze).asInstanceOf[DenseMatrix]
+ assert(mat.numRows === breeze.rows)
+ assert(mat.numCols === breeze.cols)
+ assert(mat.values.eq(breeze.data), "should not copy data")
+ // transposed matrix
+ val matTransposed = Matrices.fromBreeze(breeze.t).asInstanceOf[DenseMatrix]
+ assert(matTransposed.numRows === breeze.cols)
+ assert(matTransposed.numCols === breeze.rows)
+ assert(matTransposed.values.eq(breeze.data), "should not copy data")
+ }
+
+ test("sparse matrix to breeze") {
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(1, 2, 1, 2)
+ val mat = Matrices.sparse(3, 2, colPtrs, rowIndices, values)
+ val breeze = mat.toBreeze.asInstanceOf[BSM[Double]]
+ assert(breeze.rows === mat.numRows)
+ assert(breeze.cols === mat.numCols)
+ assert(breeze.data.eq(mat.asInstanceOf[SparseMatrix].values), "should not copy data")
+ }
+
+ test("sparse breeze matrix to sparse matrix") {
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(1, 2, 1, 2)
+ val breeze = new BSM[Double](values, 3, 2, colPtrs, rowIndices)
+ val mat = Matrices.fromBreeze(breeze).asInstanceOf[SparseMatrix]
+ assert(mat.numRows === breeze.rows)
+ assert(mat.numCols === breeze.cols)
+ assert(mat.values.eq(breeze.data), "should not copy data")
+ val matTransposed = Matrices.fromBreeze(breeze.t).asInstanceOf[SparseMatrix]
+ assert(matTransposed.numRows === breeze.cols)
+ assert(matTransposed.numCols === breeze.rows)
+ assert(!matTransposed.values.eq(breeze.data), "has to copy data")
+ }
+}
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BreezeVectorConversionSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BreezeVectorConversionSuite.scala
new file mode 100644
index 0000000000..00c9ee79eb
--- /dev/null
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BreezeVectorConversionSuite.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.linalg
+
+import breeze.linalg.{DenseVector => BDV, SparseVector => BSV}
+
+import org.apache.spark.ml.SparkMLFunSuite
+
+/**
+ * Test Breeze vector conversions.
+ */
+class BreezeVectorConversionSuite extends SparkMLFunSuite {
+
+ val arr = Array(0.1, 0.2, 0.3, 0.4)
+ val n = 20
+ val indices = Array(0, 3, 5, 10, 13)
+ val values = Array(0.1, 0.5, 0.3, -0.8, -1.0)
+
+ test("dense to breeze") {
+ val vec = Vectors.dense(arr)
+ assert(vec.toBreeze === new BDV[Double](arr))
+ }
+
+ test("sparse to breeze") {
+ val vec = Vectors.sparse(n, indices, values)
+ assert(vec.toBreeze === new BSV[Double](indices, values, n))
+ }
+
+ test("dense breeze to vector") {
+ val breeze = new BDV[Double](arr)
+ val vec = Vectors.fromBreeze(breeze).asInstanceOf[DenseVector]
+ assert(vec.size === arr.length)
+ assert(vec.values.eq(arr), "should not copy data")
+ }
+
+ test("sparse breeze to vector") {
+ val breeze = new BSV[Double](indices, values, n)
+ val vec = Vectors.fromBreeze(breeze).asInstanceOf[SparseVector]
+ assert(vec.size === n)
+ assert(vec.indices.eq(indices), "should not copy data")
+ assert(vec.values.eq(values), "should not copy data")
+ }
+
+ test("sparse breeze with partially-used arrays to vector") {
+ val activeSize = 3
+ val breeze = new BSV[Double](indices, values, activeSize, n)
+ val vec = Vectors.fromBreeze(breeze).asInstanceOf[SparseVector]
+ assert(vec.size === n)
+ assert(vec.indices === indices.slice(0, activeSize))
+ assert(vec.values === values.slice(0, activeSize))
+ }
+}
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
new file mode 100644
index 0000000000..5c69c5ed7b
--- /dev/null
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.linalg
+
+import java.util.Random
+
+import breeze.linalg.{CSCMatrix, Matrix => BM}
+import org.mockito.Mockito.when
+import org.scalatest.mock.MockitoSugar._
+import scala.collection.mutable.{Map => MutableMap}
+
+import org.apache.spark.ml.SparkMLFunSuite
+import org.apache.spark.ml.util.TestingUtils._
+
+class MatricesSuite extends SparkMLFunSuite {
+ test("dense matrix construction") {
+ val m = 3
+ val n = 2
+ val values = Array(0.0, 1.0, 2.0, 3.0, 4.0, 5.0)
+ val mat = Matrices.dense(m, n, values).asInstanceOf[DenseMatrix]
+ assert(mat.numRows === m)
+ assert(mat.numCols === n)
+ assert(mat.values.eq(values), "should not copy data")
+ }
+
+ test("dense matrix construction with wrong dimension") {
+ intercept[RuntimeException] {
+ Matrices.dense(3, 2, Array(0.0, 1.0, 2.0))
+ }
+ }
+
+ test("sparse matrix construction") {
+ val m = 3
+ val n = 4
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 2, 4, 4)
+ val rowIndices = Array(1, 2, 1, 2)
+ val mat = Matrices.sparse(m, n, colPtrs, rowIndices, values).asInstanceOf[SparseMatrix]
+ assert(mat.numRows === m)
+ assert(mat.numCols === n)
+ assert(mat.values.eq(values), "should not copy data")
+ assert(mat.colPtrs.eq(colPtrs), "should not copy data")
+ assert(mat.rowIndices.eq(rowIndices), "should not copy data")
+
+ val entries: Array[(Int, Int, Double)] = Array((2, 2, 3.0), (1, 0, 1.0), (2, 0, 2.0),
+ (1, 2, 2.0), (2, 2, 2.0), (1, 2, 2.0), (0, 0, 0.0))
+
+ val mat2 = SparseMatrix.fromCOO(m, n, entries)
+ assert(mat.toBreeze === mat2.toBreeze)
+ assert(mat2.values.length == 4)
+ }
+
+ test("sparse matrix construction with wrong number of elements") {
+ intercept[IllegalArgumentException] {
+ Matrices.sparse(3, 2, Array(0, 1), Array(1, 2, 1), Array(0.0, 1.0, 2.0))
+ }
+
+ intercept[IllegalArgumentException] {
+ Matrices.sparse(3, 2, Array(0, 1, 2), Array(1, 2), Array(0.0, 1.0, 2.0))
+ }
+ }
+
+ test("index in matrices incorrect input") {
+ val sm = Matrices.sparse(3, 2, Array(0, 2, 3), Array(1, 2, 1), Array(0.0, 1.0, 2.0))
+ val dm = Matrices.dense(3, 2, Array(0.0, 2.3, 1.4, 3.2, 1.0, 9.1))
+ Array(sm, dm).foreach { mat =>
+ intercept[IllegalArgumentException] { mat.index(4, 1) }
+ intercept[IllegalArgumentException] { mat.index(1, 4) }
+ intercept[IllegalArgumentException] { mat.index(-1, 2) }
+ intercept[IllegalArgumentException] { mat.index(1, -2) }
+ }
+ }
+
+ test("equals") {
+ val dm1 = Matrices.dense(2, 2, Array(0.0, 1.0, 2.0, 3.0))
+ assert(dm1 === dm1)
+ assert(dm1 !== dm1.transpose)
+
+ val dm2 = Matrices.dense(2, 2, Array(0.0, 2.0, 1.0, 3.0))
+ assert(dm1 === dm2.transpose)
+
+ val sm1 = dm1.asInstanceOf[DenseMatrix].toSparse
+ assert(sm1 === sm1)
+ assert(sm1 === dm1)
+ assert(sm1 !== sm1.transpose)
+
+ val sm2 = dm2.asInstanceOf[DenseMatrix].toSparse
+ assert(sm1 === sm2.transpose)
+ assert(sm1 === dm2.transpose)
+ }
+
+ test("matrix copies are deep copies") {
+ val m = 3
+ val n = 2
+
+ val denseMat = Matrices.dense(m, n, Array(0.0, 1.0, 2.0, 3.0, 4.0, 5.0))
+ val denseCopy = denseMat.copy
+
+ assert(!denseMat.toArray.eq(denseCopy.toArray))
+
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(1, 2, 1, 2)
+ val sparseMat = Matrices.sparse(m, n, colPtrs, rowIndices, values)
+ val sparseCopy = sparseMat.copy
+
+ assert(!sparseMat.toArray.eq(sparseCopy.toArray))
+ }
+
+ test("matrix indexing and updating") {
+ val m = 3
+ val n = 2
+ val allValues = Array(0.0, 1.0, 2.0, 3.0, 4.0, 0.0)
+
+ val denseMat = new DenseMatrix(m, n, allValues)
+
+ assert(denseMat(0, 1) === 3.0)
+ assert(denseMat(0, 1) === denseMat.values(3))
+ assert(denseMat(0, 1) === denseMat(3))
+ assert(denseMat(0, 0) === 0.0)
+
+ denseMat.update(0, 0, 10.0)
+ assert(denseMat(0, 0) === 10.0)
+ assert(denseMat.values(0) === 10.0)
+
+ val sparseValues = Array(1.0, 2.0, 3.0, 4.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(1, 2, 0, 1)
+ val sparseMat = new SparseMatrix(m, n, colPtrs, rowIndices, sparseValues)
+
+ assert(sparseMat(0, 1) === 3.0)
+ assert(sparseMat(0, 1) === sparseMat.values(2))
+ assert(sparseMat(0, 0) === 0.0)
+
+ intercept[NoSuchElementException] {
+ sparseMat.update(0, 0, 10.0)
+ }
+
+ intercept[NoSuchElementException] {
+ sparseMat.update(2, 1, 10.0)
+ }
+
+ sparseMat.update(0, 1, 10.0)
+ assert(sparseMat(0, 1) === 10.0)
+ assert(sparseMat.values(2) === 10.0)
+ }
+
+ test("toSparse, toDense") {
+ val m = 3
+ val n = 2
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val allValues = Array(1.0, 2.0, 0.0, 0.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(0, 1, 1, 2)
+
+ val spMat1 = new SparseMatrix(m, n, colPtrs, rowIndices, values)
+ val deMat1 = new DenseMatrix(m, n, allValues)
+
+ val spMat2 = deMat1.toSparse
+ val deMat2 = spMat1.toDense
+
+ assert(spMat1.toBreeze === spMat2.toBreeze)
+ assert(deMat1.toBreeze === deMat2.toBreeze)
+ }
+
+ test("map, update") {
+ val m = 3
+ val n = 2
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val allValues = Array(1.0, 2.0, 0.0, 0.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(0, 1, 1, 2)
+
+ val spMat1 = new SparseMatrix(m, n, colPtrs, rowIndices, values)
+ val deMat1 = new DenseMatrix(m, n, allValues)
+ val deMat2 = deMat1.map(_ * 2)
+ val spMat2 = spMat1.map(_ * 2)
+ deMat1.update(_ * 2)
+ spMat1.update(_ * 2)
+
+ assert(spMat1.toArray === spMat2.toArray)
+ assert(deMat1.toArray === deMat2.toArray)
+ }
+
+ test("transpose") {
+ val dA =
+ new DenseMatrix(4, 3, Array(0.0, 1.0, 0.0, 0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 3.0))
+ val sA = new SparseMatrix(4, 3, Array(0, 1, 3, 4), Array(1, 0, 2, 3), Array(1.0, 2.0, 1.0, 3.0))
+
+ val dAT = dA.transpose.asInstanceOf[DenseMatrix]
+ val sAT = sA.transpose.asInstanceOf[SparseMatrix]
+ val dATexpected =
+ new DenseMatrix(3, 4, Array(0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 3.0))
+ val sATexpected =
+ new SparseMatrix(3, 4, Array(0, 1, 2, 3, 4), Array(1, 0, 1, 2), Array(2.0, 1.0, 1.0, 3.0))
+
+ assert(dAT.toBreeze === dATexpected.toBreeze)
+ assert(sAT.toBreeze === sATexpected.toBreeze)
+ assert(dA(1, 0) === dAT(0, 1))
+ assert(dA(2, 1) === dAT(1, 2))
+ assert(sA(1, 0) === sAT(0, 1))
+ assert(sA(2, 1) === sAT(1, 2))
+
+ assert(!dA.toArray.eq(dAT.toArray), "has to have a new array")
+ assert(dA.values.eq(dAT.transpose.asInstanceOf[DenseMatrix].values), "should not copy array")
+
+ assert(dAT.toSparse.toBreeze === sATexpected.toBreeze)
+ assert(sAT.toDense.toBreeze === dATexpected.toBreeze)
+ }
+
+ test("foreachActive") {
+ val m = 3
+ val n = 2
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val allValues = Array(1.0, 2.0, 0.0, 0.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(0, 1, 1, 2)
+
+ val sp = new SparseMatrix(m, n, colPtrs, rowIndices, values)
+ val dn = new DenseMatrix(m, n, allValues)
+
+ val dnMap = MutableMap[(Int, Int), Double]()
+ dn.foreachActive { (i, j, value) =>
+ dnMap.put((i, j), value)
+ }
+ assert(dnMap.size === 6)
+ assert(dnMap(0, 0) === 1.0)
+ assert(dnMap(1, 0) === 2.0)
+ assert(dnMap(2, 0) === 0.0)
+ assert(dnMap(0, 1) === 0.0)
+ assert(dnMap(1, 1) === 4.0)
+ assert(dnMap(2, 1) === 5.0)
+
+ val spMap = MutableMap[(Int, Int), Double]()
+ sp.foreachActive { (i, j, value) =>
+ spMap.put((i, j), value)
+ }
+ assert(spMap.size === 4)
+ assert(spMap(0, 0) === 1.0)
+ assert(spMap(1, 0) === 2.0)
+ assert(spMap(1, 1) === 4.0)
+ assert(spMap(2, 1) === 5.0)
+ }
+
+ test("horzcat, vertcat, eye, speye") {
+ val m = 3
+ val n = 2
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val allValues = Array(1.0, 2.0, 0.0, 0.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(0, 1, 1, 2)
+ // transposed versions
+ val allValuesT = Array(1.0, 0.0, 2.0, 4.0, 0.0, 5.0)
+ val colPtrsT = Array(0, 1, 3, 4)
+ val rowIndicesT = Array(0, 0, 1, 1)
+
+ val spMat1 = new SparseMatrix(m, n, colPtrs, rowIndices, values)
+ val deMat1 = new DenseMatrix(m, n, allValues)
+ val spMat1T = new SparseMatrix(n, m, colPtrsT, rowIndicesT, values)
+ val deMat1T = new DenseMatrix(n, m, allValuesT)
+
+ // should equal spMat1 & deMat1 respectively
+ val spMat1TT = spMat1T.transpose
+ val deMat1TT = deMat1T.transpose
+
+ val deMat2 = Matrices.eye(3)
+ val spMat2 = Matrices.speye(3)
+ val deMat3 = Matrices.eye(2)
+ val spMat3 = Matrices.speye(2)
+
+ val spHorz = Matrices.horzcat(Array(spMat1, spMat2))
+ val spHorz2 = Matrices.horzcat(Array(spMat1, deMat2))
+ val spHorz3 = Matrices.horzcat(Array(deMat1, spMat2))
+ val deHorz1 = Matrices.horzcat(Array(deMat1, deMat2))
+ val deHorz2 = Matrices.horzcat(Array[Matrix]())
+
+ assert(deHorz1.numRows === 3)
+ assert(spHorz2.numRows === 3)
+ assert(spHorz3.numRows === 3)
+ assert(spHorz.numRows === 3)
+ assert(deHorz1.numCols === 5)
+ assert(spHorz2.numCols === 5)
+ assert(spHorz3.numCols === 5)
+ assert(spHorz.numCols === 5)
+ assert(deHorz2.numRows === 0)
+ assert(deHorz2.numCols === 0)
+ assert(deHorz2.toArray.length === 0)
+
+ assert(deHorz1 ~== spHorz2.asInstanceOf[SparseMatrix].toDense absTol 1e-15)
+ assert(spHorz2 ~== spHorz3 absTol 1e-15)
+ assert(spHorz(0, 0) === 1.0)
+ assert(spHorz(2, 1) === 5.0)
+ assert(spHorz(0, 2) === 1.0)
+ assert(spHorz(1, 2) === 0.0)
+ assert(spHorz(1, 3) === 1.0)
+ assert(spHorz(2, 4) === 1.0)
+ assert(spHorz(1, 4) === 0.0)
+ assert(deHorz1(0, 0) === 1.0)
+ assert(deHorz1(2, 1) === 5.0)
+ assert(deHorz1(0, 2) === 1.0)
+ assert(deHorz1(1, 2) == 0.0)
+ assert(deHorz1(1, 3) === 1.0)
+ assert(deHorz1(2, 4) === 1.0)
+ assert(deHorz1(1, 4) === 0.0)
+
+ // containing transposed matrices
+ val spHorzT = Matrices.horzcat(Array(spMat1TT, spMat2))
+ val spHorz2T = Matrices.horzcat(Array(spMat1TT, deMat2))
+ val spHorz3T = Matrices.horzcat(Array(deMat1TT, spMat2))
+ val deHorz1T = Matrices.horzcat(Array(deMat1TT, deMat2))
+
+ assert(deHorz1T ~== deHorz1 absTol 1e-15)
+ assert(spHorzT ~== spHorz absTol 1e-15)
+ assert(spHorz2T ~== spHorz2 absTol 1e-15)
+ assert(spHorz3T ~== spHorz3 absTol 1e-15)
+
+ intercept[IllegalArgumentException] {
+ Matrices.horzcat(Array(spMat1, spMat3))
+ }
+
+ intercept[IllegalArgumentException] {
+ Matrices.horzcat(Array(deMat1, spMat3))
+ }
+
+ val spVert = Matrices.vertcat(Array(spMat1, spMat3))
+ val deVert1 = Matrices.vertcat(Array(deMat1, deMat3))
+ val spVert2 = Matrices.vertcat(Array(spMat1, deMat3))
+ val spVert3 = Matrices.vertcat(Array(deMat1, spMat3))
+ val deVert2 = Matrices.vertcat(Array[Matrix]())
+
+ assert(deVert1.numRows === 5)
+ assert(spVert2.numRows === 5)
+ assert(spVert3.numRows === 5)
+ assert(spVert.numRows === 5)
+ assert(deVert1.numCols === 2)
+ assert(spVert2.numCols === 2)
+ assert(spVert3.numCols === 2)
+ assert(spVert.numCols === 2)
+ assert(deVert2.numRows === 0)
+ assert(deVert2.numCols === 0)
+ assert(deVert2.toArray.length === 0)
+
+ assert(deVert1 ~== spVert2.asInstanceOf[SparseMatrix].toDense absTol 1e-15)
+ assert(spVert2 ~== spVert3 absTol 1e-15)
+ assert(spVert(0, 0) === 1.0)
+ assert(spVert(2, 1) === 5.0)
+ assert(spVert(3, 0) === 1.0)
+ assert(spVert(3, 1) === 0.0)
+ assert(spVert(4, 1) === 1.0)
+ assert(deVert1(0, 0) === 1.0)
+ assert(deVert1(2, 1) === 5.0)
+ assert(deVert1(3, 0) === 1.0)
+ assert(deVert1(3, 1) === 0.0)
+ assert(deVert1(4, 1) === 1.0)
+
+ // containing transposed matrices
+ val spVertT = Matrices.vertcat(Array(spMat1TT, spMat3))
+ val deVert1T = Matrices.vertcat(Array(deMat1TT, deMat3))
+ val spVert2T = Matrices.vertcat(Array(spMat1TT, deMat3))
+ val spVert3T = Matrices.vertcat(Array(deMat1TT, spMat3))
+
+ assert(deVert1T ~== deVert1 absTol 1e-15)
+ assert(spVertT ~== spVert absTol 1e-15)
+ assert(spVert2T ~== spVert2 absTol 1e-15)
+ assert(spVert3T ~== spVert3 absTol 1e-15)
+
+ intercept[IllegalArgumentException] {
+ Matrices.vertcat(Array(spMat1, spMat2))
+ }
+
+ intercept[IllegalArgumentException] {
+ Matrices.vertcat(Array(deMat1, spMat2))
+ }
+ }
+
+ test("zeros") {
+ val mat = Matrices.zeros(2, 3).asInstanceOf[DenseMatrix]
+ assert(mat.numRows === 2)
+ assert(mat.numCols === 3)
+ assert(mat.values.forall(_ == 0.0))
+ }
+
+ test("ones") {
+ val mat = Matrices.ones(2, 3).asInstanceOf[DenseMatrix]
+ assert(mat.numRows === 2)
+ assert(mat.numCols === 3)
+ assert(mat.values.forall(_ == 1.0))
+ }
+
+ test("eye") {
+ val mat = Matrices.eye(2).asInstanceOf[DenseMatrix]
+ assert(mat.numCols === 2)
+ assert(mat.numCols === 2)
+ assert(mat.values.toSeq === Seq(1.0, 0.0, 0.0, 1.0))
+ }
+
+ test("rand") {
+ val rng = mock[Random]
+ when(rng.nextDouble()).thenReturn(1.0, 2.0, 3.0, 4.0)
+ val mat = Matrices.rand(2, 2, rng).asInstanceOf[DenseMatrix]
+ assert(mat.numRows === 2)
+ assert(mat.numCols === 2)
+ assert(mat.values.toSeq === Seq(1.0, 2.0, 3.0, 4.0))
+ }
+
+ test("randn") {
+ val rng = mock[Random]
+ when(rng.nextGaussian()).thenReturn(1.0, 2.0, 3.0, 4.0)
+ val mat = Matrices.randn(2, 2, rng).asInstanceOf[DenseMatrix]
+ assert(mat.numRows === 2)
+ assert(mat.numCols === 2)
+ assert(mat.values.toSeq === Seq(1.0, 2.0, 3.0, 4.0))
+ }
+
+ test("diag") {
+ val mat = Matrices.diag(Vectors.dense(1.0, 2.0)).asInstanceOf[DenseMatrix]
+ assert(mat.numRows === 2)
+ assert(mat.numCols === 2)
+ assert(mat.values.toSeq === Seq(1.0, 0.0, 0.0, 2.0))
+ }
+
+ test("sprand") {
+ val rng = mock[Random]
+ when(rng.nextInt(4)).thenReturn(0, 1, 1, 3, 2, 2, 0, 1, 3, 0)
+ when(rng.nextDouble()).thenReturn(1.0, 2.0, 3.0, 4.0, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0)
+ val mat = SparseMatrix.sprand(4, 4, 0.25, rng)
+ assert(mat.numRows === 4)
+ assert(mat.numCols === 4)
+ assert(mat.rowIndices.toSeq === Seq(3, 0, 2, 1))
+ assert(mat.values.toSeq === Seq(1.0, 2.0, 3.0, 4.0))
+ val mat2 = SparseMatrix.sprand(2, 3, 1.0, rng)
+ assert(mat2.rowIndices.toSeq === Seq(0, 1, 0, 1, 0, 1))
+ assert(mat2.colPtrs.toSeq === Seq(0, 2, 4, 6))
+ }
+
+ test("sprandn") {
+ val rng = mock[Random]
+ when(rng.nextInt(4)).thenReturn(0, 1, 1, 3, 2, 2, 0, 1, 3, 0)
+ when(rng.nextGaussian()).thenReturn(1.0, 2.0, 3.0, 4.0)
+ val mat = SparseMatrix.sprandn(4, 4, 0.25, rng)
+ assert(mat.numRows === 4)
+ assert(mat.numCols === 4)
+ assert(mat.rowIndices.toSeq === Seq(3, 0, 2, 1))
+ assert(mat.values.toSeq === Seq(1.0, 2.0, 3.0, 4.0))
+ }
+
+ test("toString") {
+ val empty = Matrices.ones(0, 0)
+ empty.toString(0, 0)
+
+ val mat = Matrices.rand(5, 10, new Random())
+ mat.toString(-1, -5)
+ mat.toString(0, 0)
+ mat.toString(Int.MinValue, Int.MinValue)
+ mat.toString(Int.MaxValue, Int.MaxValue)
+ var lines = mat.toString(6, 50).lines.toArray
+ assert(lines.size == 5 && lines.forall(_.size <= 50))
+
+ lines = mat.toString(5, 100).lines.toArray
+ assert(lines.size == 5 && lines.forall(_.size <= 100))
+ }
+
+ test("numNonzeros and numActives") {
+ val dm1 = Matrices.dense(3, 2, Array(0, 0, -1, 1, 0, 1))
+ assert(dm1.numNonzeros === 3)
+ assert(dm1.numActives === 6)
+
+ val sm1 = Matrices.sparse(3, 2, Array(0, 2, 3), Array(0, 2, 1), Array(0.0, -1.2, 0.0))
+ assert(sm1.numNonzeros === 1)
+ assert(sm1.numActives === 3)
+ }
+
+ test("fromBreeze with sparse matrix") {
+ // colPtr.last does NOT always equal to values.length in breeze SCSMatrix and
+ // invocation of compact() may be necessary. Refer to SPARK-11507
+ val bm1: BM[Double] = new CSCMatrix[Double](
+ Array(1.0, 1, 1), 3, 3, Array(0, 1, 2, 3), Array(0, 1, 2))
+ val bm2: BM[Double] = new CSCMatrix[Double](
+ Array(1.0, 2, 2, 4), 3, 3, Array(0, 0, 2, 4), Array(1, 2, 1, 2))
+ val sum = bm1 + bm2
+ Matrices.fromBreeze(sum)
+ }
+
+ test("row/col iterator") {
+ val dm = new DenseMatrix(3, 2, Array(0, 1, 2, 3, 4, 0))
+ val sm = dm.toSparse
+ val rows = Seq(Vectors.dense(0, 3), Vectors.dense(1, 4), Vectors.dense(2, 0))
+ val cols = Seq(Vectors.dense(0, 1, 2), Vectors.dense(3, 4, 0))
+ for (m <- Seq(dm, sm)) {
+ assert(m.rowIter.toSeq === rows)
+ assert(m.colIter.toSeq === cols)
+ assert(m.transpose.rowIter.toSeq === cols)
+ assert(m.transpose.colIter.toSeq === rows)
+ }
+ }
+}
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
new file mode 100644
index 0000000000..504be36413
--- /dev/null
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.linalg
+
+import scala.util.Random
+
+import breeze.linalg.{squaredDistance => breezeSquaredDistance, DenseMatrix => BDM}
+import org.json4s.jackson.JsonMethods.{parse => parseJson}
+
+import org.apache.spark.ml.SparkMLFunSuite
+import org.apache.spark.ml.util.TestingUtils._
+
+class VectorsSuite extends SparkMLFunSuite {
+
+ val arr = Array(0.1, 0.0, 0.3, 0.4)
+ val n = 4
+ val indices = Array(0, 2, 3)
+ val values = Array(0.1, 0.3, 0.4)
+
+ test("dense vector construction with varargs") {
+ val vec = Vectors.dense(arr).asInstanceOf[DenseVector]
+ assert(vec.size === arr.length)
+ assert(vec.values.eq(arr))
+ }
+
+ test("dense vector construction from a double array") {
+ val vec = Vectors.dense(arr).asInstanceOf[DenseVector]
+ assert(vec.size === arr.length)
+ assert(vec.values.eq(arr))
+ }
+
+ test("sparse vector construction") {
+ val vec = Vectors.sparse(n, indices, values).asInstanceOf[SparseVector]
+ assert(vec.size === n)
+ assert(vec.indices.eq(indices))
+ assert(vec.values.eq(values))
+ }
+
+ test("sparse vector construction with unordered elements") {
+ val vec = Vectors.sparse(n, indices.zip(values).reverse).asInstanceOf[SparseVector]
+ assert(vec.size === n)
+ assert(vec.indices === indices)
+ assert(vec.values === values)
+ }
+
+ test("sparse vector construction with mismatched indices/values array") {
+ intercept[IllegalArgumentException] {
+ Vectors.sparse(4, Array(1, 2, 3), Array(3.0, 5.0, 7.0, 9.0))
+ }
+ intercept[IllegalArgumentException] {
+ Vectors.sparse(4, Array(1, 2, 3), Array(3.0, 5.0))
+ }
+ }
+
+ test("sparse vector construction with too many indices vs size") {
+ intercept[IllegalArgumentException] {
+ Vectors.sparse(3, Array(1, 2, 3, 4), Array(3.0, 5.0, 7.0, 9.0))
+ }
+ }
+
+ test("dense to array") {
+ val vec = Vectors.dense(arr).asInstanceOf[DenseVector]
+ assert(vec.toArray.eq(arr))
+ }
+
+ test("dense argmax") {
+ val vec = Vectors.dense(Array.empty[Double]).asInstanceOf[DenseVector]
+ assert(vec.argmax === -1)
+
+ val vec2 = Vectors.dense(arr).asInstanceOf[DenseVector]
+ assert(vec2.argmax === 3)
+
+ val vec3 = Vectors.dense(Array(-1.0, 0.0, -2.0, 1.0)).asInstanceOf[DenseVector]
+ assert(vec3.argmax === 3)
+ }
+
+ test("sparse to array") {
+ val vec = Vectors.sparse(n, indices, values).asInstanceOf[SparseVector]
+ assert(vec.toArray === arr)
+ }
+
+ test("sparse argmax") {
+ val vec = Vectors.sparse(0, Array.empty[Int], Array.empty[Double]).asInstanceOf[SparseVector]
+ assert(vec.argmax === -1)
+
+ val vec2 = Vectors.sparse(n, indices, values).asInstanceOf[SparseVector]
+ assert(vec2.argmax === 3)
+
+ val vec3 = Vectors.sparse(5, Array(2, 3, 4), Array(1.0, 0.0, -.7))
+ assert(vec3.argmax === 2)
+
+ // check for case that sparse vector is created with
+ // only negative values {0.0, 0.0,-1.0, -0.7, 0.0}
+ val vec4 = Vectors.sparse(5, Array(2, 3), Array(-1.0, -.7))
+ assert(vec4.argmax === 0)
+
+ val vec5 = Vectors.sparse(11, Array(0, 3, 10), Array(-1.0, -.7, 0.0))
+ assert(vec5.argmax === 1)
+
+ val vec6 = Vectors.sparse(11, Array(0, 1, 2), Array(-1.0, -.7, 0.0))
+ assert(vec6.argmax === 2)
+
+ val vec7 = Vectors.sparse(5, Array(0, 1, 3), Array(-1.0, 0.0, -.7))
+ assert(vec7.argmax === 1)
+
+ val vec8 = Vectors.sparse(5, Array(1, 2), Array(0.0, -1.0))
+ assert(vec8.argmax === 0)
+ }
+
+ test("vector equals") {
+ val dv1 = Vectors.dense(arr.clone())
+ val dv2 = Vectors.dense(arr.clone())
+ val sv1 = Vectors.sparse(n, indices.clone(), values.clone())
+ val sv2 = Vectors.sparse(n, indices.clone(), values.clone())
+
+ val vectors = Seq(dv1, dv2, sv1, sv2)
+
+ for (v <- vectors; u <- vectors) {
+ assert(v === u)
+ assert(v.## === u.##)
+ }
+
+ val another = Vectors.dense(0.1, 0.2, 0.3, 0.4)
+
+ for (v <- vectors) {
+ assert(v != another)
+ assert(v.## != another.##)
+ }
+ }
+
+ test("vectors equals with explicit 0") {
+ val dv1 = Vectors.dense(Array(0, 0.9, 0, 0.8, 0))
+ val sv1 = Vectors.sparse(5, Array(1, 3), Array(0.9, 0.8))
+ val sv2 = Vectors.sparse(5, Array(0, 1, 2, 3, 4), Array(0, 0.9, 0, 0.8, 0))
+
+ val vectors = Seq(dv1, sv1, sv2)
+ for (v <- vectors; u <- vectors) {
+ assert(v === u)
+ assert(v.## === u.##)
+ }
+
+ val another = Vectors.sparse(5, Array(0, 1, 3), Array(0, 0.9, 0.2))
+ for (v <- vectors) {
+ assert(v != another)
+ assert(v.## != another.##)
+ }
+ }
+
+ test("indexing dense vectors") {
+ val vec = Vectors.dense(1.0, 2.0, 3.0, 4.0)
+ assert(vec(0) === 1.0)
+ assert(vec(3) === 4.0)
+ }
+
+ test("indexing sparse vectors") {
+ val vec = Vectors.sparse(7, Array(0, 2, 4, 6), Array(1.0, 2.0, 3.0, 4.0))
+ assert(vec(0) === 1.0)
+ assert(vec(1) === 0.0)
+ assert(vec(2) === 2.0)
+ assert(vec(3) === 0.0)
+ assert(vec(6) === 4.0)
+ val vec2 = Vectors.sparse(8, Array(0, 2, 4, 6), Array(1.0, 2.0, 3.0, 4.0))
+ assert(vec2(6) === 4.0)
+ assert(vec2(7) === 0.0)
+ }
+
+ test("zeros") {
+ assert(Vectors.zeros(3) === Vectors.dense(0.0, 0.0, 0.0))
+ }
+
+ test("Vector.copy") {
+ val sv = Vectors.sparse(4, Array(0, 2), Array(1.0, 2.0))
+ val svCopy = sv.copy
+ (sv, svCopy) match {
+ case (sv: SparseVector, svCopy: SparseVector) =>
+ assert(sv.size === svCopy.size)
+ assert(sv.indices === svCopy.indices)
+ assert(sv.values === svCopy.values)
+ assert(!sv.indices.eq(svCopy.indices))
+ assert(!sv.values.eq(svCopy.values))
+ case _ =>
+ throw new RuntimeException(s"copy returned ${svCopy.getClass} on ${sv.getClass}.")
+ }
+
+ val dv = Vectors.dense(1.0, 0.0, 2.0)
+ val dvCopy = dv.copy
+ (dv, dvCopy) match {
+ case (dv: DenseVector, dvCopy: DenseVector) =>
+ assert(dv.size === dvCopy.size)
+ assert(dv.values === dvCopy.values)
+ assert(!dv.values.eq(dvCopy.values))
+ case _ =>
+ throw new RuntimeException(s"copy returned ${dvCopy.getClass} on ${dv.getClass}.")
+ }
+ }
+
+ test("fromBreeze") {
+ val x = BDM.zeros[Double](10, 10)
+ val v = Vectors.fromBreeze(x(::, 0))
+ assert(v.size === x.rows)
+ }
+
+ test("sqdist") {
+ val random = new Random()
+ for (m <- 1 until 1000 by 100) {
+ val nnz = random.nextInt(m)
+
+ val indices1 = random.shuffle(0 to m - 1).slice(0, nnz).sorted.toArray
+ val values1 = Array.fill(nnz)(random.nextDouble)
+ val sparseVector1 = Vectors.sparse(m, indices1, values1)
+
+ val indices2 = random.shuffle(0 to m - 1).slice(0, nnz).sorted.toArray
+ val values2 = Array.fill(nnz)(random.nextDouble)
+ val sparseVector2 = Vectors.sparse(m, indices2, values2)
+
+ val denseVector1 = Vectors.dense(sparseVector1.toArray)
+ val denseVector2 = Vectors.dense(sparseVector2.toArray)
+
+ val squaredDist = breezeSquaredDistance(sparseVector1.toBreeze, sparseVector2.toBreeze)
+
+ // SparseVector vs. SparseVector
+ assert(Vectors.sqdist(sparseVector1, sparseVector2) ~== squaredDist relTol 1E-8)
+ // DenseVector vs. SparseVector
+ assert(Vectors.sqdist(denseVector1, sparseVector2) ~== squaredDist relTol 1E-8)
+ // DenseVector vs. DenseVector
+ assert(Vectors.sqdist(denseVector1, denseVector2) ~== squaredDist relTol 1E-8)
+ }
+ }
+
+ test("foreachActive") {
+ val dv = Vectors.dense(0.0, 1.2, 3.1, 0.0)
+ val sv = Vectors.sparse(4, Seq((1, 1.2), (2, 3.1), (3, 0.0)))
+
+ val dvMap = scala.collection.mutable.Map[Int, Double]()
+ dv.foreachActive { (index, value) =>
+ dvMap.put(index, value)
+ }
+ assert(dvMap.size === 4)
+ assert(dvMap.get(0) === Some(0.0))
+ assert(dvMap.get(1) === Some(1.2))
+ assert(dvMap.get(2) === Some(3.1))
+ assert(dvMap.get(3) === Some(0.0))
+
+ val svMap = scala.collection.mutable.Map[Int, Double]()
+ sv.foreachActive { (index, value) =>
+ svMap.put(index, value)
+ }
+ assert(svMap.size === 3)
+ assert(svMap.get(1) === Some(1.2))
+ assert(svMap.get(2) === Some(3.1))
+ assert(svMap.get(3) === Some(0.0))
+ }
+
+ test("vector p-norm") {
+ val dv = Vectors.dense(0.0, -1.2, 3.1, 0.0, -4.5, 1.9)
+ val sv = Vectors.sparse(6, Seq((1, -1.2), (2, 3.1), (3, 0.0), (4, -4.5), (5, 1.9)))
+
+ assert(Vectors.norm(dv, 1.0) ~== dv.toArray.foldLeft(0.0)((a, v) =>
+ a + math.abs(v)) relTol 1E-8)
+ assert(Vectors.norm(sv, 1.0) ~== sv.toArray.foldLeft(0.0)((a, v) =>
+ a + math.abs(v)) relTol 1E-8)
+
+ assert(Vectors.norm(dv, 2.0) ~== math.sqrt(dv.toArray.foldLeft(0.0)((a, v) =>
+ a + v * v)) relTol 1E-8)
+ assert(Vectors.norm(sv, 2.0) ~== math.sqrt(sv.toArray.foldLeft(0.0)((a, v) =>
+ a + v * v)) relTol 1E-8)
+
+ assert(Vectors.norm(dv, Double.PositiveInfinity) ~== dv.toArray.map(math.abs).max relTol 1E-8)
+ assert(Vectors.norm(sv, Double.PositiveInfinity) ~== sv.toArray.map(math.abs).max relTol 1E-8)
+
+ assert(Vectors.norm(dv, 3.7) ~== math.pow(dv.toArray.foldLeft(0.0)((a, v) =>
+ a + math.pow(math.abs(v), 3.7)), 1.0 / 3.7) relTol 1E-8)
+ assert(Vectors.norm(sv, 3.7) ~== math.pow(sv.toArray.foldLeft(0.0)((a, v) =>
+ a + math.pow(math.abs(v), 3.7)), 1.0 / 3.7) relTol 1E-8)
+ }
+
+ test("Vector numActive and numNonzeros") {
+ val dv = Vectors.dense(0.0, 2.0, 3.0, 0.0)
+ assert(dv.numActives === 4)
+ assert(dv.numNonzeros === 2)
+
+ val sv = Vectors.sparse(4, Array(0, 1, 2), Array(0.0, 2.0, 3.0))
+ assert(sv.numActives === 3)
+ assert(sv.numNonzeros === 2)
+ }
+
+ test("Vector toSparse and toDense") {
+ val dv0 = Vectors.dense(0.0, 2.0, 3.0, 0.0)
+ assert(dv0.toDense === dv0)
+ val dv0s = dv0.toSparse
+ assert(dv0s.numActives === 2)
+ assert(dv0s === dv0)
+
+ val sv0 = Vectors.sparse(4, Array(0, 1, 2), Array(0.0, 2.0, 3.0))
+ assert(sv0.toDense === sv0)
+ val sv0s = sv0.toSparse
+ assert(sv0s.numActives === 2)
+ assert(sv0s === sv0)
+ }
+
+ test("Vector.compressed") {
+ val dv0 = Vectors.dense(1.0, 2.0, 3.0, 0.0)
+ val dv0c = dv0.compressed.asInstanceOf[DenseVector]
+ assert(dv0c === dv0)
+
+ val dv1 = Vectors.dense(0.0, 2.0, 0.0, 0.0)
+ val dv1c = dv1.compressed.asInstanceOf[SparseVector]
+ assert(dv1 === dv1c)
+ assert(dv1c.numActives === 1)
+
+ val sv0 = Vectors.sparse(4, Array(1, 2), Array(2.0, 0.0))
+ val sv0c = sv0.compressed.asInstanceOf[SparseVector]
+ assert(sv0 === sv0c)
+ assert(sv0c.numActives === 1)
+
+ val sv1 = Vectors.sparse(4, Array(0, 1, 2), Array(1.0, 2.0, 3.0))
+ val sv1c = sv1.compressed.asInstanceOf[DenseVector]
+ assert(sv1 === sv1c)
+ }
+
+ test("SparseVector.slice") {
+ val v = new SparseVector(5, Array(1, 2, 4), Array(1.1, 2.2, 4.4))
+ assert(v.slice(Array(0, 2)) === new SparseVector(2, Array(1), Array(2.2)))
+ assert(v.slice(Array(2, 0)) === new SparseVector(2, Array(0), Array(2.2)))
+ assert(v.slice(Array(2, 0, 3, 4)) === new SparseVector(4, Array(0, 3), Array(2.2, 4.4)))
+ }
+
+ test("toJson/fromJson") {
+ val sv0 = Vectors.sparse(0, Array.empty, Array.empty)
+ val sv1 = Vectors.sparse(1, Array.empty, Array.empty)
+ val sv2 = Vectors.sparse(2, Array(1), Array(2.0))
+ val dv0 = Vectors.dense(Array.empty[Double])
+ val dv1 = Vectors.dense(1.0)
+ val dv2 = Vectors.dense(0.0, 2.0)
+ for (v <- Seq(sv0, sv1, sv2, dv0, dv1, dv2)) {
+ val json = v.toJson
+ parseJson(json) // `json` should be a valid JSON string
+ val u = Vectors.fromJson(json)
+ assert(u.getClass === v.getClass, "toJson/fromJson should preserve vector types.")
+ assert(u === v, "toJson/fromJson should preserve vector values.")
+ }
+ }
+}
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/util/TestingUtils.scala b/mllib-local/src/test/scala/org/apache/spark/ml/util/TestingUtils.scala
new file mode 100644
index 0000000000..2bebaa35ba
--- /dev/null
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/util/TestingUtils.scala
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.util
+
+import org.scalatest.exceptions.TestFailedException
+
+import org.apache.spark.ml.linalg.{Matrix, Vector}
+
+object TestingUtils {
+
+ val ABS_TOL_MSG = " using absolute tolerance"
+ val REL_TOL_MSG = " using relative tolerance"
+
+ /**
+ * Private helper function for comparing two values using relative tolerance.
+ * Note that if x or y is extremely close to zero, i.e., smaller than Double.MinPositiveValue,
+ * the relative tolerance is meaningless, so the exception will be raised to warn users.
+ */
+ private def RelativeErrorComparison(x: Double, y: Double, eps: Double): Boolean = {
+ val absX = math.abs(x)
+ val absY = math.abs(y)
+ val diff = math.abs(x - y)
+ if (x == y) {
+ true
+ } else if (absX < Double.MinPositiveValue || absY < Double.MinPositiveValue) {
+ throw new TestFailedException(
+ s"$x or $y is extremely close to zero, so the relative tolerance is meaningless.", 0)
+ } else {
+ diff < eps * math.min(absX, absY)
+ }
+ }
+
+ /**
+ * Private helper function for comparing two values using absolute tolerance.
+ */
+ private def AbsoluteErrorComparison(x: Double, y: Double, eps: Double): Boolean = {
+ math.abs(x - y) < eps
+ }
+
+ case class CompareDoubleRightSide(
+ fun: (Double, Double, Double) => Boolean, y: Double, eps: Double, method: String)
+
+ /**
+ * Implicit class for comparing two double values using relative tolerance or absolute tolerance.
+ */
+ implicit class DoubleWithAlmostEquals(val x: Double) {
+
+ /**
+ * When the difference of two values are within eps, returns true; otherwise, returns false.
+ */
+ def ~=(r: CompareDoubleRightSide): Boolean = r.fun(x, r.y, r.eps)
+
+ /**
+ * When the difference of two values are within eps, returns false; otherwise, returns true.
+ */
+ def !~=(r: CompareDoubleRightSide): Boolean = !r.fun(x, r.y, r.eps)
+
+ /**
+ * Throws exception when the difference of two values are NOT within eps;
+ * otherwise, returns true.
+ */
+ def ~==(r: CompareDoubleRightSide): Boolean = {
+ if (!r.fun(x, r.y, r.eps)) {
+ throw new TestFailedException(
+ s"Expected $x and ${r.y} to be within ${r.eps}${r.method}.", 0)
+ }
+ true
+ }
+
+ /**
+ * Throws exception when the difference of two values are within eps; otherwise, returns true.
+ */
+ def !~==(r: CompareDoubleRightSide): Boolean = {
+ if (r.fun(x, r.y, r.eps)) {
+ throw new TestFailedException(
+ s"Did not expect $x and ${r.y} to be within ${r.eps}${r.method}.", 0)
+ }
+ true
+ }
+
+ /**
+ * Comparison using absolute tolerance.
+ */
+ def absTol(eps: Double): CompareDoubleRightSide =
+ CompareDoubleRightSide(AbsoluteErrorComparison, x, eps, ABS_TOL_MSG)
+
+ /**
+ * Comparison using relative tolerance.
+ */
+ def relTol(eps: Double): CompareDoubleRightSide =
+ CompareDoubleRightSide(RelativeErrorComparison, x, eps, REL_TOL_MSG)
+
+ override def toString: String = x.toString
+ }
+
+ case class CompareVectorRightSide(
+ fun: (Vector, Vector, Double) => Boolean, y: Vector, eps: Double, method: String)
+
+ /**
+ * Implicit class for comparing two vectors using relative tolerance or absolute tolerance.
+ */
+ implicit class VectorWithAlmostEquals(val x: Vector) {
+
+ /**
+ * When the difference of two vectors are within eps, returns true; otherwise, returns false.
+ */
+ def ~=(r: CompareVectorRightSide): Boolean = r.fun(x, r.y, r.eps)
+
+ /**
+ * When the difference of two vectors are within eps, returns false; otherwise, returns true.
+ */
+ def !~=(r: CompareVectorRightSide): Boolean = !r.fun(x, r.y, r.eps)
+
+ /**
+ * Throws exception when the difference of two vectors are NOT within eps;
+ * otherwise, returns true.
+ */
+ def ~==(r: CompareVectorRightSide): Boolean = {
+ if (!r.fun(x, r.y, r.eps)) {
+ throw new TestFailedException(
+ s"Expected $x and ${r.y} to be within ${r.eps}${r.method} for all elements.", 0)
+ }
+ true
+ }
+
+ /**
+ * Throws exception when the difference of two vectors are within eps; otherwise, returns true.
+ */
+ def !~==(r: CompareVectorRightSide): Boolean = {
+ if (r.fun(x, r.y, r.eps)) {
+ throw new TestFailedException(
+ s"Did not expect $x and ${r.y} to be within ${r.eps}${r.method} for all elements.", 0)
+ }
+ true
+ }
+
+ /**
+ * Comparison using absolute tolerance.
+ */
+ def absTol(eps: Double): CompareVectorRightSide = CompareVectorRightSide(
+ (x: Vector, y: Vector, eps: Double) => {
+ x.toArray.zip(y.toArray).forall(x => x._1 ~= x._2 absTol eps)
+ }, x, eps, ABS_TOL_MSG)
+
+ /**
+ * Comparison using relative tolerance. Note that comparing against sparse vector
+ * with elements having value of zero will raise exception because it involves with
+ * comparing against zero.
+ */
+ def relTol(eps: Double): CompareVectorRightSide = CompareVectorRightSide(
+ (x: Vector, y: Vector, eps: Double) => {
+ x.toArray.zip(y.toArray).forall(x => x._1 ~= x._2 relTol eps)
+ }, x, eps, REL_TOL_MSG)
+
+ override def toString: String = x.toString
+ }
+
+ case class CompareMatrixRightSide(
+ fun: (Matrix, Matrix, Double) => Boolean, y: Matrix, eps: Double, method: String)
+
+ /**
+ * Implicit class for comparing two matrices using relative tolerance or absolute tolerance.
+ */
+ implicit class MatrixWithAlmostEquals(val x: Matrix) {
+
+ /**
+ * When the difference of two matrices are within eps, returns true; otherwise, returns false.
+ */
+ def ~=(r: CompareMatrixRightSide): Boolean = r.fun(x, r.y, r.eps)
+
+ /**
+ * When the difference of two matrices are within eps, returns false; otherwise, returns true.
+ */
+ def !~=(r: CompareMatrixRightSide): Boolean = !r.fun(x, r.y, r.eps)
+
+ /**
+ * Throws exception when the difference of two matrices are NOT within eps;
+ * otherwise, returns true.
+ */
+ def ~==(r: CompareMatrixRightSide): Boolean = {
+ if (!r.fun(x, r.y, r.eps)) {
+ throw new TestFailedException(
+ s"Expected \n$x\n and \n${r.y}\n to be within ${r.eps}${r.method} for all elements.", 0)
+ }
+ true
+ }
+
+ /**
+ * Throws exception when the difference of two matrices are within eps; otherwise, returns true.
+ */
+ def !~==(r: CompareMatrixRightSide): Boolean = {
+ if (r.fun(x, r.y, r.eps)) {
+ throw new TestFailedException(
+ s"Did not expect \n$x\n and \n${r.y}\n to be within " +
+ "${r.eps}${r.method} for all elements.", 0)
+ }
+ true
+ }
+
+ /**
+ * Comparison using absolute tolerance.
+ */
+ def absTol(eps: Double): CompareMatrixRightSide = CompareMatrixRightSide(
+ (x: Matrix, y: Matrix, eps: Double) => {
+ x.toArray.zip(y.toArray).forall(x => x._1 ~= x._2 absTol eps)
+ }, x, eps, ABS_TOL_MSG)
+
+ /**
+ * Comparison using relative tolerance. Note that comparing against sparse vector
+ * with elements having value of zero will raise exception because it involves with
+ * comparing against zero.
+ */
+ def relTol(eps: Double): CompareMatrixRightSide = CompareMatrixRightSide(
+ (x: Matrix, y: Matrix, eps: Double) => {
+ x.toArray.zip(y.toArray).forall(x => x._1 ~= x._2 relTol eps)
+ }, x, eps, REL_TOL_MSG)
+
+ override def toString: String = x.toString
+ }
+
+}
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/util/TestingUtilsSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/util/TestingUtilsSuite.scala
new file mode 100644
index 0000000000..e374165f75
--- /dev/null
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/util/TestingUtilsSuite.scala
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.util
+
+import org.scalatest.exceptions.TestFailedException
+
+import org.apache.spark.ml.SparkMLFunSuite
+import org.apache.spark.ml.linalg.Vectors
+import org.apache.spark.ml.util.TestingUtils._
+
+class TestingUtilsSuite extends SparkMLFunSuite {
+
+ test("Comparing doubles using relative error.") {
+
+ assert(23.1 ~== 23.52 relTol 0.02)
+ assert(23.1 ~== 22.74 relTol 0.02)
+ assert(23.1 ~= 23.52 relTol 0.02)
+ assert(23.1 ~= 22.74 relTol 0.02)
+ assert(!(23.1 !~= 23.52 relTol 0.02))
+ assert(!(23.1 !~= 22.74 relTol 0.02))
+
+ // Should throw exception with message when test fails.
+ intercept[TestFailedException](23.1 !~== 23.52 relTol 0.02)
+ intercept[TestFailedException](23.1 !~== 22.74 relTol 0.02)
+ intercept[TestFailedException](23.1 ~== 23.63 relTol 0.02)
+ intercept[TestFailedException](23.1 ~== 22.34 relTol 0.02)
+
+ assert(23.1 !~== 23.63 relTol 0.02)
+ assert(23.1 !~== 22.34 relTol 0.02)
+ assert(23.1 !~= 23.63 relTol 0.02)
+ assert(23.1 !~= 22.34 relTol 0.02)
+ assert(!(23.1 ~= 23.63 relTol 0.02))
+ assert(!(23.1 ~= 22.34 relTol 0.02))
+
+ // Comparing against zero should fail the test and throw exception with message
+ // saying that the relative error is meaningless in this situation.
+ intercept[TestFailedException](0.1 ~== 0.0 relTol 0.032)
+ intercept[TestFailedException](0.1 ~= 0.0 relTol 0.032)
+ intercept[TestFailedException](0.1 !~== 0.0 relTol 0.032)
+ intercept[TestFailedException](0.1 !~= 0.0 relTol 0.032)
+ intercept[TestFailedException](0.0 ~== 0.1 relTol 0.032)
+ intercept[TestFailedException](0.0 ~= 0.1 relTol 0.032)
+ intercept[TestFailedException](0.0 !~== 0.1 relTol 0.032)
+ intercept[TestFailedException](0.0 !~= 0.1 relTol 0.032)
+
+ // Comparisons of numbers very close to zero.
+ assert(10 * Double.MinPositiveValue ~== 9.5 * Double.MinPositiveValue relTol 0.01)
+ assert(10 * Double.MinPositiveValue !~== 11 * Double.MinPositiveValue relTol 0.01)
+
+ assert(-Double.MinPositiveValue ~== 1.18 * -Double.MinPositiveValue relTol 0.012)
+ assert(-Double.MinPositiveValue ~== 1.38 * -Double.MinPositiveValue relTol 0.012)
+ }
+
+ test("Comparing doubles using absolute error.") {
+
+ assert(17.8 ~== 17.99 absTol 0.2)
+ assert(17.8 ~== 17.61 absTol 0.2)
+ assert(17.8 ~= 17.99 absTol 0.2)
+ assert(17.8 ~= 17.61 absTol 0.2)
+ assert(!(17.8 !~= 17.99 absTol 0.2))
+ assert(!(17.8 !~= 17.61 absTol 0.2))
+
+ // Should throw exception with message when test fails.
+ intercept[TestFailedException](17.8 !~== 17.99 absTol 0.2)
+ intercept[TestFailedException](17.8 !~== 17.61 absTol 0.2)
+ intercept[TestFailedException](17.8 ~== 18.01 absTol 0.2)
+ intercept[TestFailedException](17.8 ~== 17.59 absTol 0.2)
+
+ assert(17.8 !~== 18.01 absTol 0.2)
+ assert(17.8 !~== 17.59 absTol 0.2)
+ assert(17.8 !~= 18.01 absTol 0.2)
+ assert(17.8 !~= 17.59 absTol 0.2)
+ assert(!(17.8 ~= 18.01 absTol 0.2))
+ assert(!(17.8 ~= 17.59 absTol 0.2))
+
+ // Comparisons of numbers very close to zero, and both side of zeros
+ assert(
+ Double.MinPositiveValue ~== 4 * Double.MinPositiveValue absTol 5 * Double.MinPositiveValue)
+ assert(
+ Double.MinPositiveValue !~== 6 * Double.MinPositiveValue absTol 5 * Double.MinPositiveValue)
+
+ assert(
+ -Double.MinPositiveValue ~== 3 * Double.MinPositiveValue absTol 5 * Double.MinPositiveValue)
+ assert(
+ Double.MinPositiveValue !~== -4 * Double.MinPositiveValue absTol 5 * Double.MinPositiveValue)
+ }
+
+ test("Comparing vectors using relative error.") {
+
+ // Comparisons of two dense vectors
+ assert(Vectors.dense(Array(3.1, 3.5)) ~== Vectors.dense(Array(3.130, 3.534)) relTol 0.01)
+ assert(Vectors.dense(Array(3.1, 3.5)) !~== Vectors.dense(Array(3.135, 3.534)) relTol 0.01)
+ assert(Vectors.dense(Array(3.1, 3.5)) ~= Vectors.dense(Array(3.130, 3.534)) relTol 0.01)
+ assert(Vectors.dense(Array(3.1, 3.5)) !~= Vectors.dense(Array(3.135, 3.534)) relTol 0.01)
+ assert(!(Vectors.dense(Array(3.1, 3.5)) !~= Vectors.dense(Array(3.130, 3.534)) relTol 0.01))
+ assert(!(Vectors.dense(Array(3.1, 3.5)) ~= Vectors.dense(Array(3.135, 3.534)) relTol 0.01))
+
+ // Should throw exception with message when test fails.
+ intercept[TestFailedException](
+ Vectors.dense(Array(3.1, 3.5)) !~== Vectors.dense(Array(3.130, 3.534)) relTol 0.01)
+
+ intercept[TestFailedException](
+ Vectors.dense(Array(3.1, 3.5)) ~== Vectors.dense(Array(3.135, 3.534)) relTol 0.01)
+
+ // Comparing against zero should fail the test and throw exception with message
+ // saying that the relative error is meaningless in this situation.
+ intercept[TestFailedException](
+ Vectors.dense(Array(3.1, 0.01)) ~== Vectors.dense(Array(3.13, 0.0)) relTol 0.01)
+
+ intercept[TestFailedException](
+ Vectors.dense(Array(3.1, 0.01)) ~== Vectors.sparse(2, Array(0), Array(3.13)) relTol 0.01)
+
+ // Comparisons of two sparse vectors
+ assert(Vectors.dense(Array(3.1, 3.5)) ~==
+ Vectors.sparse(2, Array(0, 1), Array(3.130, 3.534)) relTol 0.01)
+
+ assert(Vectors.dense(Array(3.1, 3.5)) !~==
+ Vectors.sparse(2, Array(0, 1), Array(3.135, 3.534)) relTol 0.01)
+ }
+
+ test("Comparing vectors using absolute error.") {
+
+ // Comparisons of two dense vectors
+ assert(Vectors.dense(Array(3.1, 3.5, 0.0)) ~==
+ Vectors.dense(Array(3.1 + 1E-8, 3.5 + 2E-7, 1E-8)) absTol 1E-6)
+
+ assert(Vectors.dense(Array(3.1, 3.5, 0.0)) !~==
+ Vectors.dense(Array(3.1 + 1E-5, 3.5 + 2E-7, 1 + 1E-3)) absTol 1E-6)
+
+ assert(Vectors.dense(Array(3.1, 3.5, 0.0)) ~=
+ Vectors.dense(Array(3.1 + 1E-8, 3.5 + 2E-7, 1E-8)) absTol 1E-6)
+
+ assert(Vectors.dense(Array(3.1, 3.5, 0.0)) !~=
+ Vectors.dense(Array(3.1 + 1E-5, 3.5 + 2E-7, 1 + 1E-3)) absTol 1E-6)
+
+ assert(!(Vectors.dense(Array(3.1, 3.5, 0.0)) !~=
+ Vectors.dense(Array(3.1 + 1E-8, 3.5 + 2E-7, 1E-8)) absTol 1E-6))
+
+ assert(!(Vectors.dense(Array(3.1, 3.5, 0.0)) ~=
+ Vectors.dense(Array(3.1 + 1E-5, 3.5 + 2E-7, 1 + 1E-3)) absTol 1E-6))
+
+ // Should throw exception with message when test fails.
+ intercept[TestFailedException](Vectors.dense(Array(3.1, 3.5, 0.0)) !~==
+ Vectors.dense(Array(3.1 + 1E-8, 3.5 + 2E-7, 1E-8)) absTol 1E-6)
+
+ intercept[TestFailedException](Vectors.dense(Array(3.1, 3.5, 0.0)) ~==
+ Vectors.dense(Array(3.1 + 1E-5, 3.5 + 2E-7, 1 + 1E-3)) absTol 1E-6)
+
+ // Comparisons of two sparse vectors
+ assert(Vectors.sparse(3, Array(0, 2), Array(3.1, 2.4)) ~==
+ Vectors.sparse(3, Array(0, 2), Array(3.1 + 1E-8, 2.4 + 1E-7)) absTol 1E-6)
+
+ assert(Vectors.sparse(3, Array(0, 2), Array(3.1 + 1E-8, 2.4 + 1E-7)) ~==
+ Vectors.sparse(3, Array(0, 2), Array(3.1, 2.4)) absTol 1E-6)
+
+ assert(Vectors.sparse(3, Array(0, 2), Array(3.1, 2.4)) !~==
+ Vectors.sparse(3, Array(0, 2), Array(3.1 + 1E-3, 2.4)) absTol 1E-6)
+
+ assert(Vectors.sparse(3, Array(0, 2), Array(3.1 + 1E-3, 2.4)) !~==
+ Vectors.sparse(3, Array(0, 2), Array(3.1, 2.4)) absTol 1E-6)
+
+ // Comparisons of a dense vector and a sparse vector
+ assert(Vectors.sparse(3, Array(0, 2), Array(3.1, 2.4)) ~==
+ Vectors.dense(Array(3.1 + 1E-8, 0, 2.4 + 1E-7)) absTol 1E-6)
+
+ assert(Vectors.dense(Array(3.1 + 1E-8, 0, 2.4 + 1E-7)) ~==
+ Vectors.sparse(3, Array(0, 2), Array(3.1, 2.4)) absTol 1E-6)
+
+ assert(Vectors.sparse(3, Array(0, 2), Array(3.1, 2.4)) !~==
+ Vectors.dense(Array(3.1, 1E-3, 2.4)) absTol 1E-6)
+ }
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-29 04:00:29 +0000