diff options
author | Sean Owen <sowen@cloudera.com> | 2016-03-08 17:47:55 +0000 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2016-03-08 17:47:55 +0000 |
commit | 54040f8d350d2aad3078dcffef808c62b7c0b73d (patch) | |
tree | 1eacb775f8426130a3152cf9afd30c47fd5f9143 /mllib/src/test/java/org | |
parent | ca1a7b9d6acf8e1f9b6ab6265f9001c2c7ff8489 (diff) | |
download | spark-54040f8d350d2aad3078dcffef808c62b7c0b73d.tar.gz spark-54040f8d350d2aad3078dcffef808c62b7c0b73d.tar.bz2 spark-54040f8d350d2aad3078dcffef808c62b7c0b73d.zip |
[SPARK-13715][MLLIB] Remove last usages of jblas in tests
## What changes were proposed in this pull request?
Remove last usage of jblas, in tests
## How was this patch tested?
Jenkins tests -- the same ones that are being modified.
Author: Sean Owen <sowen@cloudera.com>
Closes #11560 from srowen/SPARK-13715.
Diffstat (limited to 'mllib/src/test/java/org')
-rw-r--r-- | mllib/src/test/java/org/apache/spark/mllib/recommendation/JavaALSSuite.java | 39 | ||||
-rw-r--r-- | mllib/src/test/java/org/apache/spark/mllib/regression/JavaRidgeRegressionSuite.java | 17 |
2 files changed, 29 insertions, 27 deletions
diff --git a/mllib/src/test/java/org/apache/spark/mllib/recommendation/JavaALSSuite.java b/mllib/src/test/java/org/apache/spark/mllib/recommendation/JavaALSSuite.java index a6631ed7eb..d0bf7f556d 100644 --- a/mllib/src/test/java/org/apache/spark/mllib/recommendation/JavaALSSuite.java +++ b/mllib/src/test/java/org/apache/spark/mllib/recommendation/JavaALSSuite.java @@ -24,7 +24,6 @@ import java.util.List; import scala.Tuple2; import scala.Tuple3; -import org.jblas.DoubleMatrix; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -48,14 +47,14 @@ public class JavaALSSuite implements Serializable { sc = null; } - void validatePrediction( + private void validatePrediction( MatrixFactorizationModel model, int users, int products, - DoubleMatrix trueRatings, + double[] trueRatings, double matchThreshold, boolean implicitPrefs, - DoubleMatrix truePrefs) { + double[] truePrefs) { List<Tuple2<Integer, Integer>> localUsersProducts = new ArrayList<>(users * products); for (int u=0; u < users; ++u) { for (int p=0; p < products; ++p) { @@ -68,7 +67,7 @@ public class JavaALSSuite implements Serializable { if (!implicitPrefs) { for (Rating r: predictedRatings) { double prediction = r.rating(); - double correct = trueRatings.get(r.user(), r.product()); + double correct = trueRatings[r.product() * users + r.user()]; Assert.assertTrue(String.format("Prediction=%2.4f not below match threshold of %2.2f", prediction, matchThreshold), Math.abs(prediction - correct) < matchThreshold); } @@ -79,9 +78,9 @@ public class JavaALSSuite implements Serializable { double denom = 0.0; for (Rating r: predictedRatings) { double prediction = r.rating(); - double truePref = truePrefs.get(r.user(), r.product()); + double truePref = truePrefs[r.product() * users + r.user()]; double confidence = 1.0 + - /* alpha = */ 1.0 * Math.abs(trueRatings.get(r.user(), r.product())); + /* alpha = 1.0 * ... */ Math.abs(trueRatings[r.product() * users + r.user()]); double err = confidence * (truePref - prediction) * (truePref - prediction); sqErr += err; denom += confidence; @@ -98,8 +97,8 @@ public class JavaALSSuite implements Serializable { int iterations = 15; int users = 50; int products = 100; - Tuple3<List<Rating>, DoubleMatrix, DoubleMatrix> testData = ALSSuite.generateRatingsAsJavaList( - users, products, features, 0.7, false, false); + Tuple3<List<Rating>, double[], double[]> testData = + ALSSuite.generateRatingsAsJava(users, products, features, 0.7, false, false); JavaRDD<Rating> data = sc.parallelize(testData._1()); MatrixFactorizationModel model = ALS.train(data.rdd(), features, iterations); @@ -112,8 +111,8 @@ public class JavaALSSuite implements Serializable { int iterations = 15; int users = 100; int products = 200; - Tuple3<List<Rating>, DoubleMatrix, DoubleMatrix> testData = ALSSuite.generateRatingsAsJavaList( - users, products, features, 0.7, false, false); + Tuple3<List<Rating>, double[], double[]> testData = + ALSSuite.generateRatingsAsJava(users, products, features, 0.7, false, false); JavaRDD<Rating> data = sc.parallelize(testData._1()); @@ -129,8 +128,8 @@ public class JavaALSSuite implements Serializable { int iterations = 15; int users = 80; int products = 160; - Tuple3<List<Rating>, DoubleMatrix, DoubleMatrix> testData = ALSSuite.generateRatingsAsJavaList( - users, products, features, 0.7, true, false); + Tuple3<List<Rating>, double[], double[]> testData = + ALSSuite.generateRatingsAsJava(users, products, features, 0.7, true, false); JavaRDD<Rating> data = sc.parallelize(testData._1()); MatrixFactorizationModel model = ALS.trainImplicit(data.rdd(), features, iterations); @@ -143,8 +142,8 @@ public class JavaALSSuite implements Serializable { int iterations = 15; int users = 100; int products = 200; - Tuple3<List<Rating>, DoubleMatrix, DoubleMatrix> testData = ALSSuite.generateRatingsAsJavaList( - users, products, features, 0.7, true, false); + Tuple3<List<Rating>, double[], double[]> testData = + ALSSuite.generateRatingsAsJava(users, products, features, 0.7, true, false); JavaRDD<Rating> data = sc.parallelize(testData._1()); @@ -161,8 +160,8 @@ public class JavaALSSuite implements Serializable { int iterations = 15; int users = 80; int products = 160; - Tuple3<List<Rating>, DoubleMatrix, DoubleMatrix> testData = ALSSuite.generateRatingsAsJavaList( - users, products, features, 0.7, true, true); + Tuple3<List<Rating>, double[], double[]> testData = + ALSSuite.generateRatingsAsJava(users, products, features, 0.7, true, true); JavaRDD<Rating> data = sc.parallelize(testData._1()); MatrixFactorizationModel model = new ALS().setRank(features) @@ -179,9 +178,9 @@ public class JavaALSSuite implements Serializable { int iterations = 10; int users = 200; int products = 50; - Tuple3<List<Rating>, DoubleMatrix, DoubleMatrix> testData = ALSSuite.generateRatingsAsJavaList( - users, products, features, 0.7, true, false); - JavaRDD<Rating> data = sc.parallelize(testData._1()); + List<Rating> testData = ALSSuite.generateRatingsAsJava( + users, products, features, 0.7, true, false)._1(); + JavaRDD<Rating> data = sc.parallelize(testData); MatrixFactorizationModel model = new ALS().setRank(features) .setIterations(iterations) .setImplicitPrefs(true) diff --git a/mllib/src/test/java/org/apache/spark/mllib/regression/JavaRidgeRegressionSuite.java b/mllib/src/test/java/org/apache/spark/mllib/regression/JavaRidgeRegressionSuite.java index 7266eec235..c56db703ea 100644 --- a/mllib/src/test/java/org/apache/spark/mllib/regression/JavaRidgeRegressionSuite.java +++ b/mllib/src/test/java/org/apache/spark/mllib/regression/JavaRidgeRegressionSuite.java @@ -19,14 +19,13 @@ package org.apache.spark.mllib.regression; import java.io.Serializable; import java.util.List; +import java.util.Random; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.jblas.DoubleMatrix; - import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.mllib.util.LinearDataGenerator; @@ -45,7 +44,8 @@ public class JavaRidgeRegressionSuite implements Serializable { sc = null; } - double predictionError(List<LabeledPoint> validationData, RidgeRegressionModel model) { + private static double predictionError(List<LabeledPoint> validationData, + RidgeRegressionModel model) { double errorSum = 0; for (LabeledPoint point: validationData) { Double prediction = model.predict(point.features()); @@ -54,11 +54,14 @@ public class JavaRidgeRegressionSuite implements Serializable { return errorSum / validationData.size(); } - List<LabeledPoint> generateRidgeData(int numPoints, int numFeatures, double std) { - org.jblas.util.Random.seed(42); + private static List<LabeledPoint> generateRidgeData(int numPoints, int numFeatures, double std) { // Pick weights as random values distributed uniformly in [-0.5, 0.5] - DoubleMatrix w = DoubleMatrix.rand(numFeatures, 1).subi(0.5); - return LinearDataGenerator.generateLinearInputAsList(0.0, w.data, numPoints, 42, std); + Random random = new Random(42); + double[] w = new double[numFeatures]; + for (int i = 0; i < w.length; i++) { + w[i] = random.nextDouble() - 0.5; + } + return LinearDataGenerator.generateLinearInputAsList(0.0, w, numPoints, 42, std); } @Test |