diff options
author | Holden Karau <holden@pigscanfly.ca> | 2015-09-21 18:53:28 +0100 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2015-09-21 18:53:28 +0100 |
commit | 20a61dbd9b57957fcc5b58ef8935533914172b07 (patch) | |
tree | 6fd0c28146ce7ef52dae0d2991dc736d871ae2ac /mllib/src/test/java | |
parent | 01440395176bdbb2662480f03b27851cb860f385 (diff) | |
download | spark-20a61dbd9b57957fcc5b58ef8935533914172b07.tar.gz spark-20a61dbd9b57957fcc5b58ef8935533914172b07.tar.bz2 spark-20a61dbd9b57957fcc5b58ef8935533914172b07.zip |
[SPARK-10626] [MLLIB] create java friendly method for random rdd
SPARK-3136 added a large number of functions for creating Java RandomRDDs, but for people that want to use custom RandomDataGenerators we should make a Java friendly method.
Author: Holden Karau <holden@pigscanfly.ca>
Closes #8782 from holdenk/SPARK-10626-create-java-friendly-method-for-randomRDD.
Diffstat (limited to 'mllib/src/test/java')
-rw-r--r-- | mllib/src/test/java/org/apache/spark/mllib/random/JavaRandomRDDsSuite.java | 30 |
1 files changed, 30 insertions, 0 deletions
diff --git a/mllib/src/test/java/org/apache/spark/mllib/random/JavaRandomRDDsSuite.java b/mllib/src/test/java/org/apache/spark/mllib/random/JavaRandomRDDsSuite.java index 33d81b1e95..fce5f6712f 100644 --- a/mllib/src/test/java/org/apache/spark/mllib/random/JavaRandomRDDsSuite.java +++ b/mllib/src/test/java/org/apache/spark/mllib/random/JavaRandomRDDsSuite.java @@ -17,6 +17,7 @@ package org.apache.spark.mllib.random; +import java.io.Serializable; import java.util.Arrays; import org.apache.spark.api.java.JavaRDD; @@ -231,4 +232,33 @@ public class JavaRandomRDDsSuite { } } + @Test + public void testArbitrary() { + long size = 10; + long seed = 1L; + int numPartitions = 0; + StringGenerator gen = new StringGenerator(); + JavaRDD<String> rdd1 = randomJavaRDD(sc, gen, size); + JavaRDD<String> rdd2 = randomJavaRDD(sc, gen, size, numPartitions); + JavaRDD<String> rdd3 = randomJavaRDD(sc, gen, size, numPartitions, seed); + for (JavaRDD<String> rdd: Arrays.asList(rdd1, rdd2, rdd3)) { + Assert.assertEquals(size, rdd.count()); + Assert.assertEquals(2, rdd.first().length()); + } + } +} + +// This is just a test generator, it always returns a string of 42 +class StringGenerator implements RandomDataGenerator<String>, Serializable { + @Override + public String nextValue() { + return "42"; + } + @Override + public StringGenerator copy() { + return new StringGenerator(); + } + @Override + public void setSeed(long seed) { + } } |