diff options
author | Christopher Nguyen <ctn@adatao.com> | 2013-06-15 17:35:55 -0700 |
---|---|---|
committer | Christopher Nguyen <ctn@adatao.com> | 2013-06-15 17:35:55 -0700 |
commit | 479442a9b913b08a64da4bd5848111d950105336 (patch) | |
tree | 6dfdd65e213c3c9107f7850050c0c784e455b7e0 /core | |
parent | 9d359043574f6801ba15ec9d016eba0f00ac2349 (diff) | |
download | spark-479442a9b913b08a64da4bd5848111d950105336.tar.gz spark-479442a9b913b08a64da4bd5848111d950105336.tar.bz2 spark-479442a9b913b08a64da4bd5848111d950105336.zip |
Add zeroLengthPartitions() test to make sure, e.g., StatCounter.scala can handle empty partitions without incorrectly returning NaN
Diffstat (limited to 'core')
-rw-r--r-- | core/src/test/scala/spark/JavaAPISuite.java | 22 |
1 files changed, 22 insertions, 0 deletions
diff --git a/core/src/test/scala/spark/JavaAPISuite.java b/core/src/test/scala/spark/JavaAPISuite.java index 93bb69b41c..3190a43e73 100644 --- a/core/src/test/scala/spark/JavaAPISuite.java +++ b/core/src/test/scala/spark/JavaAPISuite.java @@ -315,6 +315,28 @@ public class JavaAPISuite implements Serializable { } @Test + public void zeroLengthPartitions() { + // Create RDD with some consecutive empty partitions (including the "first" one) + JavaDoubleRDD rdd = sc + .parallelizeDoubles(Arrays.asList(-1.0, -1.0, -1.0, -1.0, 2.0, 4.0, -1.0, -1.0), 8) + .filter(new Function<Double, Boolean>() { + @Override + public Boolean call(Double x) { + return x > 0.0; + } + }); + + // Run the partitions, including the consecutive empty ones, through StatCounter + StatCounter stats = rdd.stats(); + Assert.assertEquals(6.0, stats.sum(), 0.01); + Assert.assertEquals(6.0/2, rdd.mean(), 0.01); + Assert.assertEquals(1.0, rdd.variance(), 0.01); + Assert.assertEquals(1.0, rdd.stdev(), 0.01); + + // Add other tests here for classes that should be able to handle empty partitions correctly + } + + @Test public void map() { JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5)); JavaDoubleRDD doubles = rdd.map(new DoubleFunction<Integer>() { |