diff options
author | bpaulin <bob@bobpaulin.com> | 2014-07-26 10:27:09 -0700 |
---|---|---|
committer | Matei Zaharia <matei@databricks.com> | 2014-07-26 10:27:09 -0700 |
commit | c183b92c3c70ad2d36a2d60bdb10c02b65bc0212 (patch) | |
tree | e990a14d4548d0117a0ebc45a964a8188827cc12 /core | |
parent | 75663b57f90bb173f0c6c288944ec568c4719b2a (diff) | |
download | spark-c183b92c3c70ad2d36a2d60bdb10c02b65bc0212.tar.gz spark-c183b92c3c70ad2d36a2d60bdb10c02b65bc0212.tar.bz2 spark-c183b92c3c70ad2d36a2d60bdb10c02b65bc0212.zip |
[SPARK-2279] Added emptyRDD method to Java API
Added emptyRDD method to Java API with tests.
Author: bpaulin <bob@bobpaulin.com>
Closes #1597 from bobpaulin/SPARK-2279 and squashes the following commits:
5ad57c2 [bpaulin] [SPARK-2279] Added emptyRDD method to Java API
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala | 9 | ||||
-rw-r--r-- | core/src/test/java/org/apache/spark/JavaAPISuite.java | 9 |
2 files changed, 15 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala index a678355a1c..8a5f8088a0 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala @@ -34,7 +34,7 @@ import org.apache.spark._ import org.apache.spark.SparkContext.{DoubleAccumulatorParam, IntAccumulatorParam} import org.apache.spark.api.java.JavaSparkContext.fakeClassTag import org.apache.spark.broadcast.Broadcast -import org.apache.spark.rdd.RDD +import org.apache.spark.rdd.{EmptyRDD, RDD} /** * A Java-friendly version of [[org.apache.spark.SparkContext]] that returns @@ -135,6 +135,13 @@ class JavaSparkContext(val sc: SparkContext) extends JavaSparkContextVarargsWork sc.parallelize(JavaConversions.asScalaBuffer(list), numSlices) } + /** Get an RDD that has no partitions or elements. */ + def emptyRDD[T]: JavaRDD[T] = { + implicit val ctag: ClassTag[T] = fakeClassTag + JavaRDD.fromRDD(new EmptyRDD[T](sc)) + } + + /** Distribute a local Scala collection to form an RDD. */ def parallelize[T](list: java.util.List[T]): JavaRDD[T] = parallelize(list, sc.defaultParallelism) diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java index b2868b59ce..f882a8623f 100644 --- a/core/src/test/java/org/apache/spark/JavaAPISuite.java +++ b/core/src/test/java/org/apache/spark/JavaAPISuite.java @@ -118,8 +118,7 @@ public class JavaAPISuite implements Serializable { JavaRDD<Integer> intersections = s1.intersection(s2); Assert.assertEquals(3, intersections.count()); - List<Integer> list = new ArrayList<Integer>(); - JavaRDD<Integer> empty = sc.parallelize(list); + JavaRDD<Integer> empty = sc.emptyRDD(); JavaRDD<Integer> emptyIntersection = empty.intersection(s2); Assert.assertEquals(0, emptyIntersection.count()); @@ -185,6 +184,12 @@ public class JavaAPISuite implements Serializable { } @Test + public void emptyRDD() { + JavaRDD<String> rdd = sc.emptyRDD(); + Assert.assertEquals("Empty RDD shouldn't have any values", 0, rdd.count()); + } + + @Test public void sortBy() { List<Tuple2<Integer, Integer>> pairs = new ArrayList<Tuple2<Integer, Integer>>(); pairs.add(new Tuple2<Integer, Integer>(0, 4)); |