aboutsummaryrefslogtreecommitdiff
path: root/core
diff options
context:
space:
mode:
authorbpaulin <bob@bobpaulin.com>2014-07-26 10:27:09 -0700
committerMatei Zaharia <matei@databricks.com>2014-07-26 10:27:09 -0700
commitc183b92c3c70ad2d36a2d60bdb10c02b65bc0212 (patch)
treee990a14d4548d0117a0ebc45a964a8188827cc12 /core
parent75663b57f90bb173f0c6c288944ec568c4719b2a (diff)
downloadspark-c183b92c3c70ad2d36a2d60bdb10c02b65bc0212.tar.gz
spark-c183b92c3c70ad2d36a2d60bdb10c02b65bc0212.tar.bz2
spark-c183b92c3c70ad2d36a2d60bdb10c02b65bc0212.zip
[SPARK-2279] Added emptyRDD method to Java API
Added emptyRDD method to Java API with tests. Author: bpaulin <bob@bobpaulin.com> Closes #1597 from bobpaulin/SPARK-2279 and squashes the following commits: 5ad57c2 [bpaulin] [SPARK-2279] Added emptyRDD method to Java API
Diffstat (limited to 'core')
-rw-r--r--core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala9
-rw-r--r--core/src/test/java/org/apache/spark/JavaAPISuite.java9
2 files changed, 15 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index a678355a1c..8a5f8088a0 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -34,7 +34,7 @@ import org.apache.spark._
import org.apache.spark.SparkContext.{DoubleAccumulatorParam, IntAccumulatorParam}
import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.rdd.RDD
+import org.apache.spark.rdd.{EmptyRDD, RDD}
/**
* A Java-friendly version of [[org.apache.spark.SparkContext]] that returns
@@ -135,6 +135,13 @@ class JavaSparkContext(val sc: SparkContext) extends JavaSparkContextVarargsWork
sc.parallelize(JavaConversions.asScalaBuffer(list), numSlices)
}
+ /** Get an RDD that has no partitions or elements. */
+ def emptyRDD[T]: JavaRDD[T] = {
+ implicit val ctag: ClassTag[T] = fakeClassTag
+ JavaRDD.fromRDD(new EmptyRDD[T](sc))
+ }
+
+
/** Distribute a local Scala collection to form an RDD. */
def parallelize[T](list: java.util.List[T]): JavaRDD[T] =
parallelize(list, sc.defaultParallelism)
diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java
index b2868b59ce..f882a8623f 100644
--- a/core/src/test/java/org/apache/spark/JavaAPISuite.java
+++ b/core/src/test/java/org/apache/spark/JavaAPISuite.java
@@ -118,8 +118,7 @@ public class JavaAPISuite implements Serializable {
JavaRDD<Integer> intersections = s1.intersection(s2);
Assert.assertEquals(3, intersections.count());
- List<Integer> list = new ArrayList<Integer>();
- JavaRDD<Integer> empty = sc.parallelize(list);
+ JavaRDD<Integer> empty = sc.emptyRDD();
JavaRDD<Integer> emptyIntersection = empty.intersection(s2);
Assert.assertEquals(0, emptyIntersection.count());
@@ -185,6 +184,12 @@ public class JavaAPISuite implements Serializable {
}
@Test
+ public void emptyRDD() {
+ JavaRDD<String> rdd = sc.emptyRDD();
+ Assert.assertEquals("Empty RDD shouldn't have any values", 0, rdd.count());
+ }
+
+ @Test
public void sortBy() {
List<Tuple2<Integer, Integer>> pairs = new ArrayList<Tuple2<Integer, Integer>>();
pairs.add(new Tuple2<Integer, Integer>(0, 4));