diff options
author | Saurabh Rawat <sr.eklavya@gmail.com> | 2014-01-13 23:40:04 +0530 |
---|---|---|
committer | Saurabh Rawat <sr.eklavya@gmail.com> | 2014-01-13 23:40:04 +0530 |
commit | e92297337387b435c9e46f56aa1a403b78647afe (patch) | |
tree | 8e79fb781cb5f044706e1e15007e3a9b0b743a79 /core/src | |
parent | fa42951e3bbd5af2d65d11e91101a775a2bd3c3f (diff) | |
download | spark-e92297337387b435c9e46f56aa1a403b78647afe.tar.gz spark-e92297337387b435c9e46f56aa1a403b78647afe.tar.bz2 spark-e92297337387b435c9e46f56aa1a403b78647afe.zip |
Modifications as suggested in PR feedback-
- mapPartitions, foreachPartition moved to JavaRDDLike
- call scala rdd's setGenerator instead of setting directly in JavaRDD
Diffstat (limited to 'core/src')
-rw-r--r-- | core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala | 18 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala | 15 |
2 files changed, 16 insertions, 17 deletions
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala index e7b1ceebf6..e687bbdd99 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala @@ -138,24 +138,8 @@ JavaRDDLike[T, JavaRDD[T]] { /** Reset generator*/ def setGenerator(_generator: String) = { - rdd.generator = _generator + rdd.setGenerator(_generator) } - - /** - * Return a new RDD by applying a function to each partition of this RDD. - */ - def mapPartitions[U]( - f: JFMap[JIterator[T], U], preservesPartitioning: Boolean): JavaRDD[U] = { - rdd.mapPartitions[U]((x => f(asJavaIterator(x)).iterator), preservesPartitioning) - } - - /** - * Applies a function f to each partition of this RDD. - */ - def foreachPartition(f: VoidFunction[JIterator[T]]) { - rdd.foreachPartition((x => f(asJavaIterator(x)))) - } - } object JavaRDD { diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala index ebbbbd8806..eb8e34e240 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala @@ -149,6 +149,21 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends Serializable { } /** + * Return a new RDD by applying a function to each partition of this RDD. + */ + def mapPartitions[U]( + f: FlatMapFunction[java.util.Iterator[T], U], preservesPartitioning: Boolean): JavaRDD[U] = { + rdd.mapPartitions[U]((x => f(asJavaIterator(x)).iterator), preservesPartitioning) + } + + /** + * Applies a function f to each partition of this RDD. + */ + def foreachPartition(f: VoidFunction[java.util.Iterator[T]]) { + rdd.foreachPartition((x => f(asJavaIterator(x)))) + } + + /** * Return an RDD created by coalescing all elements within each partition into an array. */ def glom(): JavaRDD[JList[T]] = |