diff options
| author | Josh Rosen <joshrosen@eecs.berkeley.edu> | 2013-01-03 14:52:21 -0800 |
|---|---|---|
| committer | Josh Rosen <joshrosen@eecs.berkeley.edu> | 2013-01-03 14:52:21 -0800 |
| commit | 33beba39656fc64984db09a82fc69ca4edcc02d4 (patch) | |
| tree | f1e84417d3d9d37c1ccf4d01d20bd4930f2ecf5d /core | |
| parent | ce9f1bbe20eff794cd1d588dc88f109d32588cfe (diff) | |
| download | spark-33beba39656fc64984db09a82fc69ca4edcc02d4.tar.gz spark-33beba39656fc64984db09a82fc69ca4edcc02d4.tar.bz2 spark-33beba39656fc64984db09a82fc69ca4edcc02d4.zip | |
Change PySpark RDD.take() to not call iterator().
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/spark/api/python/PythonRDD.scala | 4 |
1 files changed, 4 insertions, 0 deletions
diff --git a/core/src/main/scala/spark/api/python/PythonRDD.scala b/core/src/main/scala/spark/api/python/PythonRDD.scala index cf60d14f03..79d824d494 100644 --- a/core/src/main/scala/spark/api/python/PythonRDD.scala +++ b/core/src/main/scala/spark/api/python/PythonRDD.scala @@ -10,6 +10,7 @@ import spark.api.java.{JavaSparkContext, JavaPairRDD, JavaRDD} import spark.broadcast.Broadcast import spark._ import spark.rdd.PipedRDD +import java.util private[spark] class PythonRDD[T: ClassManifest]( @@ -216,6 +217,9 @@ private[spark] object PythonRDD { } file.close() } + + def takePartition[T](rdd: RDD[T], partition: Int): java.util.Iterator[T] = + rdd.context.runJob(rdd, ((x: Iterator[T]) => x), Seq(partition), true).head } private object Pickle { |