diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-02-03 21:36:36 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-02-03 21:36:36 -0800 |
commit | f7b4e428be75d189b9ae50c4302c08f3c49e0161 (patch) | |
tree | 4a8d5ffa8bf1d8094295653c0a2fe8ccaa65558b /core/src | |
parent | 3bfaf3ab1d14a85a749f87f1bcd37e553e8440e7 (diff) | |
parent | e61729113d3bf165d1ab9bd83ea55d52fd0bb72e (diff) | |
download | spark-f7b4e428be75d189b9ae50c4302c08f3c49e0161.tar.gz spark-f7b4e428be75d189b9ae50c4302c08f3c49e0161.tar.bz2 spark-f7b4e428be75d189b9ae50c4302c08f3c49e0161.zip |
Merge pull request #445 from JoshRosen/pyspark_fixes
Fix exit status in PySpark unit tests; fix/optimize PySpark's RDD.take()
Diffstat (limited to 'core/src')
-rw-r--r-- | core/src/main/scala/spark/api/python/PythonRDD.scala | 11 |
1 files changed, 9 insertions, 2 deletions
diff --git a/core/src/main/scala/spark/api/python/PythonRDD.scala b/core/src/main/scala/spark/api/python/PythonRDD.scala index 39758e94f4..ab8351e55e 100644 --- a/core/src/main/scala/spark/api/python/PythonRDD.scala +++ b/core/src/main/scala/spark/api/python/PythonRDD.scala @@ -238,6 +238,11 @@ private[spark] object PythonRDD { } def writeIteratorToPickleFile[T](items: java.util.Iterator[T], filename: String) { + import scala.collection.JavaConverters._ + writeIteratorToPickleFile(items.asScala, filename) + } + + def writeIteratorToPickleFile[T](items: Iterator[T], filename: String) { val file = new DataOutputStream(new FileOutputStream(filename)) for (item <- items) { writeAsPickle(item, file) @@ -245,8 +250,10 @@ private[spark] object PythonRDD { file.close() } - def takePartition[T](rdd: RDD[T], partition: Int): java.util.Iterator[T] = - rdd.context.runJob(rdd, ((x: Iterator[T]) => x), Seq(partition), true).head + def takePartition[T](rdd: RDD[T], partition: Int): Iterator[T] = { + implicit val cm : ClassManifest[T] = rdd.elementClassManifest + rdd.context.runJob(rdd, ((x: Iterator[T]) => x.toArray), Seq(partition), true).head.iterator + } } private object Pickle { |