diff options
| author | Josh Rosen <joshrosen@eecs.berkeley.edu> | 2012-12-29 16:01:03 -0800 |
|---|---|---|
| committer | Josh Rosen <joshrosen@eecs.berkeley.edu> | 2012-12-29 16:01:03 -0800 |
| commit | 59195c68ec37acf20d527189ed757397b273a207 (patch) | |
| tree | 33d583fe91325fbc2bb94b27ba4ef3503e7fdda3 /pyspark | |
| parent | c5cee53f2092ee2825095a1831ca47f1c41afc2f (diff) | |
| download | spark-59195c68ec37acf20d527189ed757397b273a207.tar.gz spark-59195c68ec37acf20d527189ed757397b273a207.tar.bz2 spark-59195c68ec37acf20d527189ed757397b273a207.zip | |
Update PySpark for compatibility with TaskContext.
Diffstat (limited to 'pyspark')
| -rw-r--r-- | pyspark/pyspark/rdd.py | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/pyspark/pyspark/rdd.py b/pyspark/pyspark/rdd.py index 203f7377d2..21dda31c4e 100644 --- a/pyspark/pyspark/rdd.py +++ b/pyspark/pyspark/rdd.py @@ -335,9 +335,10 @@ class RDD(object): """ items = [] splits = self._jrdd.splits() + taskContext = self.ctx.jvm.spark.TaskContext(0, 0, 0) while len(items) < num and splits: split = splits.pop(0) - iterator = self._jrdd.iterator(split) + iterator = self._jrdd.iterator(split, taskContext) items.extend(self._collect_iterator_through_file(iterator)) return items[:num] |