diff options
| author | root <root@domU-12-31-39-15-18-A2.compute-1.internal> | 2010-11-04 21:34:55 +0000 |
|---|---|---|
| committer | root <root@domU-12-31-39-15-18-A2.compute-1.internal> | 2010-11-04 21:34:55 +0000 |
| commit | 4cc0984b43095aaebcb565bcb0ac0a71b6cef7ca (patch) | |
| tree | 855d6fc86ccead0f87e156f76eb8b10918b1d555 | |
| parent | 96f0be935ab2be424e56e5cfa2dfae117ec2a2b9 (diff) | |
| download | spark-4cc0984b43095aaebcb565bcb0ac0a71b6cef7ca.tar.gz spark-4cc0984b43095aaebcb565bcb0ac0a71b6cef7ca.tar.bz2 spark-4cc0984b43095aaebcb565bcb0ac0a71b6cef7ca.zip | |
Fixed a small bug in DFS shuffle -- the number of reduce tasks was not being set based on numOutputSplits
| -rw-r--r-- | src/scala/spark/DfsShuffle.scala | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/src/scala/spark/DfsShuffle.scala b/src/scala/spark/DfsShuffle.scala index 10f77a824a..256bf4ea9c 100644 --- a/src/scala/spark/DfsShuffle.scala +++ b/src/scala/spark/DfsShuffle.scala @@ -61,7 +61,8 @@ extends Logging }) // Return an RDD that does each of the merges for a given partition - return sc.parallelize(0 until numOutputSplits).flatMap((myIndex: Int) => { + val indexes = sc.parallelize(0 until numOutputSplits, numOutputSplits) + return indexes.flatMap((myIndex: Int) => { val combiners = new HashMap[K, C] val fs = DfsShuffle.getFileSystem() for (i <- Utils.shuffle(0 until numInputSplits)) { |