diff options
author | Shivaram Venkataraman <shivaram@eecs.berkeley.edu> | 2013-05-30 14:36:24 -0700 |
---|---|---|
committer | Shivaram Venkataraman <shivaram@eecs.berkeley.edu> | 2013-05-30 14:36:24 -0700 |
commit | 3b0cd173430188254e068bd72890e86b864792cd (patch) | |
tree | ed052460e9f82d887b165c876a13ab01414622f1 /core/src/test/scala | |
parent | 19fd6d54c012bd9f73620e9b817f4975de162277 (diff) | |
parent | 8cb817820f134eac19985ee86cbc92f6bc2f2f4d (diff) | |
download | spark-3b0cd173430188254e068bd72890e86b864792cd.tar.gz spark-3b0cd173430188254e068bd72890e86b864792cd.tar.bz2 spark-3b0cd173430188254e068bd72890e86b864792cd.zip |
Merge branch 'master' of git://github.com/mesos/spark
Conflicts:
core/src/test/scala/spark/ShuffleSuite.scala
Diffstat (limited to 'core/src/test/scala')
-rw-r--r-- | core/src/test/scala/spark/ShuffleSuite.scala | 26 |
1 files changed, 26 insertions, 0 deletions
diff --git a/core/src/test/scala/spark/ShuffleSuite.scala b/core/src/test/scala/spark/ShuffleSuite.scala index 271f4a4e44..b967016cf7 100644 --- a/core/src/test/scala/spark/ShuffleSuite.scala +++ b/core/src/test/scala/spark/ShuffleSuite.scala @@ -341,6 +341,32 @@ class ShuffleSuite extends FunSuite with ShouldMatchers with LocalSparkContext { assert(c.count === 10) } + test("zero sized blocks") { + // Use a local cluster with 2 processes to make sure there are both local and remote blocks + sc = new SparkContext("local-cluster[2,1,512]", "test") + + // 10 partitions from 4 keys + val NUM_BLOCKS = 10 + val a = sc.parallelize(1 to 4, NUM_BLOCKS) + val b = a.map(x => (x, x*2)) + + // NOTE: The default Java serializer doesn't create zero-sized blocks. + // So, use Kryo + val c = new ShuffledRDD(b, new HashPartitioner(10), classOf[spark.KryoSerializer].getName) + + val shuffleId = c.dependencies.head.asInstanceOf[ShuffleDependency[Int, Int]].shuffleId + assert(c.count === 4) + + val blockSizes = (0 until NUM_BLOCKS).flatMap { id => + val statuses = SparkEnv.get.mapOutputTracker.getServerStatuses(shuffleId, id) + statuses.map(x => x._2) + } + val nonEmptyBlocks = blockSizes.filter(x => x > 0) + + // We should have at most 4 non-zero sized partitions + assert(nonEmptyBlocks.size <= 4) + } + } object ShuffleSuite { |