diff options
-rw-r--r-- | core/src/test/scala/spark/RDDSuite.scala | 11 |
1 files changed, 10 insertions, 1 deletions
diff --git a/core/src/test/scala/spark/RDDSuite.scala b/core/src/test/scala/spark/RDDSuite.scala index ac406b9447..ad91263322 100644 --- a/core/src/test/scala/spark/RDDSuite.scala +++ b/core/src/test/scala/spark/RDDSuite.scala @@ -211,7 +211,7 @@ class RDDSuite extends FunSuite with SharedSparkContext { // test that you get over 90% locality in each group val minLocality = coalesced2.partitions .map( part => part.asInstanceOf[CoalescedRDDPartition].localFraction ) - .foldLeft(100.)( (perc, loc) => math.min(perc,loc) ) + .foldLeft(1.)( (perc, loc) => math.min(perc,loc) ) assert(minLocality >= 0.90, "Expected 90% locality but got " + (minLocality*100.).toInt + "%") // test that the groups are load balanced with 100 +/- 20 elements in each @@ -219,6 +219,15 @@ class RDDSuite extends FunSuite with SharedSparkContext { .map( part => part.asInstanceOf[CoalescedRDDPartition].parents.size ) .foldLeft(0)((dev, curr) => math.max(math.abs(100-curr),dev)) assert(maxImbalance <= 20, "Expected 100 +/- 20 per partition, but got " + maxImbalance) + + // TDD: Test for later when we have implemented functionality to get locality from DAGScheduler +// val data3 = sc.makeRDD(blocks).map( i => i*2 ) +// val coalesced3 = data3.coalesce(numMachines*2) +// val minLocality2 = coalesced3.partitions +// .map( part => part.asInstanceOf[CoalescedRDDPartition].localFraction ) +// .foldLeft(1.)( (perc, loc) => math.min(perc,loc) ) +// assert(minLocality2 >= 0.90, "Expected 90% locality for derived RDD but got " + +// (minLocality2*100.).toInt + "%") } test("zipped RDDs") { |