aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorWenchen Fan <wenchen@databricks.com>2017-04-13 08:38:24 +0800
committerWenchen Fan <wenchen@databricks.com>2017-04-13 08:38:24 +0800
commita7b430b5717e263c1fbb55114deca6028ea9c3b3 (patch)
treea1ff2e0e65b6dc9013c396196377789d3f646532
parent924c42477b5d6ed3c217c8eaaf4dc64b2379851a (diff)
downloadspark-a7b430b5717e263c1fbb55114deca6028ea9c3b3.tar.gz
spark-a7b430b5717e263c1fbb55114deca6028ea9c3b3.tar.bz2
spark-a7b430b5717e263c1fbb55114deca6028ea9c3b3.zip
[SPARK-15354][FLAKY-TEST] TopologyAwareBlockReplicationPolicyBehavior.Peers in 2 racks
## What changes were proposed in this pull request? `TopologyAwareBlockReplicationPolicyBehavior.Peers in 2 racks` is failing occasionally: https://spark-tests.appspot.com/test-details?suite_name=org.apache.spark.storage.TopologyAwareBlockReplicationPolicyBehavior&test_name=Peers+in+2+racks. This is because, when we generate 10 block manager id to test, they may all belong to the same rack, as the rack is randomly picked. This PR fixes this problem by forcing each rack to be picked at least once. ## How was this patch tested? N/A Author: Wenchen Fan <wenchen@databricks.com> Closes #17624 from cloud-fan/test.
-rw-r--r--core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala13
1 files changed, 11 insertions, 2 deletions
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala
index ecad0f5352..dfecd04c1b 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockReplicationPolicySuite.scala
@@ -70,9 +70,18 @@ class RandomBlockReplicationPolicyBehavior extends SparkFunSuite
}
}
+ /**
+ * Returns a sequence of [[BlockManagerId]], whose rack is randomly picked from the given `racks`.
+ * Note that, each rack will be picked at least once from `racks`, if `count` is greater or equal
+ * to the number of `racks`.
+ */
protected def generateBlockManagerIds(count: Int, racks: Seq[String]): Seq[BlockManagerId] = {
- (1 to count).map{i =>
- BlockManagerId(s"Exec-$i", s"Host-$i", 10000 + i, Some(racks(Random.nextInt(racks.size))))
+ val randomizedRacks: Seq[String] = Random.shuffle(
+ racks ++ racks.length.until(count).map(_ => racks(Random.nextInt(racks.length)))
+ )
+
+ (0 until count).map { i =>
+ BlockManagerId(s"Exec-$i", s"Host-$i", 10000 + i, Some(randomizedRacks(i)))
}
}
}