diff options
author | Charles Reiss <charles@eecs.berkeley.edu> | 2013-02-22 15:23:58 -0800 |
---|---|---|
committer | Charles Reiss <charles@eecs.berkeley.edu> | 2013-02-22 16:11:53 -0800 |
commit | 50cf8c8b79222e2b56dc5c28992adb08bb9c602b (patch) | |
tree | 976bbfed2ae8cf347736ad3d9b3077b9da46d466 /core/src/test | |
parent | c8a788692185326c001233bb249d2ed046cd7319 (diff) | |
download | spark-50cf8c8b79222e2b56dc5c28992adb08bb9c602b.tar.gz spark-50cf8c8b79222e2b56dc5c28992adb08bb9c602b.tar.bz2 spark-50cf8c8b79222e2b56dc5c28992adb08bb9c602b.zip |
Add fault tolerance test that uses replicated RDDs.
Diffstat (limited to 'core/src/test')
-rw-r--r-- | core/src/test/scala/spark/DistributedSuite.scala | 21 |
1 files changed, 21 insertions, 0 deletions
diff --git a/core/src/test/scala/spark/DistributedSuite.scala b/core/src/test/scala/spark/DistributedSuite.scala index 0e2585daa4..caa4ba3a37 100644 --- a/core/src/test/scala/spark/DistributedSuite.scala +++ b/core/src/test/scala/spark/DistributedSuite.scala @@ -217,6 +217,27 @@ class DistributedSuite extends FunSuite with ShouldMatchers with BeforeAndAfter assert(grouped.collect.size === 1) } } + + test("recover from node failures with replication") { + import DistributedSuite.{markNodeIfIdentity, failOnMarkedIdentity} + DistributedSuite.amMaster = true + // Using more than two nodes so we don't have a symmetric communication pattern and might + // cache a partially correct list of peers. + sc = new SparkContext("local-cluster[3,1,512]", "test") + for (i <- 1 to 3) { + val data = sc.parallelize(Seq(true, false, false, false), 4) + data.persist(StorageLevel.MEMORY_ONLY_2) + + assert(data.count === 4) + assert(data.map(markNodeIfIdentity).collect.size === 4) + assert(data.map(failOnMarkedIdentity).collect.size === 4) + + // Create a new replicated RDD to make sure that cached peer information doesn't cause + // problems. + val data2 = sc.parallelize(Seq(true, true), 2).persist(StorageLevel.MEMORY_ONLY_2) + assert(data2.count === 2) + } + } } object DistributedSuite { |