diff options
author | Shivaram Venkataraman <shivaram@eecs.berkeley.edu> | 2012-10-12 14:14:57 -0700 |
---|---|---|
committer | Shivaram Venkataraman <shivaram@eecs.berkeley.edu> | 2012-10-12 14:14:57 -0700 |
commit | 8577523f3766f27fa47090007276e39001e7861d (patch) | |
tree | dd40af889f6be81ec5f29e8c5a3cc83c2f13e2c6 | |
parent | 2cf40c5fd52a8f00c420bc4d342f0ad6e956ed62 (diff) | |
download | spark-8577523f3766f27fa47090007276e39001e7861d.tar.gz spark-8577523f3766f27fa47090007276e39001e7861d.tar.bz2 spark-8577523f3766f27fa47090007276e39001e7861d.zip |
Add test to verify if RDD is computed even if block manager has insufficient
memory
-rw-r--r-- | core/src/test/scala/spark/DistributedSuite.scala | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/core/src/test/scala/spark/DistributedSuite.scala b/core/src/test/scala/spark/DistributedSuite.scala index 433d2fdc19..76b0884481 100644 --- a/core/src/test/scala/spark/DistributedSuite.scala +++ b/core/src/test/scala/spark/DistributedSuite.scala @@ -27,6 +27,7 @@ class DistributedSuite extends FunSuite with ShouldMatchers with BeforeAndAfter sc = null } System.clearProperty("spark.reducer.maxMbInFlight") + System.clearProperty("spark.storage.memoryFraction") // To avoid Akka rebinding to the same port, since it doesn't unbind immediately on shutdown System.clearProperty("spark.master.port") } @@ -156,4 +157,13 @@ class DistributedSuite extends FunSuite with ShouldMatchers with BeforeAndAfter assert(data.count() === 1000) assert(data.count() === 1000) } + + test("compute without caching with low memory") { + System.setProperty("spark.storage.memoryFraction", "0.0001") + sc = new SparkContext(clusterUrl, "test") + val data = sc.parallelize(1 to 4000000, 2).persist(StorageLevel.MEMORY_ONLY) + assert(data.count() === 4000000) + assert(data.count() === 4000000) + assert(data.count() === 4000000) + } } |