diff options
author | Daoyuan <daoyuan.wang@intel.com> | 2014-07-24 00:09:36 -0700 |
---|---|---|
committer | Matei Zaharia <matei@databricks.com> | 2014-07-24 00:09:36 -0700 |
commit | 42dfab7d374cf64a39b692ebc089792a4ff7e42c (patch) | |
tree | a0771d5b120a2785eb299b3b34dccf3fe57527b6 /bagel | |
parent | e34922a221738bae1195d8ace90369c9ddc3a48d (diff) | |
download | spark-42dfab7d374cf64a39b692ebc089792a4ff7e42c.tar.gz spark-42dfab7d374cf64a39b692ebc089792a4ff7e42c.tar.bz2 spark-42dfab7d374cf64a39b692ebc089792a4ff7e42c.zip |
[SPARK-2661][bagel]unpersist old processed rdd
Unpersist useless rdd during bagel iteration to make full use of memory.
Author: Daoyuan <daoyuan.wang@intel.com>
Closes #1519 from adrian-wang/bagelunpersist and squashes the following commits:
182c9dd [Daoyuan] rename var nextUseless to lastRDD
87fd3a4 [Daoyuan] bagel unpersist old processed rdd
Diffstat (limited to 'bagel')
-rw-r--r-- | bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala b/bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala index 70a99b33d7..ef0bb2ac13 100644 --- a/bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala +++ b/bagel/src/main/scala/org/apache/spark/bagel/Bagel.scala @@ -72,6 +72,7 @@ object Bagel extends Logging { var verts = vertices var msgs = messages var noActivity = false + var lastRDD: RDD[(K, (V, Array[M]))] = null do { logInfo("Starting superstep " + superstep + ".") val startTime = System.currentTimeMillis @@ -83,6 +84,10 @@ object Bagel extends Logging { val superstep_ = superstep // Create a read-only copy of superstep for capture in closure val (processed, numMsgs, numActiveVerts) = comp[K, V, M, C](sc, grouped, compute(_, _, aggregated, superstep_), storageLevel) + if (lastRDD != null) { + lastRDD.unpersist(false) + } + lastRDD = processed val timeTaken = System.currentTimeMillis - startTime logInfo("Superstep %d took %d s".format(superstep, timeTaken / 1000)) |