diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-02-11 13:23:50 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-02-11 13:23:50 -0800 |
commit | ea08537143d58b79b3ae5d083e9b3a5647257da8 (patch) | |
tree | ff5329ab6dc12a5326ab95f85cd1b0e8eee2b231 /core | |
parent | da8afbc77e5796d45686034db5560f18c057d3c9 (diff) | |
download | spark-ea08537143d58b79b3ae5d083e9b3a5647257da8.tar.gz spark-ea08537143d58b79b3ae5d083e9b3a5647257da8.tar.bz2 spark-ea08537143d58b79b3ae5d083e9b3a5647257da8.zip |
Fixed an exponential recursion that could happen with doCheckpoint due
to lack of memoization
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/spark/RDD.scala | 14 |
1 files changed, 10 insertions, 4 deletions
diff --git a/core/src/main/scala/spark/RDD.scala b/core/src/main/scala/spark/RDD.scala index 6abb5c4792..f6e927a989 100644 --- a/core/src/main/scala/spark/RDD.scala +++ b/core/src/main/scala/spark/RDD.scala @@ -636,16 +636,22 @@ abstract class RDD[T: ClassManifest]( /** The [[spark.SparkContext]] that this RDD was created on. */ def context = sc + // Avoid handling doCheckpoint multiple times to prevent excessive recursion + private var doCheckpointCalled = false + /** * Performs the checkpointing of this RDD by saving this. It is called by the DAGScheduler * after a job using this RDD has completed (therefore the RDD has been materialized and * potentially stored in memory). doCheckpoint() is called recursively on the parent RDDs. */ private[spark] def doCheckpoint() { - if (checkpointData.isDefined) { - checkpointData.get.doCheckpoint() - } else { - dependencies.foreach(_.rdd.doCheckpoint()) + if (!doCheckpointCalled) { + doCheckpointCalled = true + if (checkpointData.isDefined) { + checkpointData.get.doCheckpoint() + } else { + dependencies.foreach(_.rdd.doCheckpoint()) + } } } |