diff options
author | Tathagata Das <tathagata.das1565@gmail.com> | 2013-01-07 14:55:49 -0800 |
---|---|---|
committer | Tathagata Das <tathagata.das1565@gmail.com> | 2013-01-07 14:55:49 -0800 |
commit | 3b0a3b89ac508b57b8afbd1ca7024ee558a5d1af (patch) | |
tree | 6be899c794b11f2346bf81ab0ae7304cb9363f3d /core | |
parent | 237bac36e9dca8828192994dad323b8da1619267 (diff) | |
download | spark-3b0a3b89ac508b57b8afbd1ca7024ee558a5d1af.tar.gz spark-3b0a3b89ac508b57b8afbd1ca7024ee558a5d1af.tar.bz2 spark-3b0a3b89ac508b57b8afbd1ca7024ee558a5d1af.zip |
Added better docs for RDDCheckpointData
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/spark/RDDCheckpointData.scala | 10 | ||||
-rw-r--r-- | core/src/main/scala/spark/rdd/CheckpointRDD.scala | 4 |
2 files changed, 13 insertions, 1 deletions
diff --git a/core/src/main/scala/spark/RDDCheckpointData.scala b/core/src/main/scala/spark/RDDCheckpointData.scala index e270b6312e..d845a522e4 100644 --- a/core/src/main/scala/spark/RDDCheckpointData.scala +++ b/core/src/main/scala/spark/RDDCheckpointData.scala @@ -14,15 +14,23 @@ private[spark] object CheckpointState extends Enumeration { } /** - * This class contains all the information of the regarding RDD checkpointing. + * This class contains all the information related to RDD checkpointing. Each instance of this class + * is associated with a RDD. It manages process of checkpointing of the associated RDD, as well as, + * manages the post-checkpoint state by providing the updated splits, iterator and preferred locations + * of the checkpointed RDD. */ private[spark] class RDDCheckpointData[T: ClassManifest](rdd: RDD[T]) extends Logging with Serializable { import CheckpointState._ + // The checkpoint state of the associated RDD. var cpState = Initialized + + // The file to which the associated RDD has been checkpointed to @transient var cpFile: Option[String] = None + + // The CheckpointRDD created from the checkpoint file, that is, the new parent the associated RDD. @transient var cpRDD: Option[RDD[T]] = None // Mark the RDD for checkpointing diff --git a/core/src/main/scala/spark/rdd/CheckpointRDD.scala b/core/src/main/scala/spark/rdd/CheckpointRDD.scala index 1a88d402c3..86c63ca2f4 100644 --- a/core/src/main/scala/spark/rdd/CheckpointRDD.scala +++ b/core/src/main/scala/spark/rdd/CheckpointRDD.scala @@ -13,6 +13,10 @@ private[spark] class CheckpointRDDSplit(idx: Int, val splitFile: String) extends override val index: Int = idx } +/** + * This RDD represents a RDD checkpoint file (similar to HadoopRDD). + */ +private[spark] class CheckpointRDD[T: ClassManifest](sc: SparkContext, checkpointPath: String) extends RDD[T](sc, Nil) { |