diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-12-04 10:28:50 -0800 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-12-04 10:28:50 -0800 |
commit | d6e5473872f405a6f4e466705e33cf893af915c1 (patch) | |
tree | c5d8e41a0e797fa8b44965579dd3ea8b0c716d26 | |
parent | 8a3475aed66617772f4e98e9f774b109756eb391 (diff) | |
parent | 974a69d79c4bf64fd9f27b65c1c464d33e647e20 (diff) | |
download | spark-d6e5473872f405a6f4e466705e33cf893af915c1.tar.gz spark-d6e5473872f405a6f4e466705e33cf893af915c1.tar.bz2 spark-d6e5473872f405a6f4e466705e33cf893af915c1.zip |
Merge pull request #223 from rxin/transient
Mark partitioner, name, and generator field in RDD as @transient.
As part of the effort to reduce serialized task size.
-rw-r--r-- | core/src/main/scala/org/apache/spark/rdd/RDD.scala | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala index 5b1285307d..893708f8f2 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala @@ -101,7 +101,7 @@ abstract class RDD[T: ClassManifest]( protected def getPreferredLocations(split: Partition): Seq[String] = Nil /** Optionally overridden by subclasses to specify how they are partitioned. */ - val partitioner: Option[Partitioner] = None + @transient val partitioner: Option[Partitioner] = None // ======================================================================= // Methods and fields available on all RDDs @@ -114,7 +114,7 @@ abstract class RDD[T: ClassManifest]( val id: Int = sc.newRddId() /** A friendly name for this RDD */ - var name: String = null + @transient var name: String = null /** Assign a name to this RDD */ def setName(_name: String) = { @@ -123,7 +123,7 @@ abstract class RDD[T: ClassManifest]( } /** User-defined generator of this RDD*/ - var generator = Utils.getCallSiteInfo.firstUserClass + @transient var generator = Utils.getCallSiteInfo.firstUserClass /** Reset generator*/ def setGenerator(_generator: String) = { @@ -940,7 +940,7 @@ abstract class RDD[T: ClassManifest]( private var storageLevel: StorageLevel = StorageLevel.NONE /** Record user function generating this RDD. */ - private[spark] val origin = Utils.formatSparkCallSite + @transient private[spark] val origin = Utils.formatSparkCallSite private[spark] def elementClassManifest: ClassManifest[T] = classManifest[T] @@ -955,7 +955,7 @@ abstract class RDD[T: ClassManifest]( def context = sc // Avoid handling doCheckpoint multiple times to prevent excessive recursion - private var doCheckpointCalled = false + @transient private var doCheckpointCalled = false /** * Performs the checkpointing of this RDD by saving this. It is called by the DAGScheduler |