diff options
| author | Sandy Ryza <sandy@cloudera.com> | 2015-06-18 19:36:05 -0700 |
|---|---|---|
| committer | Josh Rosen <joshrosen@databricks.com> | 2015-06-18 19:36:05 -0700 |
| commit | 43f50decdd20fafc55913c56ffa30f56040090e4 (patch) | |
| tree | da3cf219841826b9d1bc6c5870994f6d1bfe7d32 /core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala | |
| parent | dc413138995b45a7a957acae007dc11622110310 (diff) | |
| download | spark-43f50decdd20fafc55913c56ffa30f56040090e4.tar.gz spark-43f50decdd20fafc55913c56ffa30f56040090e4.tar.bz2 spark-43f50decdd20fafc55913c56ffa30f56040090e4.zip | |
[SPARK-8135] Don't load defaults when reconstituting Hadoop Configurations
Author: Sandy Ryza <sandy@cloudera.com>
Closes #6679 from sryza/sandy-spark-8135 and squashes the following commits:
c5554ff [Sandy Ryza] SPARK-8135. In SerializableWritable, don't load defaults when instantiating Configuration
Diffstat (limited to 'core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala index 59ac82ccec..f5dd36cbcf 100644 --- a/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala +++ b/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.mapred.SparkHadoopMapRedUtil import org.apache.spark.rdd.HadoopRDD +import org.apache.spark.util.SerializableJobConf /** * Internal helper class that saves an RDD using a Hadoop OutputFormat. @@ -42,7 +43,7 @@ class SparkHadoopWriter(@transient jobConf: JobConf) with Serializable { private val now = new Date() - private val conf = new SerializableWritable(jobConf) + private val conf = new SerializableJobConf(jobConf) private var jobID = 0 private var splitID = 0 |