diff options
| author | Henry Saputra <hsaputra@apache.org> | 2014-01-07 22:55:56 -0800 |
|---|---|---|
| committer | Henry Saputra <hsaputra@apache.org> | 2014-01-07 22:55:56 -0800 |
| commit | 4517326ec68d15f7d9bedade5f9bba33d760e6b4 (patch) | |
| tree | efa0c68273bb393785a0f3881c04bf2197b7e7b3 | |
| parent | 61674bcadfe64e0d8e05f3daab3274af3023dffa (diff) | |
| download | spark-4517326ec68d15f7d9bedade5f9bba33d760e6b4.tar.gz spark-4517326ec68d15f7d9bedade5f9bba33d760e6b4.tar.bz2 spark-4517326ec68d15f7d9bedade5f9bba33d760e6b4.zip | |
Remove calls to deprecated mapred's OutputCommitter.cleanupJob because since Hadoop 1.0.4
the mapred OutputCommitter.commitJob should do cleanup job.
In fact the implementation of mapred OutputCommitter.commitJob looks like this:
public void commitJob(JobContext jobContext) throws IOException {
cleanupJob(jobContext);
}
(The jobContext input argument is type of org.apache.hadoop.mapred.JobContext)
| -rw-r--r-- | core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala | 4 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 |
2 files changed, 0 insertions, 6 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala index 103a1c2051..618d95015f 100644 --- a/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala +++ b/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala @@ -127,10 +127,6 @@ class SparkHadoopWriter(@transient jobConf: JobConf) cmtr.commitJob(getJobContext()) } - def cleanup() { - getOutputCommitter().cleanupJob(getJobContext()) - } - // ********* Private Functions ********* private def getOutputFormat(): OutputFormat[AnyRef,AnyRef] = { diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 04a8d05988..629fb390af 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -638,7 +638,6 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) jobCommitter.setupJob(jobTaskContext) val count = self.context.runJob(self, writeShard _).sum jobCommitter.commitJob(jobTaskContext) - jobCommitter.cleanupJob(jobTaskContext) } /** @@ -728,7 +727,6 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) self.context.runJob(self, writeToFile _) writer.commitJob() - writer.cleanup() } /** |