diff options
| author | Reynold Xin <rxin@apache.org> | 2014-01-08 11:47:28 -0800 |
|---|---|---|
| committer | Reynold Xin <rxin@apache.org> | 2014-01-08 11:47:28 -0800 |
| commit | 5cae05f59e766057032270a3c2bb5c99c2d64f77 (patch) | |
| tree | 9922d7b9833dca13575098e530f3e63e6e916482 /core | |
| parent | 6eef78d769379b98a1fa6d4f2fa606216e5063a6 (diff) | |
| parent | 4517326ec68d15f7d9bedade5f9bba33d760e6b4 (diff) | |
| download | spark-5cae05f59e766057032270a3c2bb5c99c2d64f77.tar.gz spark-5cae05f59e766057032270a3c2bb5c99c2d64f77.tar.bz2 spark-5cae05f59e766057032270a3c2bb5c99c2d64f77.zip | |
Merge pull request #356 from hsaputra/remove_deprecated_cleanup_method
Remove calls to deprecated mapred's OutputCommitter.cleanupJob
Since Hadoop 1.0.4 the mapred OutputCommitter.commitJob should do cleanup job via call to OutputCommitter.cleanupJob,
Remove SparkHadoopWriter.cleanup since it is used only by PairRDDFunctions.
In fact the implementation of mapred OutputCommitter.commitJob looks like this:
public void commitJob(JobContext jobContext) throws IOException {
cleanupJob(jobContext);
}
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala | 4 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 |
2 files changed, 0 insertions, 6 deletions
diff --git a/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala index 103a1c2051..618d95015f 100644 --- a/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala +++ b/core/src/main/scala/org/apache/spark/SparkHadoopWriter.scala @@ -127,10 +127,6 @@ class SparkHadoopWriter(@transient jobConf: JobConf) cmtr.commitJob(getJobContext()) } - def cleanup() { - getOutputCommitter().cleanupJob(getJobContext()) - } - // ********* Private Functions ********* private def getOutputFormat(): OutputFormat[AnyRef,AnyRef] = { diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 04a8d05988..629fb390af 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -638,7 +638,6 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) jobCommitter.setupJob(jobTaskContext) val count = self.context.runJob(self, writeShard _).sum jobCommitter.commitJob(jobTaskContext) - jobCommitter.cleanupJob(jobTaskContext) } /** @@ -728,7 +727,6 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) self.context.runJob(self, writeToFile _) writer.commitJob() - writer.cleanup() } /** |