diff options
| author | Thomas Graves <tgraves@apache.org> | 2014-03-19 08:09:20 -0500 |
|---|---|---|
| committer | Thomas Graves <tgraves@apache.org> | 2014-03-19 08:09:20 -0500 |
| commit | 6112270c94f1a30a461a91f6e56485a5eaec2606 (patch) | |
| tree | b4fe4f39e93d17f0b9791aba297b229984c25423 | |
| parent | d55ec86de2e96f7dc9d1dd107daa35c3823791ec (diff) | |
| download | spark-6112270c94f1a30a461a91f6e56485a5eaec2606.tar.gz spark-6112270c94f1a30a461a91f6e56485a5eaec2606.tar.bz2 spark-6112270c94f1a30a461a91f6e56485a5eaec2606.zip | |
SPARK-1203 fix saving to hdfs from yarn
Author: Thomas Graves <tgraves@apache.org>
Closes #173 from tgravescs/SPARK-1203 and squashes the following commits:
4fd5ded [Thomas Graves] adding import
964e3f7 [Thomas Graves] SPARK-1203 fix saving to hdfs from yarn
| -rw-r--r-- | core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala index 447deafff5..75fc02acd1 100644 --- a/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala +++ b/core/src/main/scala/org/apache/spark/rdd/PairRDDFunctions.scala @@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.lib.output.{FileOutputFormat => NewFileOutput import org.apache.hadoop.mapred.SparkHadoopWriter import org.apache.spark._ +import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.Partitioner.defaultPartitioner import org.apache.spark.SparkContext._ import org.apache.spark.partial.{BoundedDouble, PartialResult} @@ -723,6 +724,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) if (valueClass == null) { throw new SparkException("Output value class not set") } + SparkHadoopUtil.get.addCredentials(conf) logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", " + valueClass.getSimpleName + ")") |