diff options
| author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-07-06 15:26:19 -0700 |
|---|---|---|
| committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-07-06 15:26:19 -0700 |
| commit | 94871e4703d8566359a405da58c60f411b14857a (patch) | |
| tree | 072691bb7b2c8f93221d786b5cfaf80de9533fea /streaming | |
| parent | 3f918b33f8e9cdb3e56f9ee7c88bc760ac9848bb (diff) | |
| parent | 923cf929003c67963e273fcdcd5b01baf68df8b5 (diff) | |
| download | spark-94871e4703d8566359a405da58c60f411b14857a.tar.gz spark-94871e4703d8566359a405da58c60f411b14857a.tar.bz2 spark-94871e4703d8566359a405da58c60f411b14857a.zip | |
Merge pull request #655 from tgravescs/master
Add support for running Spark on Yarn on a secure Hadoop Cluster
Diffstat (limited to 'streaming')
| -rw-r--r-- | streaming/src/main/scala/spark/streaming/PairDStreamFunctions.scala | 3 |
1 files changed, 2 insertions, 1 deletions
diff --git a/streaming/src/main/scala/spark/streaming/PairDStreamFunctions.scala b/streaming/src/main/scala/spark/streaming/PairDStreamFunctions.scala index 3ec922957d..8d0a83d439 100644 --- a/streaming/src/main/scala/spark/streaming/PairDStreamFunctions.scala +++ b/streaming/src/main/scala/spark/streaming/PairDStreamFunctions.scala @@ -14,6 +14,7 @@ import scala.collection.mutable.ArrayBuffer import org.apache.hadoop.mapred.{JobConf, OutputFormat} import org.apache.hadoop.mapreduce.{OutputFormat => NewOutputFormat} import org.apache.hadoop.mapred.OutputFormat +import org.apache.hadoop.security.UserGroupInformation import org.apache.hadoop.conf.Configuration class PairDStreamFunctions[K: ClassManifest, V: ClassManifest](self: DStream[(K,V)]) @@ -470,7 +471,7 @@ extends Serializable { valueClass: Class[_], outputFormatClass: Class[_ <: OutputFormat[_, _]], conf: JobConf = new JobConf - ) { + ) { val saveFunc = (rdd: RDD[(K, V)], time: Time) => { val file = rddToFileName(prefix, suffix, time) rdd.saveAsHadoopFile(file, keyClass, valueClass, outputFormatClass, conf) |