diff options
author | Mridul Muralidharan <mridul@gmail.com> | 2013-04-22 08:01:13 +0530 |
---|---|---|
committer | Mridul Muralidharan <mridul@gmail.com> | 2013-04-22 08:01:13 +0530 |
commit | 7acab3ab45df421601ee9a076a61de00561a0308 (patch) | |
tree | ee2e9264e72bfd10e878c09765b5d580e382aca7 /core/src/hadoop2-yarn/scala | |
parent | ac2e8e8720f10efd640a67ad85270719ab2d43e9 (diff) | |
download | spark-7acab3ab45df421601ee9a076a61de00561a0308.tar.gz spark-7acab3ab45df421601ee9a076a61de00561a0308.tar.bz2 spark-7acab3ab45df421601ee9a076a61de00561a0308.zip |
Fix review comments, add a new api to SparkHadoopUtil to create appropriate Configuration. Modify an example to show how to use SplitInfo
Diffstat (limited to 'core/src/hadoop2-yarn/scala')
-rw-r--r-- | core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala | 6 |
1 files changed, 5 insertions, 1 deletions
diff --git a/core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala b/core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala index 66e5ad8491..ab1ab9d8a7 100644 --- a/core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala +++ b/core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala @@ -12,7 +12,7 @@ import java.security.PrivilegedExceptionAction */ object SparkHadoopUtil { - val yarnConf = new YarnConfiguration(new Configuration()) + val yarnConf = newConfiguration() def getUserNameFromEnvironment(): String = { // defaulting to env if -D is not present ... @@ -56,4 +56,8 @@ object SparkHadoopUtil { def setYarnMode(env: HashMap[String, String]) { env("SPARK_YARN_MODE") = "true" } + + // Return an appropriate (subclass) of Configuration. Creating config can initializes some hadoop subsystems + // Always create a new config, dont reuse yarnConf. + def newConfiguration(): Configuration = new YarnConfiguration(new Configuration()) } |