Fix review comments, add a new api to SparkHadoopUtil to create appropriate Configuration. Modify an example to show how to use SplitInfo

1 files changed, 5 insertions, 1 deletions

diff --git a/core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala b/core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala
index 66e5ad8491..ab1ab9d8a7 100644
--- a/core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/hadoop2-yarn/scala/spark/deploy/SparkHadoopUtil.scala
@@ -12,7 +12,7 @@ import java.security.PrivilegedExceptionAction
  */
 object SparkHadoopUtil {
 
-  val yarnConf = new YarnConfiguration(new Configuration())
+  val yarnConf = newConfiguration()
 
   def getUserNameFromEnvironment(): String = {
     // defaulting to env if -D is not present ...
@@ -56,4 +56,8 @@ object SparkHadoopUtil {
   def setYarnMode(env: HashMap[String, String]) {
     env("SPARK_YARN_MODE") = "true"
   }
+
+  // Return an appropriate (subclass) of Configuration. Creating config can initializes some hadoop subsystems
+  // Always create a new config, dont reuse yarnConf.
+  def newConfiguration(): Configuration = new YarnConfiguration(new Configuration())
 }