diff options
| author | Jey Kottalam <jey@cs.berkeley.edu> | 2013-07-24 14:01:48 -0700 |
|---|---|---|
| committer | Jey Kottalam <jey@cs.berkeley.edu> | 2013-08-15 16:50:37 -0700 |
| commit | e2d7656ca3d561f0a6fc8dd81ef46e4aa6ba608e (patch) | |
| tree | d8bca46dc6f9aa1d4a431c0000dd5d0c9c6b4e70 /core | |
| parent | bd0bab47c9602462628b1d3c90d5eb5d889f4596 (diff) | |
| download | spark-e2d7656ca3d561f0a6fc8dd81ef46e4aa6ba608e.tar.gz spark-e2d7656ca3d561f0a6fc8dd81ef46e4aa6ba608e.tar.bz2 spark-e2d7656ca3d561f0a6fc8dd81ef46e4aa6ba608e.zip | |
re-enable YARN support
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/spark/SparkEnv.scala | 14 |
1 files changed, 13 insertions, 1 deletions
diff --git a/core/src/main/scala/spark/SparkEnv.scala b/core/src/main/scala/spark/SparkEnv.scala index 73990f0423..5f71df33b6 100644 --- a/core/src/main/scala/spark/SparkEnv.scala +++ b/core/src/main/scala/spark/SparkEnv.scala @@ -61,9 +61,21 @@ class SparkEnv ( // If executorId is NOT found, return defaultHostPort var executorIdToHostPort: Option[(String, String) => String]) { - val hadoop = new SparkHadoopUtil private val pythonWorkers = mutable.HashMap[(String, Map[String, String]), PythonWorkerFactory]() + val hadoop = { + val yarnMode = java.lang.Boolean.valueOf(System.getProperty("SPARK_YARN_MODE", System.getenv("SPARK_YARN_MODE"))) + if(yarnMode) { + try { + Class.forName("spark.deploy.yarn.YarnSparkHadoopUtil").newInstance.asInstanceOf[SparkHadoopUtil] + } catch { + case th: Throwable => throw new SparkException("Unable to load YARN support", th) + } + } else { + new SparkHadoopUtil + } + } + def stop() { pythonWorkers.foreach { case(key, worker) => worker.stop() } httpFileServer.stop() |