diff options
| author | Raymond Liu <raymond.liu@intel.com> | 2014-01-06 16:12:25 +0800 |
|---|---|---|
| committer | Raymond Liu <raymond.liu@intel.com> | 2014-01-07 10:24:11 +0800 |
| commit | 67af8031368c4a3558aeb46c290b7b7a09516539 (patch) | |
| tree | c592f30cfa0ba7abd610c28df0728eeead2baf48 /yarn | |
| parent | da4694a0d85433832fda1dd917fc698a08e65907 (diff) | |
| download | spark-67af8031368c4a3558aeb46c290b7b7a09516539.tar.gz spark-67af8031368c4a3558aeb46c290b7b7a09516539.tar.bz2 spark-67af8031368c4a3558aeb46c290b7b7a09516539.zip | |
Export --file for YarnClient mode to support sending extra files to worker on yarn cluster
Diffstat (limited to 'yarn')
| -rw-r--r-- | yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala index 324ef4616f..4b1b5da048 100644 --- a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala +++ b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala @@ -39,6 +39,7 @@ private[spark] class YarnClientSchedulerBackend( val defaultWorkerNumber = "1" val userJar = System.getenv("SPARK_YARN_APP_JAR") + val distFiles = System.getenv("SPARK_YARN_DIST_FILES") var workerCores = System.getenv("SPARK_WORKER_CORES") var workerMemory = System.getenv("SPARK_WORKER_MEMORY") var workerNumber = System.getenv("SPARK_WORKER_INSTANCES") @@ -64,7 +65,8 @@ private[spark] class YarnClientSchedulerBackend( "--worker-memory", workerMemory, "--worker-cores", workerCores, "--num-workers", workerNumber, - "--master-class", "org.apache.spark.deploy.yarn.WorkerLauncher" + "--master-class", "org.apache.spark.deploy.yarn.WorkerLauncher", + "--files", distFiles ) val args = new ClientArguments(argsArray, conf) |