diff options
author | Jeff Zhang <zjffdu@apache.org> | 2016-08-11 20:08:25 -0700 |
---|---|---|
committer | Marcelo Vanzin <vanzin@cloudera.com> | 2016-08-11 20:08:39 -0700 |
commit | 7a9e25c38380e6c62080d62ad38a4830e44fe753 (patch) | |
tree | c7ee1437a3dec8abc0fef57b10406a5bd0b72987 /launcher/src/main/java/org | |
parent | ea0bf91b4a2ca3ef472906e50e31fd6268b6f53e (diff) | |
download | spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.tar.gz spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.tar.bz2 spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.zip |
[SPARK-13081][PYSPARK][SPARK_SUBMIT] Allow set pythonExec of driver and executor through conf…
Before this PR, user have to export environment variable to specify the python of driver & executor which is not so convenient for users. This PR is trying to allow user to specify python through configuration "--pyspark-driver-python" & "--pyspark-executor-python"
Manually test in local & yarn mode for pyspark-shell and pyspark batch mode.
Author: Jeff Zhang <zjffdu@apache.org>
Closes #13146 from zjffdu/SPARK-13081.
Diffstat (limited to 'launcher/src/main/java/org')
-rw-r--r-- | launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java | 4 | ||||
-rw-r--r-- | launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java | 18 |
2 files changed, 19 insertions, 3 deletions
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java index 41f7f1f3ed..7b7a7bf57b 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java @@ -64,6 +64,10 @@ public class SparkLauncher { /** Configuration key for the number of executor CPU cores. */ public static final String EXECUTOR_CORES = "spark.executor.cores"; + static final String PYSPARK_DRIVER_PYTHON = "spark.pyspark.driver.python"; + + static final String PYSPARK_PYTHON = "spark.pyspark.python"; + /** Logger name to use when launching a child process. */ public static final String CHILD_PROCESS_LOGGER_NAME = "spark.launcher.childProcLoggerName"; diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index b3ccc4805f..f6da644e4c 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -294,11 +294,23 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { appResource = PYSPARK_SHELL_RESOURCE; constructEnvVarArgs(env, "PYSPARK_SUBMIT_ARGS"); - // The executable is the PYSPARK_DRIVER_PYTHON env variable set by the pyspark script, - // followed by PYSPARK_DRIVER_PYTHON_OPTS. + // Will pick up the binary executable in the following order + // 1. conf spark.pyspark.driver.python + // 2. conf spark.pyspark.python + // 3. environment variable PYSPARK_DRIVER_PYTHON + // 4. environment variable PYSPARK_PYTHON + // 5. python List<String> pyargs = new ArrayList<>(); - pyargs.add(firstNonEmpty(System.getenv("PYSPARK_DRIVER_PYTHON"), "python")); + pyargs.add(firstNonEmpty(conf.get(SparkLauncher.PYSPARK_DRIVER_PYTHON), + conf.get(SparkLauncher.PYSPARK_PYTHON), + System.getenv("PYSPARK_DRIVER_PYTHON"), + System.getenv("PYSPARK_PYTHON"), + "python")); String pyOpts = System.getenv("PYSPARK_DRIVER_PYTHON_OPTS"); + if (conf.containsKey(SparkLauncher.PYSPARK_PYTHON)) { + // pass conf spark.pyspark.python to python by environment variable. + env.put("PYSPARK_PYTHON", conf.get(SparkLauncher.PYSPARK_PYTHON)); + } if (!isEmpty(pyOpts)) { pyargs.addAll(parseOptionString(pyOpts)); } |