diff options
| author | Jeff Zhang <zjffdu@apache.org> | 2016-08-11 20:08:25 -0700 |
|---|---|---|
| committer | Marcelo Vanzin <vanzin@cloudera.com> | 2016-08-11 20:08:39 -0700 |
| commit | 7a9e25c38380e6c62080d62ad38a4830e44fe753 (patch) | |
| tree | c7ee1437a3dec8abc0fef57b10406a5bd0b72987 /core/src/test/java | |
| parent | ea0bf91b4a2ca3ef472906e50e31fd6268b6f53e (diff) | |
| download | spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.tar.gz spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.tar.bz2 spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.zip | |
[SPARK-13081][PYSPARK][SPARK_SUBMIT] Allow set pythonExec of driver and executor through conf…
Before this PR, user have to export environment variable to specify the python of driver & executor which is not so convenient for users. This PR is trying to allow user to specify python through configuration "--pyspark-driver-python" & "--pyspark-executor-python"
Manually test in local & yarn mode for pyspark-shell and pyspark batch mode.
Author: Jeff Zhang <zjffdu@apache.org>
Closes #13146 from zjffdu/SPARK-13081.
Diffstat (limited to 'core/src/test/java')
| -rw-r--r-- | core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java index e393db06a0..682d98867b 100644 --- a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java +++ b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java @@ -28,6 +28,8 @@ import org.slf4j.LoggerFactory; import org.slf4j.bridge.SLF4JBridgeHandler; import static org.junit.Assert.*; +import org.apache.spark.internal.config.package$; + /** * These tests require the Spark assembly to be built before they can be run. */ @@ -89,6 +91,12 @@ public class SparkLauncherSuite { launcher.setConf("spark.foo", "foo"); launcher.addSparkArg(opts.CONF, "spark.foo=bar"); assertEquals("bar", launcher.builder.conf.get("spark.foo")); + + launcher.setConf(SparkLauncher.PYSPARK_DRIVER_PYTHON, "python3.4"); + launcher.setConf(SparkLauncher.PYSPARK_PYTHON, "python3.5"); + assertEquals("python3.4", launcher.builder.conf.get( + package$.MODULE$.PYSPARK_DRIVER_PYTHON().key())); + assertEquals("python3.5", launcher.builder.conf.get(package$.MODULE$.PYSPARK_PYTHON().key())); } @Test(expected=IllegalStateException.class) |