diff options
| author | Jeff Zhang <zjffdu@apache.org> | 2016-08-11 20:08:25 -0700 |
|---|---|---|
| committer | Marcelo Vanzin <vanzin@cloudera.com> | 2016-08-11 20:08:39 -0700 |
| commit | 7a9e25c38380e6c62080d62ad38a4830e44fe753 (patch) | |
| tree | c7ee1437a3dec8abc0fef57b10406a5bd0b72987 /core/src/test/scala/org/apache | |
| parent | ea0bf91b4a2ca3ef472906e50e31fd6268b6f53e (diff) | |
| download | spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.tar.gz spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.tar.bz2 spark-7a9e25c38380e6c62080d62ad38a4830e44fe753.zip | |
[SPARK-13081][PYSPARK][SPARK_SUBMIT] Allow set pythonExec of driver and executor through conf…
Before this PR, user have to export environment variable to specify the python of driver & executor which is not so convenient for users. This PR is trying to allow user to specify python through configuration "--pyspark-driver-python" & "--pyspark-executor-python"
Manually test in local & yarn mode for pyspark-shell and pyspark batch mode.
Author: Jeff Zhang <zjffdu@apache.org>
Closes #13146 from zjffdu/SPARK-13081.
Diffstat (limited to 'core/src/test/scala/org/apache')
| -rw-r--r-- | core/src/test/scala/org/apache/spark/SparkConfSuite.scala | 2 | ||||
| -rw-r--r-- | core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 5 |
2 files changed, 7 insertions, 0 deletions
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala index a883d1b57e..1f0f655a15 100644 --- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala @@ -51,8 +51,10 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst test("loading from system properties") { System.setProperty("spark.test.testProperty", "2") + System.setProperty("nonspark.test.testProperty", "0") val conf = new SparkConf() assert(conf.get("spark.test.testProperty") === "2") + assert(!conf.contains("nonspark.test.testProperty")) } test("initializing without loading defaults") { diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index b2bc886108..961ece3e00 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark._ import org.apache.spark.api.r.RUtils import org.apache.spark.deploy.SparkSubmit._ import org.apache.spark.deploy.SparkSubmitUtils.MavenCoordinate +import org.apache.spark.internal.config._ import org.apache.spark.internal.Logging import org.apache.spark.TestUtils.JavaSourceFromString import org.apache.spark.util.{ResetSystemProperties, Utils} @@ -512,6 +513,8 @@ class SparkSubmitSuite val clArgs3 = Seq( "--master", "local", "--py-files", pyFiles, + "--conf", "spark.pyspark.driver.python=python3.4", + "--conf", "spark.pyspark.python=python3.5", "mister.py" ) val appArgs3 = new SparkSubmitArguments(clArgs3) @@ -519,6 +522,8 @@ class SparkSubmitSuite appArgs3.pyFiles should be (Utils.resolveURIs(pyFiles)) sysProps3("spark.submit.pyFiles") should be ( PythonRunner.formatPaths(Utils.resolveURIs(pyFiles)).mkString(",")) + sysProps3(PYSPARK_DRIVER_PYTHON.key) should be ("python3.4") + sysProps3(PYSPARK_PYTHON.key) should be ("python3.5") } test("resolves config paths correctly") { |