diff options
| author | Marcelo Vanzin <vanzin@cloudera.com> | 2015-03-18 09:06:57 -0400 |
|---|---|---|
| committer | Sean Owen <sowen@cloudera.com> | 2015-03-18 09:06:57 -0400 |
| commit | 6205a255aea0652dddadf953771f5405065b5bec (patch) | |
| tree | 938e4f90bdf74afa09f7dc5ad44aaef6b9ce08c5 /launcher | |
| parent | 3579003115fa3217cff6aa400729d96b0c7b257b (diff) | |
| download | spark-6205a255aea0652dddadf953771f5405065b5bec.tar.gz spark-6205a255aea0652dddadf953771f5405065b5bec.tar.bz2 spark-6205a255aea0652dddadf953771f5405065b5bec.zip | |
[SPARK-6372] [core] Propagate --conf to child processes.
And add unit test.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #5057 from vanzin/SPARK-6372 and squashes the following commits:
b33728b [Marcelo Vanzin] [SPARK-6372] [core] Propagate --conf to child processes.
Diffstat (limited to 'launcher')
| -rw-r--r-- | launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java | 10 | ||||
| -rw-r--r-- | launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java | 4 |
2 files changed, 5 insertions, 9 deletions
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 6ffdff63d3..91dcf70f10 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -253,12 +253,6 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { private class OptionParser extends SparkSubmitOptionParser { - private final List<String> driverJvmKeys = Arrays.asList( - SparkLauncher.DRIVER_EXTRA_CLASSPATH, - SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, - SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, - SparkLauncher.DRIVER_MEMORY); - @Override protected boolean handle(String opt, String value) { if (opt.equals(MASTER)) { @@ -278,9 +272,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { } else if (opt.equals(CONF)) { String[] setConf = value.split("=", 2); checkArgument(setConf.length == 2, "Invalid argument to %s: %s", CONF, value); - if (driverJvmKeys.contains(setConf[0])) { - conf.put(setConf[0], setConf[1]); - } + conf.put(setConf[0], setConf[1]); } else if (opt.equals(CLASS)) { // The special classes require some special command line handling, since they allow // mixing spark-submit arguments with arguments that should be propagated to the shell diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 815edc4e49..626116a9e7 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -68,6 +68,8 @@ public class SparkSubmitCommandBuilderSuite { parser.DRIVER_JAVA_OPTIONS, "extraJavaOpt", parser.CONF, + "spark.randomOption=foo", + parser.CONF, SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH + "=/driverLibPath"); Map<String, String> env = new HashMap<String, String>(); List<String> cmd = buildCommand(sparkSubmitArgs, env); @@ -77,6 +79,8 @@ public class SparkSubmitCommandBuilderSuite { assertTrue(findInStringList(findArgValue(cmd, "-cp"), File.pathSeparator, "/driverCp")); assertTrue("Driver -Xms should be configured.", cmd.contains("-Xms42g")); assertTrue("Driver -Xmx should be configured.", cmd.contains("-Xmx42g")); + assertTrue("Command should contain user-defined conf.", + Collections.indexOfSubList(cmd, Arrays.asList(parser.CONF, "spark.randomOption=foo")) > 0); } @Test |