diff options
author | Jeff Zhang <zjffdu@apache.org> | 2015-10-15 09:49:19 -0700 |
---|---|---|
committer | Marcelo Vanzin <vanzin@cloudera.com> | 2015-10-15 09:50:00 -0700 |
commit | aec4400beffc569c13cceea2d0c481dfa3f34175 (patch) | |
tree | c1f47f5aa8cac4a42754c9f1862184327d56e144 /launcher/src | |
parent | 0f62c2282bb30cb4fb6eea9d28b198d557a79b22 (diff) | |
download | spark-aec4400beffc569c13cceea2d0c481dfa3f34175.tar.gz spark-aec4400beffc569c13cceea2d0c481dfa3f34175.tar.bz2 spark-aec4400beffc569c13cceea2d0c481dfa3f34175.zip |
[SPARK-11099] [SPARK SHELL] [SPARK SUBMIT] Default conf property file i…
Please help review it. Thanks
Author: Jeff Zhang <zjffdu@apache.org>
Closes #9114 from zjffdu/SPARK-11099.
Diffstat (limited to 'launcher/src')
3 files changed, 45 insertions, 18 deletions
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index cf3729b7fe..3ee6bd92e4 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -272,15 +272,11 @@ abstract class AbstractCommandBuilder { Map<String, String> getEffectiveConfig() throws IOException { if (effectiveConfig == null) { - if (propertiesFile == null) { - effectiveConfig = conf; - } else { - effectiveConfig = new HashMap<>(conf); - Properties p = loadPropertiesFile(); - for (String key : p.stringPropertyNames()) { - if (!effectiveConfig.containsKey(key)) { - effectiveConfig.put(key, p.getProperty(key)); - } + effectiveConfig = new HashMap<>(conf); + Properties p = loadPropertiesFile(); + for (String key : p.stringPropertyNames()) { + if (!effectiveConfig.containsKey(key)) { + effectiveConfig.put(key, p.getProperty(key)); } } } diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index d5397b0685..6aad47adbc 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -48,12 +48,14 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite { @Test public void testDriverCmdBuilder() throws Exception { - testCmdBuilder(true); + testCmdBuilder(true, true); + testCmdBuilder(true, false); } @Test public void testClusterCmdBuilder() throws Exception { - testCmdBuilder(false); + testCmdBuilder(false, true); + testCmdBuilder(false, false); } @Test @@ -149,7 +151,7 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite { assertEquals("arg1", cmd.get(cmd.size() - 1)); } - private void testCmdBuilder(boolean isDriver) throws Exception { + private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) throws Exception { String deployMode = isDriver ? "client" : "cluster"; SparkSubmitCommandBuilder launcher = @@ -161,14 +163,20 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite { launcher.appResource = "/foo"; launcher.appName = "MyApp"; launcher.mainClass = "my.Class"; - launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath()); launcher.appArgs.add("foo"); launcher.appArgs.add("bar"); - launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g"); - launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver"); - launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m"); - launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native"); launcher.conf.put("spark.foo", "foo"); + // either set the property through "--conf" or through default property file + if (!useDefaultPropertyFile) { + launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath()); + launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g"); + launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver"); + launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m"); + launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native"); + } else { + launcher.childEnv.put("SPARK_CONF_DIR", System.getProperty("spark.test.home") + + "/launcher/src/test/resources"); + } Map<String, String> env = new HashMap<String, String>(); List<String> cmd = launcher.buildCommand(env); @@ -216,7 +224,9 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite { } // Checks below are the same for both driver and non-driver mode. - assertEquals(dummyPropsFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE)); + if (!useDefaultPropertyFile) { + assertEquals(dummyPropsFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE)); + } assertEquals("yarn", findArgValue(cmd, parser.MASTER)); assertEquals(deployMode, findArgValue(cmd, parser.DEPLOY_MODE)); assertEquals("my.Class", findArgValue(cmd, parser.CLASS)); diff --git a/launcher/src/test/resources/spark-defaults.conf b/launcher/src/test/resources/spark-defaults.conf new file mode 100644 index 0000000000..239fc57883 --- /dev/null +++ b/launcher/src/test/resources/spark-defaults.conf @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spark.driver.memory=1g +spark.driver.extraClassPath=/driver +spark.driver.extraJavaOptions=-Ddriver -XX:MaxPermSize=256m +spark.driver.extraLibraryPath=/native
\ No newline at end of file |