diff options
author | Marcelo Vanzin <vanzin@cloudera.com> | 2016-05-10 10:35:54 -0700 |
---|---|---|
committer | Marcelo Vanzin <vanzin@cloudera.com> | 2016-05-10 10:35:54 -0700 |
commit | 0b9cae42426e14060bc6182c037fd715f35a2d23 (patch) | |
tree | fbf356424566ce8bd027fa034f0be6e58ce73066 /launcher/src/main | |
parent | 36c5892b4631d1d5d70e569fce12efedd82bdef0 (diff) | |
download | spark-0b9cae42426e14060bc6182c037fd715f35a2d23.tar.gz spark-0b9cae42426e14060bc6182c037fd715f35a2d23.tar.bz2 spark-0b9cae42426e14060bc6182c037fd715f35a2d23.zip |
[SPARK-11249][LAUNCHER] Throw error if app resource is not provided.
Without this, the code would build an invalid spark-submit command line,
and a more cryptic error would be presented to the user. Also, expose
a constant that allows users to set a dummy resource in cases where
they don't need an actual resource file; for backwards compatibility,
that uses the same "spark-internal" resource that Spark itself uses.
Tested via unit tests, run-example, spark-shell, and running the
thrift server with mixed spark and hive command line arguments.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #12909 from vanzin/SPARK-11249.
Diffstat (limited to 'launcher/src/main')
-rw-r--r-- | launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java | 7 | ||||
-rw-r--r-- | launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java | 29 |
2 files changed, 23 insertions, 13 deletions
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java index a083f05a2a..08873f5811 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java @@ -67,6 +67,13 @@ public class SparkLauncher { public static final String CHILD_PROCESS_LOGGER_NAME = "spark.launcher.childProcLoggerName"; /** + * A special value for the resource that tells Spark to not try to process the app resource as a + * file. This is useful when the class being executed is added to the application using other + * means - for example, by adding jars using the package download feature. + */ + public static final String NO_RESOURCE = "spark-internal"; + + /** * Maximum time (in ms) to wait for a child process to connect back to the launcher server * when using @link{#start()}. */ diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 6941ca903c..76897c4f75 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -83,9 +83,9 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { static { specialClasses.put("org.apache.spark.repl.Main", "spark-shell"); specialClasses.put("org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver", - "spark-internal"); + SparkLauncher.NO_RESOURCE); specialClasses.put("org.apache.spark.sql.hive.thriftserver.HiveThriftServer2", - "spark-internal"); + SparkLauncher.NO_RESOURCE); } final List<String> sparkArgs; @@ -112,11 +112,11 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { List<String> submitArgs = args; if (args.size() > 0 && args.get(0).equals(PYSPARK_SHELL)) { this.allowsMixedArguments = true; - appResource = PYSPARK_SHELL_RESOURCE; + appResource = PYSPARK_SHELL; submitArgs = args.subList(1, args.size()); } else if (args.size() > 0 && args.get(0).equals(SPARKR_SHELL)) { this.allowsMixedArguments = true; - appResource = SPARKR_SHELL_RESOURCE; + appResource = SPARKR_SHELL; submitArgs = args.subList(1, args.size()); } else if (args.size() > 0 && args.get(0).equals(RUN_EXAMPLE)) { isExample = true; @@ -134,9 +134,9 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { @Override public List<String> buildCommand(Map<String, String> env) throws IOException, IllegalArgumentException { - if (PYSPARK_SHELL_RESOURCE.equals(appResource) && !printInfo) { + if (PYSPARK_SHELL.equals(appResource) && !printInfo) { return buildPySparkShellCommand(env); - } else if (SPARKR_SHELL_RESOURCE.equals(appResource) && !printInfo) { + } else if (SPARKR_SHELL.equals(appResource) && !printInfo) { return buildSparkRCommand(env); } else { return buildSparkSubmitCommand(env); @@ -147,6 +147,10 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { List<String> args = new ArrayList<>(); SparkSubmitOptionParser parser = new SparkSubmitOptionParser(); + if (!allowsMixedArguments) { + checkArgument(appResource != null, "Missing application resource."); + } + if (verbose) { args.add(parser.VERBOSE); } @@ -278,6 +282,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { // When launching the pyspark shell, the spark-submit arguments should be stored in the // PYSPARK_SUBMIT_ARGS env variable. + appResource = PYSPARK_SHELL_RESOURCE; constructEnvVarArgs(env, "PYSPARK_SUBMIT_ARGS"); // The executable is the PYSPARK_DRIVER_PYTHON env variable set by the pyspark script, @@ -301,6 +306,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { } // When launching the SparkR shell, store the spark-submit arguments in the SPARKR_SUBMIT_ARGS // env variable. + appResource = SPARKR_SHELL_RESOURCE; constructEnvVarArgs(env, "SPARKR_SUBMIT_ARGS"); // Set shell.R as R_PROFILE_USER to load the SparkR package when the shell comes up. @@ -435,22 +441,19 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { className = EXAMPLE_CLASS_PREFIX + className; } mainClass = className; - appResource = "spark-internal"; + appResource = SparkLauncher.NO_RESOURCE; return false; } else { checkArgument(!opt.startsWith("-"), "Unrecognized option: %s", opt); - sparkArgs.add(opt); + checkState(appResource == null, "Found unrecognized argument but resource is already set."); + appResource = opt; return false; } } @Override protected void handleExtraArgs(List<String> extra) { - if (isExample) { - appArgs.addAll(extra); - } else { - sparkArgs.addAll(extra); - } + appArgs.addAll(extra); } } |