diff options
author | Masayoshi TSUZUKI <tsudukim@oss.nttdata.co.jp> | 2014-12-19 19:19:53 -0800 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2014-12-19 19:22:42 -0800 |
commit | 8d932475e6759e869c16ce6cac203a2e56558716 (patch) | |
tree | aab671a7f8a14c7e9ea3b48b4e0c0fe697124a32 /bin/windows-utils.cmd | |
parent | c25c669d95293ce7f72f2758d6737fabd0d4390a (diff) | |
download | spark-8d932475e6759e869c16ce6cac203a2e56558716.tar.gz spark-8d932475e6759e869c16ce6cac203a2e56558716.tar.bz2 spark-8d932475e6759e869c16ce6cac203a2e56558716.zip |
[SPARK-3060] spark-shell.cmd doesn't accept application options in Windows OS
Added equivalent module as utils.sh and modified spark-shell2.cmd to use it to parse options.
Now we can use application options.
ex) `bin\spark-shell.cmd --master spark://master:7077 -i path\to\script.txt`
Author: Masayoshi TSUZUKI <tsudukim@oss.nttdata.co.jp>
Closes #3350 from tsudukim/feature/SPARK-3060 and squashes the following commits:
4551e56 [Masayoshi TSUZUKI] Modified too long line which defines the submission options to pass findstr command.
3a11361 [Masayoshi TSUZUKI] [SPARK-3060] spark-shell.cmd doesn't accept application options in Windows OS
Diffstat (limited to 'bin/windows-utils.cmd')
-rw-r--r-- | bin/windows-utils.cmd | 59 |
1 files changed, 59 insertions, 0 deletions
diff --git a/bin/windows-utils.cmd b/bin/windows-utils.cmd new file mode 100644 index 0000000000..1082a952da --- /dev/null +++ b/bin/windows-utils.cmd @@ -0,0 +1,59 @@ +rem +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. +rem + +rem Gather all spark-submit options into SUBMISSION_OPTS + +set SUBMISSION_OPTS= +set APPLICATION_OPTS= + +rem NOTE: If you add or remove spark-sumbmit options, +rem modify NOT ONLY this script but also SparkSubmitArgument.scala + +:OptsLoop +if "x%1"=="x" ( + goto :OptsLoopEnd +) + +SET opts="\<--master\> \<--deploy-mode\> \<--class\> \<--name\> \<--jars\> \<--py-files\> \<--files\>" +SET opts="%opts:~1,-1% \<--conf\> \<--properties-file\> \<--driver-memory\> \<--driver-java-options\>" +SET opts="%opts:~1,-1% \<--driver-library-path\> \<--driver-class-path\> \<--executor-memory\>" +SET opts="%opts:~1,-1% \<--driver-cores\> \<--total-executor-cores\> \<--executor-cores\> \<--queue\>" +SET opts="%opts:~1,-1% \<--num-executors\> \<--archives\>" + +echo %1 | findstr %opts% >nul +if %ERRORLEVEL% equ 0 ( + if "x%2"=="x" ( + echo "%1" requires an argument. >&2 + exit /b 1 + ) + set SUBMISSION_OPTS=%SUBMISSION_OPTS% %1 %2 + shift + shift + goto :OptsLoop +) +echo %1 | findstr "\<--verbose\> \<-v\> \<--supervise\>" >nul +if %ERRORLEVEL% equ 0 ( + set SUBMISSION_OPTS=%SUBMISSION_OPTS% %1 + shift + goto :OptsLoop +) +set APPLICATION_OPTS=%APPLICATION_OPTS% %1 +shift +goto :OptsLoop + +:OptsLoopEnd +exit /b 0 |