diff options
author | Sun Rui <rui.sun@intel.com> | 2015-09-30 11:03:08 -0700 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2015-09-30 11:03:08 -0700 |
commit | c7b29ae6418368a1266b960ba8776317fd867313 (patch) | |
tree | 9ef9f4ab26bc3683d10ed5fad71e621265453849 /core/src | |
parent | 16fd2a2f4225771af43403f20fbc79b6c914ee4d (diff) | |
download | spark-c7b29ae6418368a1266b960ba8776317fd867313.tar.gz spark-c7b29ae6418368a1266b960ba8776317fd867313.tar.bz2 spark-c7b29ae6418368a1266b960ba8776317fd867313.zip |
[SPARK-10851] [SPARKR] Exception not failing R applications (in yarn cluster mode)
The YARN backend doesn't like when user code calls System.exit, since it cannot know the exit status and thus cannot set an appropriate final status for the application.
This PR remove the usage of system.exit to exit the RRunner. Instead, when the R process running an SparkR script returns an exit code other than 0, throws SparkUserAppException which will be caught by ApplicationMaster and ApplicationMaster knows it failed. For other failures, throws SparkException.
Author: Sun Rui <rui.sun@intel.com>
Closes #8938 from sun-rui/SPARK-10851.
Diffstat (limited to 'core/src')
-rw-r--r-- | core/src/main/scala/org/apache/spark/deploy/RRunner.scala | 10 |
1 files changed, 7 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/deploy/RRunner.scala b/core/src/main/scala/org/apache/spark/deploy/RRunner.scala index 05b954ce36..58cc1f9d96 100644 --- a/core/src/main/scala/org/apache/spark/deploy/RRunner.scala +++ b/core/src/main/scala/org/apache/spark/deploy/RRunner.scala @@ -25,6 +25,7 @@ import scala.collection.JavaConverters._ import org.apache.hadoop.fs.Path import org.apache.spark.api.r.{RBackend, RUtils} +import org.apache.spark.{SparkException, SparkUserAppException} import org.apache.spark.util.RedirectThread /** @@ -84,12 +85,15 @@ object RRunner { } finally { sparkRBackend.close() } - System.exit(returnCode) + if (returnCode != 0) { + throw new SparkUserAppException(returnCode) + } } else { + val errorMessage = s"SparkR backend did not initialize in $backendTimeout seconds" // scalastyle:off println - System.err.println("SparkR backend did not initialize in " + backendTimeout + " seconds") + System.err.println(errorMessage) // scalastyle:on println - System.exit(-1) + throw new SparkException(errorMessage) } } } |