diff options
| author | Devaraj K <devaraj@apache.org> | 2015-12-15 18:30:59 -0800 |
|---|---|---|
| committer | Andrew Or <andrew@databricks.com> | 2015-12-15 18:30:59 -0800 |
| commit | ca0690b5ef10b14ce57a0c30d5308eb02f163f39 (patch) | |
| tree | 6694b3d74a429bb595fe9422751ff96c87723c04 /yarn | |
| parent | a89e8b6122ee5a1517fbcf405b1686619db56696 (diff) | |
| download | spark-ca0690b5ef10b14ce57a0c30d5308eb02f163f39.tar.gz spark-ca0690b5ef10b14ce57a0c30d5308eb02f163f39.tar.bz2 spark-ca0690b5ef10b14ce57a0c30d5308eb02f163f39.zip | |
[SPARK-4117][YARN] Spark on Yarn handle AM being told command from RM
Spark on Yarn handle AM being told command from RM
When RM throws ApplicationAttemptNotFoundException for allocate
invocation, making the ApplicationMaster to finish immediately without any
retries.
Author: Devaraj K <devaraj@apache.org>
Closes #10129 from devaraj-kavali/SPARK-4117.
Diffstat (limited to 'yarn')
| -rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala | 9 |
1 files changed, 8 insertions, 1 deletions
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 1970f7d150..fc742df73d 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -376,7 +376,14 @@ private[spark] class ApplicationMaster( case i: InterruptedException => case e: Throwable => { failureCount += 1 - if (!NonFatal(e) || failureCount >= reporterMaxFailures) { + // this exception was introduced in hadoop 2.4 and this code would not compile + // with earlier versions if we refer it directly. + if ("org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException" == + e.getClass().getName()) { + logError("Exception from Reporter thread.", e) + finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE, + e.getMessage) + } else if (!NonFatal(e) || failureCount >= reporterMaxFailures) { finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE, "Exception was thrown " + s"$failureCount time(s) from Reporter thread.") |