diff options
author | Yadong Qi <qiyadong2010@gmail.com> | 2015-08-09 19:54:05 +0100 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2015-08-10 15:40:01 -0700 |
commit | 51406becc78857dbc36c3330f3afe8dd5926dc2f (patch) | |
tree | af2d149af96b46af654e8749ca0182b3809c3c46 | |
parent | 0e4f58ed9b63b7eade96b8e4b2c087b182915b40 (diff) | |
download | spark-51406becc78857dbc36c3330f3afe8dd5926dc2f.tar.gz spark-51406becc78857dbc36c3330f3afe8dd5926dc2f.tar.bz2 spark-51406becc78857dbc36c3330f3afe8dd5926dc2f.zip |
[SPARK-9737] [YARN] Add the suggested configuration when required executor memory is above the max threshold of this cluster on YARN mode
Author: Yadong Qi <qiyadong2010@gmail.com>
Closes #8028 from watermen/SPARK-9737 and squashes the following commits:
48bdf3d [Yadong Qi] Add suggested configuration.
(cherry picked from commit 86fa4ba6d13f909cb508b7cb3b153d586fe59bc3)
Signed-off-by: Reynold Xin <rxin@databricks.com>
-rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index fc11bbf97e..b4ba3f0221 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -203,12 +203,14 @@ private[spark] class Client( val executorMem = args.executorMemory + executorMemoryOverhead if (executorMem > maxMem) { throw new IllegalArgumentException(s"Required executor memory (${args.executorMemory}" + - s"+$executorMemoryOverhead MB) is above the max threshold ($maxMem MB) of this cluster!") + s"+$executorMemoryOverhead MB) is above the max threshold ($maxMem MB) of this cluster! " + + "Please increase the value of 'yarn.scheduler.maximum-allocation-mb'.") } val amMem = args.amMemory + amMemoryOverhead if (amMem > maxMem) { throw new IllegalArgumentException(s"Required AM memory (${args.amMemory}" + - s"+$amMemoryOverhead MB) is above the max threshold ($maxMem MB) of this cluster!") + s"+$amMemoryOverhead MB) is above the max threshold ($maxMem MB) of this cluster! " + + "Please increase the value of 'yarn.scheduler.maximum-allocation-mb'.") } logInfo("Will allocate AM container, with %d MB memory including %d MB overhead".format( amMem, |