diff options
author | Ryan Williams <ryan.blake.williams@gmail.com> | 2015-06-03 16:54:46 -0700 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2015-06-03 16:54:46 -0700 |
commit | 51898b5158ac7e7e67b0539bc062c9c16ce9a7ce (patch) | |
tree | 0c5aa3776ca45f82686a898d617fbfc3a3d199d3 /core | |
parent | 566cb5947925c79ef90af72346672ab7d27bf4df (diff) | |
download | spark-51898b5158ac7e7e67b0539bc062c9c16ce9a7ce.tar.gz spark-51898b5158ac7e7e67b0539bc062c9c16ce9a7ce.tar.bz2 spark-51898b5158ac7e7e67b0539bc062c9c16ce9a7ce.zip |
[SPARK-8088] don't attempt to lower number of executors by 0
Author: Ryan Williams <ryan.blake.williams@gmail.com>
Closes #6624 from ryan-williams/execs and squashes the following commits:
b6f71d4 [Ryan Williams] don't attempt to lower number of executors by 0
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala | 10 |
1 files changed, 7 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala index 9514604752..f7323a4d9d 100644 --- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala +++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala @@ -266,10 +266,14 @@ private[spark] class ExecutorAllocationManager( // executors and inform the cluster manager to cancel the extra pending requests val oldNumExecutorsTarget = numExecutorsTarget numExecutorsTarget = math.max(maxNeeded, minNumExecutors) - client.requestTotalExecutors(numExecutorsTarget) numExecutorsToAdd = 1 - logInfo(s"Lowering target number of executors to $numExecutorsTarget because " + - s"not all requests are actually needed (previously $oldNumExecutorsTarget)") + + // If the new target has not changed, avoid sending a message to the cluster manager + if (numExecutorsTarget < oldNumExecutorsTarget) { + client.requestTotalExecutors(numExecutorsTarget) + logInfo(s"Lowering target number of executors to $numExecutorsTarget (previously " + + s"$oldNumExecutorsTarget) because not all requested executors are actually needed") + } numExecutorsTarget - oldNumExecutorsTarget } else if (addTime != NOT_SET && now >= addTime) { val delta = addExecutors(maxNeeded) |