diff options
| author | lianhuiwang <lianhuiwang09@gmail.com> | 2014-07-15 00:22:06 -0700 |
|---|---|---|
| committer | Reynold Xin <rxin@apache.org> | 2014-07-15 00:22:06 -0700 |
| commit | 7446f5ff93142d2dd5c79c63fa947f47a1d4db8b (patch) | |
| tree | 1bcb61349ea7abe402405b6fa05628fb9af0bd67 /core/src/main/scala | |
| parent | c7c7ac83392b10abb011e6aead1bf92e7c73695e (diff) | |
| download | spark-7446f5ff93142d2dd5c79c63fa947f47a1d4db8b.tar.gz spark-7446f5ff93142d2dd5c79c63fa947f47a1d4db8b.tar.bz2 spark-7446f5ff93142d2dd5c79c63fa947f47a1d4db8b.zip | |
discarded exceeded completedDrivers
When completedDrivers number exceeds the threshold, the first Max(spark.deploy.retainedDrivers, 1) will be discarded.
Author: lianhuiwang <lianhuiwang09@gmail.com>
Closes #1114 from lianhuiwang/retained-drivers and squashes the following commits:
8789418 [lianhuiwang] discarded exceeded completedDrivers
Diffstat (limited to 'core/src/main/scala')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/deploy/master/Master.scala | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala index d9f8105992..9fa556d522 100644 --- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala +++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala @@ -57,6 +57,7 @@ private[spark] class Master( def createDateFormat = new SimpleDateFormat("yyyyMMddHHmmss") // For application IDs val WORKER_TIMEOUT = conf.getLong("spark.worker.timeout", 60) * 1000 val RETAINED_APPLICATIONS = conf.getInt("spark.deploy.retainedApplications", 200) + val RETAINED_DRIVERS = conf.getInt("spark.deploy.retainedDrivers", 200) val REAPER_ITERATIONS = conf.getInt("spark.dead.worker.persistence", 15) val RECOVERY_DIR = conf.get("spark.deploy.recoveryDirectory", "") val RECOVERY_MODE = conf.get("spark.deploy.recoveryMode", "NONE") @@ -741,6 +742,10 @@ private[spark] class Master( case Some(driver) => logInfo(s"Removing driver: $driverId") drivers -= driver + if (completedDrivers.size >= RETAINED_DRIVERS) { + val toRemove = math.max(RETAINED_DRIVERS / 10, 1) + completedDrivers.trimStart(toRemove) + } completedDrivers += driver persistenceEngine.removeDriver(driver) driver.state = finalState |