diff options
author | andrewor14 <andrew@databricks.com> | 2014-10-31 14:36:55 -0700 |
---|---|---|
committer | Andrew Or <andrew@databricks.com> | 2014-10-31 14:36:55 -0700 |
commit | 087e31a7a0550f520c2e21b0ceaf36b969e050b4 (patch) | |
tree | e75bf93cb118c4c6c36e0f2453af52f16f853aca /yarn/common/src/main | |
parent | 55ab77707891408e635843cf80079747625bd28e (diff) | |
download | spark-087e31a7a0550f520c2e21b0ceaf36b969e050b4.tar.gz spark-087e31a7a0550f520c2e21b0ceaf36b969e050b4.tar.bz2 spark-087e31a7a0550f520c2e21b0ceaf36b969e050b4.zip |
[HOT FIX] Yarn stable tests don't compile
This is caused by this commit: acd4ac7c9a503445e27739708cf36e19119b8ddc
Author: andrewor14 <andrew@databricks.com>
Author: Andrew Or <andrew@databricks.com>
Closes #3041 from andrewor14/yarn-hot-fix and squashes the following commits:
e5deba1 [andrewor14] Add new line at the end (minor)
aa998e8 [Andrew Or] Compilation hot fix
Diffstat (limited to 'yarn/common/src/main')
-rw-r--r-- | yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 30 |
1 files changed, 17 insertions, 13 deletions
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index e6196194ac..b32e15738f 100644 --- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -64,6 +64,8 @@ private[yarn] abstract class YarnAllocator( securityMgr: SecurityManager) extends Logging { + import YarnAllocator._ + // These three are locked on allocatedHostToContainersMap. Complementary data structures // allocatedHostToContainersMap : containers which are running : host, Set<containerid> // allocatedContainerToHostMap: container to host mapping. @@ -439,19 +441,6 @@ private[yarn] abstract class YarnAllocator( } } - private val MEM_REGEX = "[0-9.]+ [KMG]B" - private val PMEM_EXCEEDED_PATTERN = - Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used") - private val VMEM_EXCEEDED_PATTERN = - Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used") - - def memLimitExceededLogMessage(diagnostics: String, pattern: Pattern): String = { - val matcher = pattern.matcher(diagnostics) - val diag = if (matcher.find()) " " + matcher.group() + "." else "" - ("Container killed by YARN for exceeding memory limits." + diag - + " Consider boosting spark.yarn.executor.memoryOverhead.") - } - protected def allocatedContainersOnHost(host: String): Int = { var retval = 0 allocatedHostToContainersMap.synchronized { @@ -532,3 +521,18 @@ private[yarn] abstract class YarnAllocator( } } + +private object YarnAllocator { + val MEM_REGEX = "[0-9.]+ [KMG]B" + val PMEM_EXCEEDED_PATTERN = + Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used") + val VMEM_EXCEEDED_PATTERN = + Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used") + + def memLimitExceededLogMessage(diagnostics: String, pattern: Pattern): String = { + val matcher = pattern.matcher(diagnostics) + val diag = if (matcher.find()) " " + matcher.group() + "." else "" + ("Container killed by YARN for exceeding memory limits." + diag + + " Consider boosting spark.yarn.executor.memoryOverhead.") + } +} |