aboutsummaryrefslogtreecommitdiff
path: root/yarn/common
diff options
context:
space:
mode:
authorandrewor14 <andrew@databricks.com>2014-10-31 14:36:55 -0700
committerAndrew Or <andrew@databricks.com>2014-10-31 14:36:55 -0700
commit087e31a7a0550f520c2e21b0ceaf36b969e050b4 (patch)
treee75bf93cb118c4c6c36e0f2453af52f16f853aca /yarn/common
parent55ab77707891408e635843cf80079747625bd28e (diff)
downloadspark-087e31a7a0550f520c2e21b0ceaf36b969e050b4.tar.gz
spark-087e31a7a0550f520c2e21b0ceaf36b969e050b4.tar.bz2
spark-087e31a7a0550f520c2e21b0ceaf36b969e050b4.zip
[HOT FIX] Yarn stable tests don't compile
This is caused by this commit: acd4ac7c9a503445e27739708cf36e19119b8ddc Author: andrewor14 <andrew@databricks.com> Author: Andrew Or <andrew@databricks.com> Closes #3041 from andrewor14/yarn-hot-fix and squashes the following commits: e5deba1 [andrewor14] Add new line at the end (minor) aa998e8 [Andrew Or] Compilation hot fix
Diffstat (limited to 'yarn/common')
-rw-r--r--yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala30
-rw-r--r--yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala4
2 files changed, 19 insertions, 15 deletions
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index e6196194ac..b32e15738f 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -64,6 +64,8 @@ private[yarn] abstract class YarnAllocator(
securityMgr: SecurityManager)
extends Logging {
+ import YarnAllocator._
+
// These three are locked on allocatedHostToContainersMap. Complementary data structures
// allocatedHostToContainersMap : containers which are running : host, Set<containerid>
// allocatedContainerToHostMap: container to host mapping.
@@ -439,19 +441,6 @@ private[yarn] abstract class YarnAllocator(
}
}
- private val MEM_REGEX = "[0-9.]+ [KMG]B"
- private val PMEM_EXCEEDED_PATTERN =
- Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used")
- private val VMEM_EXCEEDED_PATTERN =
- Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used")
-
- def memLimitExceededLogMessage(diagnostics: String, pattern: Pattern): String = {
- val matcher = pattern.matcher(diagnostics)
- val diag = if (matcher.find()) " " + matcher.group() + "." else ""
- ("Container killed by YARN for exceeding memory limits." + diag
- + " Consider boosting spark.yarn.executor.memoryOverhead.")
- }
-
protected def allocatedContainersOnHost(host: String): Int = {
var retval = 0
allocatedHostToContainersMap.synchronized {
@@ -532,3 +521,18 @@ private[yarn] abstract class YarnAllocator(
}
}
+
+private object YarnAllocator {
+ val MEM_REGEX = "[0-9.]+ [KMG]B"
+ val PMEM_EXCEEDED_PATTERN =
+ Pattern.compile(s"$MEM_REGEX of $MEM_REGEX physical memory used")
+ val VMEM_EXCEEDED_PATTERN =
+ Pattern.compile(s"$MEM_REGEX of $MEM_REGEX virtual memory used")
+
+ def memLimitExceededLogMessage(diagnostics: String, pattern: Pattern): String = {
+ val matcher = pattern.matcher(diagnostics)
+ val diag = if (matcher.find()) " " + matcher.group() + "." else ""
+ ("Container killed by YARN for exceeding memory limits." + diag
+ + " Consider boosting spark.yarn.executor.memoryOverhead.")
+ }
+}
diff --git a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
index 9fff63fb25..8d184a09d6 100644
--- a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
+++ b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/YarnAllocatorSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.deploy.yarn
-import org.apache.spark.deploy.yarn.MemLimitLogger._
+import org.apache.spark.deploy.yarn.YarnAllocator._
import org.scalatest.FunSuite
class YarnAllocatorSuite extends FunSuite {
@@ -31,4 +31,4 @@ class YarnAllocatorSuite extends FunSuite {
assert(vmemMsg.contains("5.8 GB of 4.2 GB virtual memory used."))
assert(pmemMsg.contains("2.1 MB of 2 GB physical memory used."))
}
-} \ No newline at end of file
+}