diff options
| author | Marcelo Vanzin <vanzin@cloudera.com> | 2015-03-02 16:41:43 -0600 |
|---|---|---|
| committer | Thomas Graves <tgraves@apache.org> | 2015-03-02 16:41:43 -0600 |
| commit | 6b348d90f475440c285a4b636134ffa9351580b9 (patch) | |
| tree | affd6f497680a0f8619e93b41d283cdbf49ff0c1 /yarn | |
| parent | 582e5a24c55e8c876733537c9910001affc8b29b (diff) | |
| download | spark-6b348d90f475440c285a4b636134ffa9351580b9.tar.gz spark-6b348d90f475440c285a4b636134ffa9351580b9.tar.bz2 spark-6b348d90f475440c285a4b636134ffa9351580b9.zip | |
[SPARK-6050] [yarn] Relax matching of vcore count in received containers.
Some YARN configurations return a vcore count for allocated
containers that does not match the requested resource. That means
Spark would always ignore those containers. So relax the the matching
of the vcore count to allow the Spark jobs to run.
Author: Marcelo Vanzin <vanzin@cloudera.com>
Closes #4818 from vanzin/SPARK-6050 and squashes the following commits:
991c803 [Marcelo Vanzin] Remove config option, standardize on legacy behavior (no vcore matching).
8c9c346 [Marcelo Vanzin] Restrict lax matching to vcores only.
3359692 [Marcelo Vanzin] [SPARK-6050] [yarn] Add config option to do lax resource matching.
Diffstat (limited to 'yarn')
| -rw-r--r-- | yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index 12c62a659d..55bfbcd9cb 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -290,8 +290,14 @@ private[yarn] class YarnAllocator( location: String, containersToUse: ArrayBuffer[Container], remaining: ArrayBuffer[Container]): Unit = { + // SPARK-6050: certain Yarn configurations return a virtual core count that doesn't match the + // request; for example, capacity scheduler + DefaultResourceCalculator. So match on requested + // memory, but use the asked vcore count for matching, effectively disabling matching on vcore + // count. + val matchingResource = Resource.newInstance(allocatedContainer.getResource.getMemory, + resource.getVirtualCores) val matchingRequests = amClient.getMatchingRequests(allocatedContainer.getPriority, location, - allocatedContainer.getResource) + matchingResource) // Match the allocation to a request if (!matchingRequests.isEmpty) { @@ -318,7 +324,7 @@ private[yarn] class YarnAllocator( assert(container.getResource.getMemory >= resource.getMemory) logInfo("Launching container %s for on host %s".format(containerId, executorHostname)) - executorIdToContainer(executorId) = container + executorIdToContainer(executorId) = container val containerSet = allocatedHostToContainersMap.getOrElseUpdate(executorHostname, new HashSet[ContainerId]) |