diff options
author | Sital Kedia <skedia@fb.com> | 2016-03-18 12:56:06 -0700 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2016-03-18 12:56:06 -0700 |
commit | 2e0c5284fd88ba89f53f93dcf1eb26bca2be49c5 (patch) | |
tree | 21a49395240364f85abdb9de6108072d03378d84 | |
parent | 3537782168aa9278ac4add1a25afac0ec6e17085 (diff) | |
download | spark-2e0c5284fd88ba89f53f93dcf1eb26bca2be49c5.tar.gz spark-2e0c5284fd88ba89f53f93dcf1eb26bca2be49c5.tar.bz2 spark-2e0c5284fd88ba89f53f93dcf1eb26bca2be49c5.zip |
[SPARK-13958] Executor OOM due to unbounded growth of pointer array in…
## What changes were proposed in this pull request?
This change fixes the executor OOM which was recently introduced in PR apache/spark#11095
(Please fill in changes proposed in this fix)
## How was this patch tested?
Tested by running a spark job on the cluster.
(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
(If this patch involves UI changes, please attach a screenshot; otherwise, remove this)
… Sorter
Author: Sital Kedia <skedia@fb.com>
Closes #11794 from sitalkedia/SPARK-13958.
-rw-r--r-- | core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java | 13 | ||||
-rw-r--r-- | core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java | 13 |
2 files changed, 24 insertions, 2 deletions
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java index 7a114df2d6..c7d89e6b09 100644 --- a/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java +++ b/core/src/main/java/org/apache/spark/shuffle/sort/ShuffleExternalSorter.java @@ -320,7 +320,18 @@ final class ShuffleExternalSorter extends MemoryConsumer { assert(inMemSorter != null); if (!inMemSorter.hasSpaceForAnotherRecord()) { long used = inMemSorter.getMemoryUsage(); - LongArray array = allocateArray(used / 8 * 2); + LongArray array; + try { + // could trigger spilling + array = allocateArray(used / 8 * 2); + } catch (OutOfMemoryError e) { + // should have trigger spilling + if (!inMemSorter.hasSpaceForAnotherRecord()) { + logger.error("Unable to grow the pointer array"); + throw e; + } + return; + } // check if spilling is triggered or not if (inMemSorter.hasSpaceForAnotherRecord()) { freeArray(array); diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java index 9236bd2c04..db8e7a759a 100644 --- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java +++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java @@ -293,7 +293,18 @@ public final class UnsafeExternalSorter extends MemoryConsumer { assert(inMemSorter != null); if (!inMemSorter.hasSpaceForAnotherRecord()) { long used = inMemSorter.getMemoryUsage(); - LongArray array = allocateArray(used / 8 * 2); + LongArray array; + try { + // could trigger spilling + array = allocateArray(used / 8 * 2); + } catch (OutOfMemoryError e) { + // should have trigger spilling + if (!inMemSorter.hasSpaceForAnotherRecord()) { + logger.error("Unable to grow the pointer array"); + throw e; + } + return; + } // check if spilling is triggered or not if (inMemSorter.hasSpaceForAnotherRecord()) { freeArray(array); |