diff options
| author | Zhang, Liye <liye.zhang@intel.com> | 2014-08-12 23:43:36 -0700 |
|---|---|---|
| committer | Reynold Xin <rxin@apache.org> | 2014-08-12 23:43:36 -0700 |
| commit | 2bd812639c3d8c62a725fb7577365ef0816f2898 (patch) | |
| tree | e235c51bef56ff676dd218559045d193eaff400b /core | |
| parent | 246cb3f158686348a698d1c0da3001c314727129 (diff) | |
| download | spark-2bd812639c3d8c62a725fb7577365ef0816f2898.tar.gz spark-2bd812639c3d8c62a725fb7577365ef0816f2898.tar.bz2 spark-2bd812639c3d8c62a725fb7577365ef0816f2898.zip | |
[SPARK-1777 (partial)] bugfix: make size of requested memory correctly
Author: Zhang, Liye <liye.zhang@intel.com>
Closes #1892 from liyezhang556520/lazy_memory_request and squashes the following commits:
335ab61 [Zhang, Liye] [SPARK-1777 (partial)] bugfix: make size of requested memory correctly
Diffstat (limited to 'core')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/storage/MemoryStore.scala | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala b/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala index 28f675c2bb..0a09c24d61 100644 --- a/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala +++ b/core/src/main/scala/org/apache/spark/storage/MemoryStore.scala @@ -238,7 +238,7 @@ private[spark] class MemoryStore(blockManager: BlockManager, maxMemory: Long) // If our vector's size has exceeded the threshold, request more memory val currentSize = vector.estimateSize() if (currentSize >= memoryThreshold) { - val amountToRequest = (currentSize * (memoryGrowthFactor - 1)).toLong + val amountToRequest = (currentSize * memoryGrowthFactor - memoryThreshold).toLong // Hold the accounting lock, in case another thread concurrently puts a block that // takes up the unrolling space we just ensured here accountingLock.synchronized { @@ -254,7 +254,7 @@ private[spark] class MemoryStore(blockManager: BlockManager, maxMemory: Long) } } // New threshold is currentSize * memoryGrowthFactor - memoryThreshold = currentSize + amountToRequest + memoryThreshold += amountToRequest } } elementsUnrolled += 1 |