diff options
author | Josh Rosen <joshrosen@databricks.com> | 2015-08-04 14:42:11 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2015-08-04 14:42:11 -0700 |
commit | ab8ee1a3b93286a62949569615086ef5030e9fae (patch) | |
tree | 88aa364451320f2a303bbeeb4857bcba57896c84 /unsafe/src/main | |
parent | f4b1ac08a1327e6d0ddc317cdf3997a0f68dec72 (diff) | |
download | spark-ab8ee1a3b93286a62949569615086ef5030e9fae.tar.gz spark-ab8ee1a3b93286a62949569615086ef5030e9fae.tar.bz2 spark-ab8ee1a3b93286a62949569615086ef5030e9fae.zip |
[SPARK-9452] [SQL] Support records larger than page size in UnsafeExternalSorter
This patch extends UnsafeExternalSorter to support records larger than the page size. The basic strategy is the same as in #7762: store large records in their own overflow pages.
Author: Josh Rosen <joshrosen@databricks.com>
Closes #7891 from JoshRosen/large-records-in-sql-sorter and squashes the following commits:
967580b [Josh Rosen] Merge remote-tracking branch 'origin/master' into large-records-in-sql-sorter
948c344 [Josh Rosen] Add large records tests for KV sorter.
3c17288 [Josh Rosen] Combine memory and disk cleanup into general cleanupResources() method
380f217 [Josh Rosen] Merge remote-tracking branch 'origin/master' into large-records-in-sql-sorter
27eafa0 [Josh Rosen] Fix page size in PackedRecordPointerSuite
a49baef [Josh Rosen] Address initial round of review comments
3edb931 [Josh Rosen] Remove accidentally-committed debug statements.
2b164e2 [Josh Rosen] Support large records in UnsafeExternalSorter.
Diffstat (limited to 'unsafe/src/main')
-rw-r--r-- | unsafe/src/main/java/org/apache/spark/unsafe/memory/HeapMemoryAllocator.java | 3 | ||||
-rw-r--r-- | unsafe/src/main/java/org/apache/spark/unsafe/memory/UnsafeMemoryAllocator.java | 3 |
2 files changed, 6 insertions, 0 deletions
diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/memory/HeapMemoryAllocator.java b/unsafe/src/main/java/org/apache/spark/unsafe/memory/HeapMemoryAllocator.java index bbe83d36cf..6722301df1 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/memory/HeapMemoryAllocator.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/memory/HeapMemoryAllocator.java @@ -24,6 +24,9 @@ public class HeapMemoryAllocator implements MemoryAllocator { @Override public MemoryBlock allocate(long size) throws OutOfMemoryError { + if (size % 8 != 0) { + throw new IllegalArgumentException("Size " + size + " was not a multiple of 8"); + } long[] array = new long[(int) (size / 8)]; return MemoryBlock.fromLongArray(array); } diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/memory/UnsafeMemoryAllocator.java b/unsafe/src/main/java/org/apache/spark/unsafe/memory/UnsafeMemoryAllocator.java index 15898771fe..62f4459696 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/memory/UnsafeMemoryAllocator.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/memory/UnsafeMemoryAllocator.java @@ -26,6 +26,9 @@ public class UnsafeMemoryAllocator implements MemoryAllocator { @Override public MemoryBlock allocate(long size) throws OutOfMemoryError { + if (size % 8 != 0) { + throw new IllegalArgumentException("Size " + size + " was not a multiple of 8"); + } long address = PlatformDependent.UNSAFE.allocateMemory(size); return new MemoryBlock(null, address, size); } |