aboutsummaryrefslogtreecommitdiff
path: root/sql/core/src/main/java
diff options
context:
space:
mode:
authorYin Huai <yhuai@databricks.com>2015-08-05 19:19:09 -0700
committerYin Huai <yhuai@databricks.com>2015-08-05 19:19:09 -0700
commit4581badbc8aa7e5a37ba7f7f83cc3860240f5dd3 (patch)
tree7a67455f97f5915aa1f47a0efadd33f812dede1e /sql/core/src/main/java
parent4399b7b0903d830313ab7e69731c11d587ae567c (diff)
downloadspark-4581badbc8aa7e5a37ba7f7f83cc3860240f5dd3.tar.gz
spark-4581badbc8aa7e5a37ba7f7f83cc3860240f5dd3.tar.bz2
spark-4581badbc8aa7e5a37ba7f7f83cc3860240f5dd3.zip
[SPARK-9611] [SQL] Fixes a few corner cases when we spill a UnsafeFixedWidthAggregationMap
This PR has the following three small fixes. 1. UnsafeKVExternalSorter does not use 0 as the initialSize to create an UnsafeInMemorySorter if its BytesToBytesMap is empty. 2. We will not not spill a InMemorySorter if it is empty. 3. We will not add a SpillReader to a SpillMerger if this SpillReader is empty. JIRA: https://issues.apache.org/jira/browse/SPARK-9611 Author: Yin Huai <yhuai@databricks.com> Closes #7948 from yhuai/unsafeEmptyMap and squashes the following commits: 9727abe [Yin Huai] Address Josh's comments. 34b6f76 [Yin Huai] 1. UnsafeKVExternalSorter does not use 0 as the initialSize to create an UnsafeInMemorySorter if its BytesToBytesMap is empty. 2. Do not spill a InMemorySorter if it is empty. 3. Do not add spill to SpillMerger if this spill is empty.
Diffstat (limited to 'sql/core/src/main/java')
-rw-r--r--sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java6
1 files changed, 4 insertions, 2 deletions
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
index 86a563df99..6c1cf136d9 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
@@ -82,8 +82,11 @@ public final class UnsafeKVExternalSorter {
pageSizeBytes);
} else {
// Insert the records into the in-memory sorter.
+ // We will use the number of elements in the map as the initialSize of the
+ // UnsafeInMemorySorter. Because UnsafeInMemorySorter does not accept 0 as the initialSize,
+ // we will use 1 as its initial size if the map is empty.
final UnsafeInMemorySorter inMemSorter = new UnsafeInMemorySorter(
- taskMemoryManager, recordComparator, prefixComparator, map.numElements());
+ taskMemoryManager, recordComparator, prefixComparator, Math.max(1, map.numElements()));
final int numKeyFields = keySchema.size();
BytesToBytesMap.BytesToBytesMapIterator iter = map.iterator();
@@ -214,7 +217,6 @@ public final class UnsafeKVExternalSorter {
// Note that recordLen = keyLen + valueLen + 4 bytes (for the keyLen itself)
int keyLen = PlatformDependent.UNSAFE.getInt(baseObj, recordOffset);
int valueLen = recordLen - keyLen - 4;
-
key.pointTo(baseObj, recordOffset + 4, numKeyFields, keyLen);
value.pointTo(baseObj, recordOffset + 4 + keyLen, numValueFields, valueLen);