diff options
author | Davies Liu <davies@databricks.com> | 2015-11-05 19:02:18 -0800 |
---|---|---|
committer | Josh Rosen <joshrosen@databricks.com> | 2015-11-05 19:02:18 -0800 |
commit | eec74ba8bde7f9446cc38e687bda103e85669d35 (patch) | |
tree | 5e656d6333afde0255e96d930b245df28994bf9b /unsafe/src | |
parent | 3cc2c053b5d68c747a30bd58cf388b87b1922f13 (diff) | |
download | spark-eec74ba8bde7f9446cc38e687bda103e85669d35.tar.gz spark-eec74ba8bde7f9446cc38e687bda103e85669d35.tar.bz2 spark-eec74ba8bde7f9446cc38e687bda103e85669d35.zip |
[SPARK-7542][SQL] Support off-heap index/sort buffer
This brings the support of off-heap memory for array inside BytesToBytesMap and InMemorySorter, then we could allocate all the memory from off-heap for execution.
Closes #8068
Author: Davies Liu <davies@databricks.com>
Closes #9477 from davies/unsafe_timsort.
Diffstat (limited to 'unsafe/src')
-rw-r--r-- | unsafe/src/main/java/org/apache/spark/unsafe/array/LongArray.java | 18 | ||||
-rw-r--r-- | unsafe/src/test/java/org/apache/spark/unsafe/array/LongArraySuite.java | 4 |
2 files changed, 21 insertions, 1 deletions
diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/array/LongArray.java b/unsafe/src/main/java/org/apache/spark/unsafe/array/LongArray.java index 74105050e4..1a3cdff638 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/array/LongArray.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/array/LongArray.java @@ -39,7 +39,6 @@ public final class LongArray { private final long length; public LongArray(MemoryBlock memory) { - assert memory.size() % WIDTH == 0 : "Memory not aligned (" + memory.size() + ")"; assert memory.size() < (long) Integer.MAX_VALUE * 8: "Array size > 4 billion elements"; this.memory = memory; this.baseObj = memory.getBaseObject(); @@ -51,6 +50,14 @@ public final class LongArray { return memory; } + public Object getBaseObject() { + return baseObj; + } + + public long getBaseOffset() { + return baseOffset; + } + /** * Returns the number of elements this array can hold. */ @@ -59,6 +66,15 @@ public final class LongArray { } /** + * Fill this all with 0L. + */ + public void zeroOut() { + for (long off = baseOffset; off < baseOffset + length * WIDTH; off += WIDTH) { + Platform.putLong(baseObj, off, 0); + } + } + + /** * Sets the value at position {@code index}. */ public void set(int index, long value) { diff --git a/unsafe/src/test/java/org/apache/spark/unsafe/array/LongArraySuite.java b/unsafe/src/test/java/org/apache/spark/unsafe/array/LongArraySuite.java index 5974cf91ff..fb8e53b334 100644 --- a/unsafe/src/test/java/org/apache/spark/unsafe/array/LongArraySuite.java +++ b/unsafe/src/test/java/org/apache/spark/unsafe/array/LongArraySuite.java @@ -34,5 +34,9 @@ public class LongArraySuite { Assert.assertEquals(2, arr.size()); Assert.assertEquals(1L, arr.get(0)); Assert.assertEquals(3L, arr.get(1)); + + arr.zeroOut(); + Assert.assertEquals(0L, arr.get(0)); + Assert.assertEquals(0L, arr.get(1)); } } |