diff options
author | Reynold Xin <rxin@databricks.com> | 2015-07-30 13:09:43 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2015-07-30 13:09:43 -0700 |
commit | a20e743fb863de809863652931bc982aac2d1f86 (patch) | |
tree | f948f807fb34b8861809e91a8b04fa22fb752af9 /unsafe/src/test | |
parent | 6d94bf6ac10ac851636c62439f8f2737f3526a2a (diff) | |
download | spark-a20e743fb863de809863652931bc982aac2d1f86.tar.gz spark-a20e743fb863de809863652931bc982aac2d1f86.tar.bz2 spark-a20e743fb863de809863652931bc982aac2d1f86.zip |
[SPARK-9460] Fix prefix generation for UTF8String.
Previously we could be getting garbage data if the number of bytes is 0, or on JVMs that are 4 byte aligned, or when compressedoops is on.
Author: Reynold Xin <rxin@databricks.com>
Closes #7789 from rxin/utf8string and squashes the following commits:
86ffa3e [Reynold Xin] Mask out data outside of valid range.
4d647ed [Reynold Xin] Mask out data.
c6e8794 [Reynold Xin] [SPARK-9460] Fix prefix generation for UTF8String.
Diffstat (limited to 'unsafe/src/test')
-rw-r--r-- | unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java index 42e09e435a..f2cc19ca6b 100644 --- a/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java +++ b/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java @@ -71,6 +71,14 @@ public class UTF8StringSuite { fromString("abbbbbbbbbbbasdf").getPrefix() - fromString("bbbbbbbbbbbbasdf").getPrefix() < 0); assertTrue(fromString("").getPrefix() - fromString("a").getPrefix() < 0); assertTrue(fromString("你好").getPrefix() - fromString("世界").getPrefix() > 0); + + byte[] buf1 = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + byte[] buf2 = {1, 2, 3}; + UTF8String str1 = UTF8String.fromBytes(buf1, 0, 3); + UTF8String str2 = UTF8String.fromBytes(buf1, 0, 8); + UTF8String str3 = UTF8String.fromBytes(buf2); + assertTrue(str1.getPrefix() - str2.getPrefix() < 0); + assertEquals(str1.getPrefix(), str3.getPrefix()); } @Test |