diff options
author | Wenchen Fan <cloud0fan@163.com> | 2015-10-05 17:30:54 -0700 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2015-10-05 17:31:54 -0700 |
commit | a609eb20d964a7b92f1066300443415f6db181e3 (patch) | |
tree | 740fe5e8dde05e912f04a5a537a24997a9bf537f | |
parent | c4871369db96fc33c465d11b3bbd1ffeb3b94e89 (diff) | |
download | spark-a609eb20d964a7b92f1066300443415f6db181e3.tar.gz spark-a609eb20d964a7b92f1066300443415f6db181e3.tar.bz2 spark-a609eb20d964a7b92f1066300443415f6db181e3.zip |
[SPARK-10934] [SQL] handle hashCode of unsafe array correctly
`Murmur3_x86_32.hashUnsafeWords` only accepts word-aligned bytes, but unsafe array is not.
Author: Wenchen Fan <cloud0fan@163.com>
Closes #8987 from cloud-fan/hash.
-rw-r--r-- | sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java | 6 | ||||
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala | 7 |
2 files changed, 12 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java index da9538b3f1..6a16d34083 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java @@ -285,7 +285,11 @@ public class UnsafeArrayData extends ArrayData { @Override public int hashCode() { - return Murmur3_x86_32.hashUnsafeWords(baseObject, baseOffset, sizeInBytes, 42); + int result = 37; + for (int i = 0; i < sizeInBytes; i++) { + result = 37 * result + Platform.getByte(baseObject, baseOffset + i); + } + return result; } @Override diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala index 2476b10e3c..944d4e1134 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala @@ -131,4 +131,11 @@ class UnsafeRowSuite extends SparkFunSuite { assert(emptyRow.getInt(0) === unsafeRow.getInt(0)) assert(emptyRow.getUTF8String(1) === unsafeRow.getUTF8String(1)) } + + test("calling hashCode on unsafe array returned by getArray(ordinal)") { + val row = InternalRow.apply(new GenericArrayData(Array(1L))) + val unsafeRow = UnsafeProjection.create(Array[DataType](ArrayType(LongType))).apply(row) + // Makes sure hashCode on unsafe array won't crash + unsafeRow.getArray(0).hashCode() + } } |