diff options
author | Pete Robbins <robbinspg@gmail.com> | 2016-01-04 10:43:21 -0800 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2016-01-04 10:43:21 -0800 |
commit | b504b6a90a95a723210beb0031ed41a75d702f66 (patch) | |
tree | 2208eab4cf1ac5b1ab25cccbc59fbb5b096e7669 | |
parent | 6c83d938cc61bd5fabaf2157fcc3936364a83f02 (diff) | |
download | spark-b504b6a90a95a723210beb0031ed41a75d702f66.tar.gz spark-b504b6a90a95a723210beb0031ed41a75d702f66.tar.bz2 spark-b504b6a90a95a723210beb0031ed41a75d702f66.zip |
[SPARK-12470] [SQL] Fix size reduction calculation
also only allocate required buffer size
Author: Pete Robbins <robbinspg@gmail.com>
Closes #10421 from robbinspg/master.
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala index c9ff357bf3..037ae83d48 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala @@ -61,9 +61,9 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U val outputBitsetWords = (schema1.size + schema2.size + 63) / 64 val bitset1Remainder = schema1.size % 64 - // The number of words we can reduce when we concat two rows together. + // The number of bytes we can reduce when we concat two rows together. // The only reduction comes from merging the bitset portion of the two rows, saving 1 word. - val sizeReduction = bitset1Words + bitset2Words - outputBitsetWords + val sizeReduction = (bitset1Words + bitset2Words - outputBitsetWords) * 8 // --------------------- copy bitset from row 1 and row 2 --------------------------- // val copyBitset = Seq.tabulate(outputBitsetWords) { i => @@ -171,7 +171,7 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U | // row1: ${schema1.size} fields, $bitset1Words words in bitset | // row2: ${schema2.size}, $bitset2Words words in bitset | // output: ${schema1.size + schema2.size} fields, $outputBitsetWords words in bitset - | final int sizeInBytes = row1.getSizeInBytes() + row2.getSizeInBytes(); + | final int sizeInBytes = row1.getSizeInBytes() + row2.getSizeInBytes() - $sizeReduction; | if (sizeInBytes > buf.length) { | buf = new byte[sizeInBytes]; | } @@ -188,7 +188,7 @@ object GenerateUnsafeRowJoiner extends CodeGenerator[(StructType, StructType), U | $copyVariableLengthRow2 | $updateOffset | - | out.pointTo(buf, sizeInBytes - $sizeReduction); + | out.pointTo(buf, sizeInBytes); | | return out; | } |