diff options
author | Michael Armbrust <michael@databricks.com> | 2014-06-19 23:39:03 -0700 |
---|---|---|
committer | Reynold Xin <rxin@apache.org> | 2014-06-19 23:39:14 -0700 |
commit | cc2e4ca01ad0b7dff917556580880865a7a0ccd1 (patch) | |
tree | 8fc6024fd1762816dd50df33f166c663fe1721d3 /sql | |
parent | a0e22d3983e1b066373eacc4cc1e2da1229f5d00 (diff) | |
download | spark-cc2e4ca01ad0b7dff917556580880865a7a0ccd1.tar.gz spark-cc2e4ca01ad0b7dff917556580880865a7a0ccd1.tar.bz2 spark-cc2e4ca01ad0b7dff917556580880865a7a0ccd1.zip |
[SQL] Improve Speed of InsertIntoHiveTable
Author: Michael Armbrust <michael@databricks.com>
Closes #1130 from marmbrus/noFunctional and squashes the following commits:
ccdb68c [Michael Armbrust] Remove functional programming and Array allocations from fast path in InsertIntoHiveTable.
(cherry picked from commit d3b7671c1f9c1eca956fda15fa7573649fd284b3)
Signed-off-by: Reynold Xin <rxin@apache.org>
Diffstat (limited to 'sql')
-rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala | 14 |
1 files changed, 10 insertions, 4 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala index 240aa0df49..b19579331f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala @@ -371,12 +371,18 @@ case class InsertIntoHiveTable( ObjectInspectorCopyOption.JAVA) .asInstanceOf[StructObjectInspector] + + val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector).toArray + val outputData = new Array[Any](fieldOIs.length) iter.map { row => - // Casts Strings to HiveVarchars when necessary. - val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector) - val mappedRow = row.zip(fieldOIs).map(wrap) + var i = 0 + while (i < row.length) { + // Casts Strings to HiveVarchars when necessary. + outputData(i) = wrap(row(i), fieldOIs(i)) + i += 1 + } - serializer.serialize(mappedRow.toArray, standardOI) + serializer.serialize(outputData, standardOI) } } |