diff options
author | Michael Armbrust <michael@databricks.com> | 2014-06-19 23:39:03 -0700 |
---|---|---|
committer | Reynold Xin <rxin@apache.org> | 2014-06-19 23:39:03 -0700 |
commit | d3b7671c1f9c1eca956fda15fa7573649fd284b3 (patch) | |
tree | 9893923ff7028a40a4a93416737591e1a1337042 | |
parent | 278ec8a203c7f1de2716d8284f9bdafa54eee1cb (diff) | |
download | spark-d3b7671c1f9c1eca956fda15fa7573649fd284b3.tar.gz spark-d3b7671c1f9c1eca956fda15fa7573649fd284b3.tar.bz2 spark-d3b7671c1f9c1eca956fda15fa7573649fd284b3.zip |
[SQL] Improve Speed of InsertIntoHiveTable
Author: Michael Armbrust <michael@databricks.com>
Closes #1130 from marmbrus/noFunctional and squashes the following commits:
ccdb68c [Michael Armbrust] Remove functional programming and Array allocations from fast path in InsertIntoHiveTable.
-rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala | 14 |
1 files changed, 10 insertions, 4 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala index 240aa0df49..b19579331f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/hiveOperators.scala @@ -371,12 +371,18 @@ case class InsertIntoHiveTable( ObjectInspectorCopyOption.JAVA) .asInstanceOf[StructObjectInspector] + + val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector).toArray + val outputData = new Array[Any](fieldOIs.length) iter.map { row => - // Casts Strings to HiveVarchars when necessary. - val fieldOIs = standardOI.getAllStructFieldRefs.map(_.getFieldObjectInspector) - val mappedRow = row.zip(fieldOIs).map(wrap) + var i = 0 + while (i < row.length) { + // Casts Strings to HiveVarchars when necessary. + outputData(i) = wrap(row(i), fieldOIs(i)) + i += 1 + } - serializer.serialize(mappedRow.toArray, standardOI) + serializer.serialize(outputData, standardOI) } } |