diff options
author | Yin Huai <huai@cse.ohio-state.edu> | 2014-08-08 11:01:51 -0700 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2014-08-08 11:02:01 -0700 |
commit | 3eb5dd043427de8c050687231011863b22feecdb (patch) | |
tree | 0be1fb66cd7214f7c43272767b39c194aab0fe33 | |
parent | a54b5d955df151562721fc04b438337d15ab1dec (diff) | |
download | spark-3eb5dd043427de8c050687231011863b22feecdb.tar.gz spark-3eb5dd043427de8c050687231011863b22feecdb.tar.bz2 spark-3eb5dd043427de8c050687231011863b22feecdb.zip |
[SPARK-2888] [SQL] Fix addColumnMetadataToConf in HiveTableScan
JIRA: https://issues.apache.org/jira/browse/SPARK-2888
Author: Yin Huai <huai@cse.ohio-state.edu>
Closes #1817 from yhuai/fixAddColumnMetadataToConf and squashes the following commits:
fba728c [Yin Huai] Fix addColumnMetadataToConf.
(cherry picked from commit 9016af3f2729101027e33593e094332f05f48d92)
Signed-off-by: Michael Armbrust <michael@databricks.com>
-rw-r--r-- | sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala | 17 |
1 files changed, 6 insertions, 11 deletions
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala index 8920e2a76a..577ca928b4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScan.scala @@ -72,17 +72,12 @@ case class HiveTableScan( } private def addColumnMetadataToConf(hiveConf: HiveConf) { - // Specifies IDs and internal names of columns to be scanned. - val neededColumnIDs = attributes.map(a => relation.output.indexWhere(_.name == a.name): Integer) - val columnInternalNames = neededColumnIDs.map(HiveConf.getColumnInternalName(_)).mkString(",") - - if (attributes.size == relation.output.size) { - // SQLContext#pruneFilterProject guarantees no duplicated value in `attributes` - ColumnProjectionUtils.setFullyReadColumns(hiveConf) - } else { - ColumnProjectionUtils.appendReadColumnIDs(hiveConf, neededColumnIDs) - } + // Specifies needed column IDs for those non-partitioning columns. + val neededColumnIDs = + attributes.map(a => + relation.attributes.indexWhere(_.name == a.name): Integer).filter(index => index >= 0) + ColumnProjectionUtils.appendReadColumnIDs(hiveConf, neededColumnIDs) ColumnProjectionUtils.appendReadColumnNames(hiveConf, attributes.map(_.name)) // Specifies types and object inspectors of columns to be scanned. @@ -99,7 +94,7 @@ case class HiveTableScan( .mkString(",") hiveConf.set(serdeConstants.LIST_COLUMN_TYPES, columnTypeNames) - hiveConf.set(serdeConstants.LIST_COLUMNS, columnInternalNames) + hiveConf.set(serdeConstants.LIST_COLUMNS, relation.attributes.map(_.name).mkString(",")) } addColumnMetadataToConf(context.hiveconf) |