From 473d78649dec7583bcc4ec24b6f38303c38e81a2 Mon Sep 17 00:00:00 2001 From: Brian Cho Date: Thu, 1 Sep 2016 14:13:17 -0700 Subject: [SPARK-16926] [SQL] Remove partition columns from partition metadata. ## What changes were proposed in this pull request? This removes partition columns from column metadata of partitions to match tables. A change introduced in SPARK-14388 removed partition columns from the column metadata of tables, but not for partitions. This causes TableReader to believe that the schema is different between table and partition, and create an unnecessary conversion object inspector in TableReader. ## How was this patch tested? Existing unit tests. Author: Brian Cho Closes #14515 from dafrista/partition-columns-metadata. --- .../main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'sql') diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala index d62bc983d0..a90da98811 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala @@ -161,7 +161,13 @@ private[hive] case class MetastoreRelation( val sd = new org.apache.hadoop.hive.metastore.api.StorageDescriptor() tPartition.setSd(sd) - sd.setCols(catalogTable.schema.map(toHiveColumn).asJava) + + // Note: In Hive the schema and partition columns must be disjoint sets + val schema = catalogTable.schema.map(toHiveColumn).filter { c => + !catalogTable.partitionColumnNames.contains(c.getName) + } + sd.setCols(schema.asJava) + p.storage.locationUri.foreach(sd.setLocation) p.storage.inputFormat.foreach(sd.setInputFormat) p.storage.outputFormat.foreach(sd.setOutputFormat) -- cgit v1.2.3