diff options
| author | Eric Liang <ekl@databricks.com> | 2016-11-09 15:00:46 +0800 |
|---|---|---|
| committer | Wenchen Fan <wenchen@databricks.com> | 2016-11-09 15:00:46 +0800 |
| commit | 4afa39e223c70e91b6ee19e9ea76fa9115203d74 (patch) | |
| tree | 1dfd0300fe96940878c495d209594ac8f8090137 /sql/core | |
| parent | 55964c15a7b639f920dfe6c104ae4fdcd673705c (diff) | |
| download | spark-4afa39e223c70e91b6ee19e9ea76fa9115203d74.tar.gz spark-4afa39e223c70e91b6ee19e9ea76fa9115203d74.tar.bz2 spark-4afa39e223c70e91b6ee19e9ea76fa9115203d74.zip | |
[SPARK-18333][SQL] Revert hacks in parquet and orc reader to support case insensitive resolution
## What changes were proposed in this pull request?
These are no longer needed after https://issues.apache.org/jira/browse/SPARK-17183
cc cloud-fan
## How was this patch tested?
Existing parquet and orc tests.
Author: Eric Liang <ekl@databricks.com>
Closes #15799 from ericl/sc-4929.
Diffstat (limited to 'sql/core')
2 files changed, 1 insertions, 33 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala index 4dea8cf29e..f1a35dd8a6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala @@ -269,15 +269,11 @@ private[parquet] object ParquetReadSupport { */ private def clipParquetGroupFields( parquetRecord: GroupType, structType: StructType): Seq[Type] = { - val parquetFieldMap = parquetRecord.getFields.asScala - .map(f => f.getName -> f).toMap - val caseInsensitiveParquetFieldMap = parquetRecord.getFields.asScala - .map(f => f.getName.toLowerCase -> f).toMap + val parquetFieldMap = parquetRecord.getFields.asScala.map(f => f.getName -> f).toMap val toParquet = new ParquetSchemaConverter(writeLegacyParquetFormat = false) structType.map { f => parquetFieldMap .get(f.name) - .orElse(caseInsensitiveParquetFieldMap.get(f.name.toLowerCase)) .map(clipParquetType(_, f.dataType)) .getOrElse(toParquet.convertField(f)) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala index c3d202ced2..8a980a7eb5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaSuite.scala @@ -1081,34 +1081,6 @@ class ParquetSchemaSuite extends ParquetSchemaTest { } testSchemaClipping( - "falls back to case insensitive resolution", - - parquetSchema = - """message root { - | required group A { - | optional int32 B; - | } - | optional int32 c; - |} - """.stripMargin, - - catalystSchema = { - val nestedType = new StructType().add("b", IntegerType, nullable = true) - new StructType() - .add("a", nestedType, nullable = true) - .add("c", IntegerType, nullable = true) - }, - - expectedSchema = - """message root { - | required group A { - | optional int32 B; - | } - | optional int32 c; - |} - """.stripMargin) - - testSchemaClipping( "simple nested struct", parquetSchema = |