diff options
| author | Reynold Xin <rxin@databricks.com> | 2016-07-07 18:09:18 +0800 |
|---|---|---|
| committer | Cheng Lian <lian@databricks.com> | 2016-07-07 18:09:18 +0800 |
| commit | 986b2514013ed9ebab526f2cf3dc714cc9e480bf (patch) | |
| tree | 2c1fbd3515c18a50702bb67399339163ecd42196 /sql/catalyst | |
| parent | ab05db0b48f395543cd7d91e2ad9dd760516868b (diff) | |
| download | spark-986b2514013ed9ebab526f2cf3dc714cc9e480bf.tar.gz spark-986b2514013ed9ebab526f2cf3dc714cc9e480bf.tar.bz2 spark-986b2514013ed9ebab526f2cf3dc714cc9e480bf.zip | |
[SPARK-16400][SQL] Remove InSet filter pushdown from Parquet
## What changes were proposed in this pull request?
This patch removes InSet filter pushdown from Parquet data source, since row-based pushdown is not beneficial to Spark and brings extra complexity to the code base.
## How was this patch tested?
N/A
Author: Reynold Xin <rxin@databricks.com>
Closes #14076 from rxin/SPARK-16400.
Diffstat (limited to 'sql/catalyst')
| -rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala | 7 |
1 files changed, 6 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala index 0284ecc0d9..0c2ebb0e5b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala @@ -22,7 +22,7 @@ import scala.util.Try import org.json4s.JsonDSL._ -import org.apache.spark.{SparkEnv, SparkException} +import org.apache.spark.SparkException import org.apache.spark.annotation.DeveloperApi import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, InterpretedOrdering} import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, LegacyTypeStringParser} @@ -389,6 +389,11 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru object StructType extends AbstractDataType { + /** + * A key used in field metadata to indicate that the field comes from the result of merging + * two different StructTypes that do not always contain the field. That is to say, the field + * might be missing (optional) from one of the StructTypes. + */ private[sql] val metadataKeyForOptionalField = "_OPTIONAL_" override private[sql] def defaultConcreteType: DataType = new StructType |