aboutsummaryrefslogtreecommitdiff
path: root/sql
diff options
context:
space:
mode:
authorCheng Lian <lian@databricks.com>2015-04-02 16:15:34 -0700
committerReynold Xin <rxin@databricks.com>2015-04-02 16:15:34 -0700
commitd3944b6f2aeb36629bf89207629cc5e55d327241 (patch)
treef3b5aee5a1443fe5e8777484ed611f23aecfecf0 /sql
parent251698fb7335a3bb465f1cd0c29e7e74e0361f4a (diff)
downloadspark-d3944b6f2aeb36629bf89207629cc5e55d327241.tar.gz
spark-d3944b6f2aeb36629bf89207629cc5e55d327241.tar.bz2
spark-d3944b6f2aeb36629bf89207629cc5e55d327241.zip
[Minor] [SQL] Follow-up of PR #5210
This PR addresses rxin's comments in PR #5210. <!-- Reviewable:start --> [<img src="https://reviewable.io/review_button.png" height=40 alt="Review on Reviewable"/>](https://reviewable.io/reviews/apache/spark/5219) <!-- Reviewable:end --> Author: Cheng Lian <lian@databricks.com> Closes #5219 from liancheng/spark-6554-followup and squashes the following commits: 41f3a09 [Cheng Lian] Addresses comments in #5210
Diffstat (limited to 'sql')
-rw-r--r--sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala9
1 files changed, 5 insertions, 4 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
index e12531480c..583bac42fd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
@@ -434,17 +434,18 @@ private[sql] case class ParquetRelation2(
FileInputFormat.setInputPaths(job, selectedFiles.map(_.getPath): _*)
}
- // Push down filters when possible. Notice that not all filters can be converted to Parquet
- // filter predicate. Here we try to convert each individual predicate and only collect those
- // convertible ones.
+ // Try to push down filters when filter push-down is enabled.
if (sqlContext.conf.parquetFilterPushDown) {
+ val partitionColNames = partitionColumns.map(_.name).toSet
predicates
// Don't push down predicates which reference partition columns
.filter { pred =>
- val partitionColNames = partitionColumns.map(_.name).toSet
val referencedColNames = pred.references.map(_.name).toSet
referencedColNames.intersect(partitionColNames).isEmpty
}
+ // Collects all converted Parquet filter predicates. Notice that not all predicates can be
+ // converted (`ParquetFilters.createFilter` returns an `Option`). That's why a `flatMap`
+ // is used here.
.flatMap(ParquetFilters.createFilter)
.reduceOption(FilterApi.and)
.foreach(ParquetInputFormat.setFilterPredicate(jobConf, _))