From 67e59d464f782ff5f509234212aa072a7653d7bf Mon Sep 17 00:00:00 2001 From: Reynold Xin Date: Fri, 19 Aug 2016 21:11:35 +0800 Subject: [SPARK-16994][SQL] Whitelist operators for predicate pushdown ## What changes were proposed in this pull request? This patch changes predicate pushdown optimization rule (PushDownPredicate) from using a blacklist to a whitelist. That is to say, operators must be explicitly allowed. This approach is more future-proof: previously it was possible for us to introduce a new operator and then render the optimization rule incorrect. This also fixes the bug that previously we allowed pushing filter beneath limit, which was incorrect. That is to say, before this patch, the optimizer would rewrite ``` select * from (select * from range(10) limit 5) where id > 3 to select * from range(10) where id > 3 limit 5 ``` ## How was this patch tested? - a unit test case in FilterPushdownSuite - an end-to-end test in limit.sql Author: Reynold Xin Closes #14713 from rxin/SPARK-16994. --- sql/core/src/test/resources/sql-tests/inputs/limit.sql | 3 +++ sql/core/src/test/resources/sql-tests/results/limit.sql.out | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'sql/core/src/test/resources') diff --git a/sql/core/src/test/resources/sql-tests/inputs/limit.sql b/sql/core/src/test/resources/sql-tests/inputs/limit.sql index 892a1bb4b5..2ea35f7f3a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/limit.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/limit.sql @@ -18,3 +18,6 @@ select * from testdata limit key > 3; -- limit must be integer select * from testdata limit true; select * from testdata limit 'a'; + +-- limit within a subquery +select * from (select * from range(10) limit 5) where id > 3; diff --git a/sql/core/src/test/resources/sql-tests/results/limit.sql.out b/sql/core/src/test/resources/sql-tests/results/limit.sql.out index b71b058869..cb4e4d0481 100644 --- a/sql/core/src/test/resources/sql-tests/results/limit.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/limit.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 9 +-- Number of queries: 10 -- !query 0 @@ -81,3 +81,11 @@ struct<> -- !query 8 output org.apache.spark.sql.AnalysisException The limit expression must be integer type, but got string; + + +-- !query 9 +select * from (select * from range(10) limit 5) where id > 3 +-- !query 9 schema +struct +-- !query 9 output +4 -- cgit v1.2.3