diff options
author | Kousuke Saruta <sarutak@oss.nttdata.co.jp> | 2014-11-24 12:54:37 -0800 |
---|---|---|
committer | Michael Armbrust <michael@databricks.com> | 2014-11-24 12:54:50 -0800 |
commit | 97b7eb4d99613944d39f1421dccc2724c4165c9e (patch) | |
tree | 14bfeb03e3bf4f9b0c21ecf5b08531b76970cbb0 /sql | |
parent | 0e7fa7f632ebe4db60938f2087c1f1a4d614ab32 (diff) | |
download | spark-97b7eb4d99613944d39f1421dccc2724c4165c9e.tar.gz spark-97b7eb4d99613944d39f1421dccc2724c4165c9e.tar.bz2 spark-97b7eb4d99613944d39f1421dccc2724c4165c9e.zip |
[SPARK-4487][SQL] Fix attribute reference resolution error when using ORDER BY.
When we use ORDER BY clause, at first, attributes referenced by projection are resolved (1).
And then, attributes referenced at ORDER BY clause are resolved (2).
But when resolving attributes referenced at ORDER BY clause, the resolution result generated in (1) is discarded so for example, following query fails.
SELECT c1 + c2 FROM mytable ORDER BY c1;
The query above fails because when resolving the attribute reference 'c1', the resolution result of 'c2' is discarded.
Author: Kousuke Saruta <sarutak@oss.nttdata.co.jp>
Closes #3363 from sarutak/SPARK-4487 and squashes the following commits:
fd314f3 [Kousuke Saruta] Fixed attribute resolution logic in Analyzer
6e60c20 [Kousuke Saruta] Fixed conflicts
cb5b7e9 [Kousuke Saruta] Added test case for SPARK-4487
282d529 [Kousuke Saruta] Fixed attributes reference resolution error
b6123e6 [Kousuke Saruta] Merge branch 'master' of git://git.apache.org/spark into concat-feature
317b7fb [Kousuke Saruta] WIP
(cherry picked from commit dd1c9cb36cde8202cede8014b5641ae8a0197812)
Signed-off-by: Michael Armbrust <michael@databricks.com>
Diffstat (limited to 'sql')
-rw-r--r-- | sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 | ||||
-rw-r--r-- | sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 7 |
2 files changed, 8 insertions, 1 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index d3b4cf8e34..facbd8b975 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -179,7 +179,7 @@ class Analyzer(catalog: Catalog, registry: FunctionRegistry, caseSensitive: Bool val missingInProject = requiredAttributes -- p.output if (missingInProject.nonEmpty) { // Add missing attributes and then project them away after the sort. - Project(projectList, + Project(projectList.map(_.toAttribute), Sort(ordering, Project(projectList ++ missingInProject, child))) } else { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 0a96831c76..84ee3051eb 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -974,6 +974,13 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { dropTempTable("data") } + test("SPARK-4432 Fix attribute reference resolution error when using ORDER BY") { + checkAnswer( + sql("SELECT a + b FROM testData2 ORDER BY a"), + Seq(2, 3, 3 ,4 ,4 ,5).map(Seq(_)) + ) + } + test("Supporting relational operator '<=>' in Spark SQL") { val nullCheckData1 = TestData(1,"1") :: TestData(2,null) :: Nil val rdd1 = sparkContext.parallelize((0 to 1).map(i => nullCheckData1(i))) |