diff options
| author | Yin Huai <yhuai@databricks.com> | 2015-10-21 13:43:17 -0700 |
|---|---|---|
| committer | Michael Armbrust <michael@databricks.com> | 2015-10-21 13:43:17 -0700 |
| commit | 3afe448d39dc4877b2f2c62b3059aeb3ced0bd96 (patch) | |
| tree | 4923d317fbcbeb4c0ca9dbc257406708ce705241 /sql/hive | |
| parent | f8c6bec65784de89b47e96a367d3f9790c1b3115 (diff) | |
| download | spark-3afe448d39dc4877b2f2c62b3059aeb3ced0bd96.tar.gz spark-3afe448d39dc4877b2f2c62b3059aeb3ced0bd96.tar.bz2 spark-3afe448d39dc4877b2f2c62b3059aeb3ced0bd96.zip | |
[SPARK-9740][SPARK-9592][SPARK-9210][SQL] Change the default behavior of First/Last to RESPECT NULLS.
I am changing the default behavior of `First`/`Last` to respect null values (the SQL standard default behavior).
https://issues.apache.org/jira/browse/SPARK-9740
Author: Yin Huai <yhuai@databricks.com>
Closes #8113 from yhuai/firstLast.
Diffstat (limited to 'sql/hive')
| -rw-r--r-- | sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala | 38 |
1 files changed, 38 insertions, 0 deletions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala index c9e1bb1995..f38a3f63c3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala @@ -323,6 +323,44 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te Row(11.125) :: Nil) } + test("first_value and last_value") { + // We force to use a single partition for the sort and aggregate to make result + // deterministic. + withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { + checkAnswer( + sqlContext.sql( + """ + |SELECT + | first_valUE(key), + | lasT_value(key), + | firSt(key), + | lASt(key), + | first_valUE(key, true), + | lasT_value(key, true), + | firSt(key, true), + | lASt(key, true) + |FROM (SELECT key FROM agg1 ORDER BY key) tmp + """.stripMargin), + Row(null, 3, null, 3, 1, 3, 1, 3) :: Nil) + + checkAnswer( + sqlContext.sql( + """ + |SELECT + | first_valUE(key), + | lasT_value(key), + | firSt(key), + | lASt(key), + | first_valUE(key, true), + | lasT_value(key, true), + | firSt(key, true), + | lASt(key, true) + |FROM (SELECT key FROM agg1 ORDER BY key DESC) tmp + """.stripMargin), + Row(3, null, 3, null, 3, 1, 3, 1) :: Nil) + } + } + test("udaf") { checkAnswer( sqlContext.sql( |