From 3afe448d39dc4877b2f2c62b3059aeb3ced0bd96 Mon Sep 17 00:00:00 2001 From: Yin Huai Date: Wed, 21 Oct 2015 13:43:17 -0700 Subject: [SPARK-9740][SPARK-9592][SPARK-9210][SQL] Change the default behavior of First/Last to RESPECT NULLS. I am changing the default behavior of `First`/`Last` to respect null values (the SQL standard default behavior). https://issues.apache.org/jira/browse/SPARK-9740 Author: Yin Huai Closes #8113 from yhuai/firstLast. --- .../sql/hive/execution/AggregationQuerySuite.scala | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'sql/hive') diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala index c9e1bb1995..f38a3f63c3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala @@ -323,6 +323,44 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te Row(11.125) :: Nil) } + test("first_value and last_value") { + // We force to use a single partition for the sort and aggregate to make result + // deterministic. + withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { + checkAnswer( + sqlContext.sql( + """ + |SELECT + | first_valUE(key), + | lasT_value(key), + | firSt(key), + | lASt(key), + | first_valUE(key, true), + | lasT_value(key, true), + | firSt(key, true), + | lASt(key, true) + |FROM (SELECT key FROM agg1 ORDER BY key) tmp + """.stripMargin), + Row(null, 3, null, 3, 1, 3, 1, 3) :: Nil) + + checkAnswer( + sqlContext.sql( + """ + |SELECT + | first_valUE(key), + | lasT_value(key), + | firSt(key), + | lASt(key), + | first_valUE(key, true), + | lasT_value(key, true), + | firSt(key, true), + | lASt(key, true) + |FROM (SELECT key FROM agg1 ORDER BY key DESC) tmp + """.stripMargin), + Row(3, null, 3, null, 3, 1, 3, 1) :: Nil) + } + } + test("udaf") { checkAnswer( sqlContext.sql( -- cgit v1.2.3