diff options
| author | Yin Huai <yhuai@databricks.com> | 2015-10-21 13:43:17 -0700 |
|---|---|---|
| committer | Michael Armbrust <michael@databricks.com> | 2015-10-21 13:43:17 -0700 |
| commit | 3afe448d39dc4877b2f2c62b3059aeb3ced0bd96 (patch) | |
| tree | 4923d317fbcbeb4c0ca9dbc257406708ce705241 /sql/core | |
| parent | f8c6bec65784de89b47e96a367d3f9790c1b3115 (diff) | |
| download | spark-3afe448d39dc4877b2f2c62b3059aeb3ced0bd96.tar.gz spark-3afe448d39dc4877b2f2c62b3059aeb3ced0bd96.tar.bz2 spark-3afe448d39dc4877b2f2c62b3059aeb3ced0bd96.zip | |
[SPARK-9740][SPARK-9592][SPARK-9210][SQL] Change the default behavior of First/Last to RESPECT NULLS.
I am changing the default behavior of `First`/`Last` to respect null values (the SQL standard default behavior).
https://issues.apache.org/jira/browse/SPARK-9740
Author: Yin Huai <yhuai@databricks.com>
Closes #8113 from yhuai/firstLast.
Diffstat (limited to 'sql/core')
| -rw-r--r-- | sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala | 13 |
1 files changed, 9 insertions, 4 deletions
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala index c3d2246297..8b9247adea 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/WindowSpec.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.expressions import org.apache.spark.annotation.Experimental +import org.apache.spark.sql.types.BooleanType import org.apache.spark.sql.{Column, catalyst} import org.apache.spark.sql.catalyst.expressions._ @@ -149,13 +150,17 @@ class WindowSpec private[sql]( case Count(child) => WindowExpression( UnresolvedWindowFunction("count", child :: Nil), WindowSpecDefinition(partitionSpec, orderSpec, frame)) - case First(child) => WindowExpression( + case First(child, ignoreNulls) => WindowExpression( // TODO this is a hack for Hive UDAF first_value - UnresolvedWindowFunction("first_value", child :: Nil), + UnresolvedWindowFunction( + "first_value", + child :: ignoreNulls :: Nil), WindowSpecDefinition(partitionSpec, orderSpec, frame)) - case Last(child) => WindowExpression( + case Last(child, ignoreNulls) => WindowExpression( // TODO this is a hack for Hive UDAF last_value - UnresolvedWindowFunction("last_value", child :: Nil), + UnresolvedWindowFunction( + "last_value", + child :: ignoreNulls :: Nil), WindowSpecDefinition(partitionSpec, orderSpec, frame)) case Min(child) => WindowExpression( UnresolvedWindowFunction("min", child :: Nil), |