diff options
author | Tom Magrino <tmagrino@fb.com> | 2016-08-02 09:16:44 -0700 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2016-08-02 09:16:44 -0700 |
commit | 1dab63d8d3c59a3d6b4ee8e777810c44849e58b8 (patch) | |
tree | e85fe2ff8c94b10a4b3dca3f007adb025ef4290d | |
parent | 36827ddafeaa7a683362eb8da31065aaff9676d5 (diff) | |
download | spark-1dab63d8d3c59a3d6b4ee8e777810c44849e58b8.tar.gz spark-1dab63d8d3c59a3d6b4ee8e777810c44849e58b8.tar.bz2 spark-1dab63d8d3c59a3d6b4ee8e777810c44849e58b8.zip |
[SPARK-16837][SQL] TimeWindow incorrectly drops slideDuration in constructors
## What changes were proposed in this pull request?
Fix of incorrect arguments (dropping slideDuration and using windowDuration) in constructors for TimeWindow.
The JIRA this addresses is here: https://issues.apache.org/jira/browse/SPARK-16837
## How was this patch tested?
Added a test to TimeWindowSuite to check that the results of TimeWindow object apply and TimeWindow class constructor are equivalent.
Author: Tom Magrino <tmagrino@fb.com>
Closes #14441 from tmagrino/windowing-fix.
2 files changed, 14 insertions, 2 deletions
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala index 66c4bf29ea..7ff61ee479 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala @@ -45,12 +45,12 @@ case class TimeWindow( slideDuration: Expression, startTime: Expression) = { this(timeColumn, TimeWindow.parseExpression(windowDuration), - TimeWindow.parseExpression(windowDuration), TimeWindow.parseExpression(startTime)) + TimeWindow.parseExpression(slideDuration), TimeWindow.parseExpression(startTime)) } def this(timeColumn: Expression, windowDuration: Expression, slideDuration: Expression) = { this(timeColumn, TimeWindow.parseExpression(windowDuration), - TimeWindow.parseExpression(windowDuration), 0) + TimeWindow.parseExpression(slideDuration), 0) } def this(timeColumn: Expression, windowDuration: Expression) = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeWindowSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeWindowSuite.scala index b82cf8d169..d6c8fcf291 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeWindowSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeWindowSuite.scala @@ -108,4 +108,16 @@ class TimeWindowSuite extends SparkFunSuite with ExpressionEvalHelper with Priva TimeWindow.invokePrivate(parseExpression(Rand(123))) } } + + test("SPARK-16837: TimeWindow.apply equivalent to TimeWindow constructor") { + val slideLength = "1 second" + for (windowLength <- Seq("10 second", "1 minute", "2 hours")) { + val applyValue = TimeWindow(Literal(10L), windowLength, slideLength, "0 seconds") + val constructed = new TimeWindow(Literal(10L), + Literal(windowLength), + Literal(slideLength), + Literal("0 seconds")) + assert(applyValue == constructed) + } + } } |