diff options
author | Yijie Shen <henry.yijieshen@gmail.com> | 2015-08-04 18:19:26 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2015-08-04 18:19:26 -0700 |
commit | a7fe48f68727d5c0247698cff329fb12faff1d50 (patch) | |
tree | 90af6794e5af96213d3740d51eb2d1eb1d349f0b /sql/catalyst | |
parent | d92fa14179287c996407d9c7d249103109f9cdef (diff) | |
download | spark-a7fe48f68727d5c0247698cff329fb12faff1d50.tar.gz spark-a7fe48f68727d5c0247698cff329fb12faff1d50.tar.bz2 spark-a7fe48f68727d5c0247698cff329fb12faff1d50.zip |
[SPARK-9432][SQL] Audit expression unit tests to make sure we pass the proper numeric ranges
JIRA: https://issues.apache.org/jira/browse/SPARK-9432
Author: Yijie Shen <henry.yijieshen@gmail.com>
Closes #7933 from yjshen/numeric_ranges and squashes the following commits:
e719f78 [Yijie Shen] proper integral range check
Diffstat (limited to 'sql/catalyst')
5 files changed, 164 insertions, 4 deletions
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala index 0bae8fe2fd..a1f15e4f0f 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala @@ -23,6 +23,8 @@ import org.apache.spark.sql.types._ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { + import IntegralLiteralTestUtils._ + /** * Runs through the testFunc for all numeric data types. * @@ -47,6 +49,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(Add(Literal.create(null, left.dataType), right), null) checkEvaluation(Add(left, Literal.create(null, right.dataType)), null) } + checkEvaluation(Add(positiveShortLit, negativeShortLit), -1.toShort) + checkEvaluation(Add(positiveIntLit, negativeIntLit), -1) + checkEvaluation(Add(positiveLongLit, negativeLongLit), -1L) } test("- (UnaryMinus)") { @@ -60,6 +65,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(UnaryMinus(Literal(Int.MinValue)), Int.MinValue) checkEvaluation(UnaryMinus(Literal(Short.MinValue)), Short.MinValue) checkEvaluation(UnaryMinus(Literal(Byte.MinValue)), Byte.MinValue) + checkEvaluation(UnaryMinus(positiveShortLit), (- positiveShort).toShort) + checkEvaluation(UnaryMinus(negativeShortLit), (- negativeShort).toShort) + checkEvaluation(UnaryMinus(positiveIntLit), - positiveInt) + checkEvaluation(UnaryMinus(negativeIntLit), - negativeInt) + checkEvaluation(UnaryMinus(positiveLongLit), - positiveLong) + checkEvaluation(UnaryMinus(negativeLongLit), - negativeLong) } test("- (Minus)") { @@ -70,6 +81,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(Subtract(Literal.create(null, left.dataType), right), null) checkEvaluation(Subtract(left, Literal.create(null, right.dataType)), null) } + checkEvaluation(Subtract(positiveShortLit, negativeShortLit), + (positiveShort - negativeShort).toShort) + checkEvaluation(Subtract(positiveIntLit, negativeIntLit), positiveInt - negativeInt) + checkEvaluation(Subtract(positiveLongLit, negativeLongLit), positiveLong - negativeLong) } test("* (Multiply)") { @@ -80,6 +95,10 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(Multiply(Literal.create(null, left.dataType), right), null) checkEvaluation(Multiply(left, Literal.create(null, right.dataType)), null) } + checkEvaluation(Multiply(positiveShortLit, negativeShortLit), + (positiveShort * negativeShort).toShort) + checkEvaluation(Multiply(positiveIntLit, negativeIntLit), positiveInt * negativeInt) + checkEvaluation(Multiply(positiveLongLit, negativeLongLit), positiveLong * negativeLong) } test("/ (Divide) basic") { @@ -99,6 +118,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(Divide(Literal(1.toShort), Literal(2.toShort)), 0.toShort) checkEvaluation(Divide(Literal(1), Literal(2)), 0) checkEvaluation(Divide(Literal(1.toLong), Literal(2.toLong)), 0.toLong) + checkEvaluation(Divide(positiveShortLit, negativeShortLit), 0.toShort) + checkEvaluation(Divide(positiveIntLit, negativeIntLit), 0) + checkEvaluation(Divide(positiveLongLit, negativeLongLit), 0L) } test("/ (Divide) for floating point") { @@ -116,6 +138,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(Remainder(left, Literal.create(null, right.dataType)), null) checkEvaluation(Remainder(left, Literal(convert(0))), null) // mod by 0 } + checkEvaluation(Remainder(positiveShortLit, positiveShortLit), 0.toShort) + checkEvaluation(Remainder(negativeShortLit, negativeShortLit), 0.toShort) + checkEvaluation(Remainder(positiveIntLit, positiveIntLit), 0) + checkEvaluation(Remainder(negativeIntLit, negativeIntLit), 0) + checkEvaluation(Remainder(positiveLongLit, positiveLongLit), 0L) + checkEvaluation(Remainder(negativeLongLit, negativeLongLit), 0L) } test("Abs") { @@ -127,6 +155,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(Abs(Literal(convert(-1))), convert(1)) checkEvaluation(Abs(Literal.create(null, dataType)), null) } + checkEvaluation(Abs(positiveShortLit), positiveShort) + checkEvaluation(Abs(negativeShortLit), (- negativeShort).toShort) + checkEvaluation(Abs(positiveIntLit), positiveInt) + checkEvaluation(Abs(negativeIntLit), - negativeInt) + checkEvaluation(Abs(positiveLongLit), positiveLong) + checkEvaluation(Abs(negativeLongLit), - negativeLong) } test("MaxOf basic") { @@ -138,6 +172,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(MaxOf(Literal.create(null, small.dataType), large), convert(2)) checkEvaluation(MaxOf(large, Literal.create(null, small.dataType)), convert(2)) } + checkEvaluation(MaxOf(positiveShortLit, negativeShortLit), (positiveShort).toShort) + checkEvaluation(MaxOf(positiveIntLit, negativeIntLit), positiveInt) + checkEvaluation(MaxOf(positiveLongLit, negativeLongLit), positiveLong) } test("MaxOf for atomic type") { @@ -156,6 +193,9 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(MinOf(Literal.create(null, small.dataType), large), convert(2)) checkEvaluation(MinOf(small, Literal.create(null, small.dataType)), convert(1)) } + checkEvaluation(MinOf(positiveShortLit, negativeShortLit), (negativeShort).toShort) + checkEvaluation(MinOf(positiveIntLit, negativeIntLit), negativeInt) + checkEvaluation(MinOf(positiveLongLit, negativeLongLit), negativeLong) } test("MinOf for atomic type") { @@ -174,9 +214,12 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper checkEvaluation(Pmod(left, Literal.create(null, right.dataType)), null) checkEvaluation(Remainder(left, Literal(convert(0))), null) // mod by 0 } - checkEvaluation(Pmod(-7, 3), 2) - checkEvaluation(Pmod(7.2D, 4.1D), 3.1000000000000005) - checkEvaluation(Pmod(Decimal(0.7), Decimal(0.2)), Decimal(0.1)) - checkEvaluation(Pmod(2L, Long.MaxValue), 2L) + checkEvaluation(Pmod(Literal(-7), Literal(3)), 2) + checkEvaluation(Pmod(Literal(7.2D), Literal(4.1D)), 3.1000000000000005) + checkEvaluation(Pmod(Literal(Decimal(0.7)), Literal(Decimal(0.2))), Decimal(0.1)) + checkEvaluation(Pmod(Literal(2L), Literal(Long.MaxValue)), 2L) + checkEvaluation(Pmod(positiveShort, negativeShort), positiveShort.toShort) + checkEvaluation(Pmod(positiveInt, negativeInt), positiveInt) + checkEvaluation(Pmod(positiveLong, negativeLong), positiveLong) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseFunctionsSuite.scala index fa30fbe528..4fc1c06153 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseFunctionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/BitwiseFunctionsSuite.scala @@ -23,6 +23,8 @@ import org.apache.spark.sql.types._ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { + import IntegralLiteralTestUtils._ + test("BitwiseNOT") { def check(input: Any, expected: Any): Unit = { val expr = BitwiseNot(Literal(input)) @@ -37,6 +39,12 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { check(123456789123L, ~123456789123L) checkEvaluation(BitwiseNot(Literal.create(null, IntegerType)), null) + checkEvaluation(BitwiseNot(positiveShortLit), (~positiveShort).toShort) + checkEvaluation(BitwiseNot(negativeShortLit), (~negativeShort).toShort) + checkEvaluation(BitwiseNot(positiveIntLit), ~positiveInt) + checkEvaluation(BitwiseNot(negativeIntLit), ~negativeInt) + checkEvaluation(BitwiseNot(positiveLongLit), ~positiveLong) + checkEvaluation(BitwiseNot(negativeLongLit), ~negativeLong) } test("BitwiseAnd") { @@ -56,6 +64,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(BitwiseAnd(nullLit, Literal(1)), null) checkEvaluation(BitwiseAnd(Literal(1), nullLit), null) checkEvaluation(BitwiseAnd(nullLit, nullLit), null) + checkEvaluation(BitwiseAnd(positiveShortLit, negativeShortLit), + (positiveShort & negativeShort).toShort) + checkEvaluation(BitwiseAnd(positiveIntLit, negativeIntLit), positiveInt & negativeInt) + checkEvaluation(BitwiseAnd(positiveLongLit, negativeLongLit), positiveLong & negativeLong) } test("BitwiseOr") { @@ -75,6 +87,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(BitwiseOr(nullLit, Literal(1)), null) checkEvaluation(BitwiseOr(Literal(1), nullLit), null) checkEvaluation(BitwiseOr(nullLit, nullLit), null) + checkEvaluation(BitwiseOr(positiveShortLit, negativeShortLit), + (positiveShort | negativeShort).toShort) + checkEvaluation(BitwiseOr(positiveIntLit, negativeIntLit), positiveInt | negativeInt) + checkEvaluation(BitwiseOr(positiveLongLit, negativeLongLit), positiveLong | negativeLong) } test("BitwiseXor") { @@ -94,5 +110,10 @@ class BitwiseFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(BitwiseXor(nullLit, Literal(1)), null) checkEvaluation(BitwiseXor(Literal(1), nullLit), null) checkEvaluation(BitwiseXor(nullLit, nullLit), null) + + checkEvaluation(BitwiseXor(positiveShortLit, negativeShortLit), + (positiveShort ^ negativeShort).toShort) + checkEvaluation(BitwiseXor(positiveIntLit, negativeIntLit), positiveInt ^ negativeInt) + checkEvaluation(BitwiseXor(positiveLongLit, negativeLongLit), positiveLong ^ negativeLong) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index e6e8790e90..f9b73f1a75 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -28,6 +28,8 @@ import org.apache.spark.unsafe.types.CalendarInterval class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { + import IntegralLiteralTestUtils._ + val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") val sdfDate = new SimpleDateFormat("yyyy-MM-dd") val d = new Date(sdf.parse("2015-04-08 13:10:15").getTime) @@ -212,6 +214,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { null) checkEvaluation(DateAdd(Literal.create(null, DateType), Literal.create(null, IntegerType)), null) + checkEvaluation( + DateAdd(Literal(Date.valueOf("2016-02-28")), positiveIntLit), 49627) + checkEvaluation( + DateAdd(Literal(Date.valueOf("2016-02-28")), negativeIntLit), -15910) } test("date_sub") { @@ -226,6 +232,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { null) checkEvaluation(DateSub(Literal.create(null, DateType), Literal.create(null, IntegerType)), null) + checkEvaluation( + DateSub(Literal(Date.valueOf("2016-02-28")), positiveIntLit), -15909) + checkEvaluation( + DateSub(Literal(Date.valueOf("2016-02-28")), negativeIntLit), 49628) } test("time_add") { @@ -282,6 +292,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { null) checkEvaluation( AddMonths(Literal(Date.valueOf("2015-01-30")), Literal(Int.MinValue)), -7293498) + checkEvaluation( + AddMonths(Literal(Date.valueOf("2016-02-28")), positiveIntLit), 1014213) + checkEvaluation( + AddMonths(Literal(Date.valueOf("2016-02-28")), negativeIntLit), -980528) } test("months_between") { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntegralLiteralTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntegralLiteralTestUtils.scala new file mode 100644 index 0000000000..2e5a121f4e --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntegralLiteralTestUtils.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +/** + * Utilities to make sure we pass the proper numeric ranges + */ +object IntegralLiteralTestUtils { + + val positiveShort: Short = (Byte.MaxValue + 1).toShort + val negativeShort: Short = (Byte.MinValue - 1).toShort + + val positiveShortLit: Literal = Literal(positiveShort) + val negativeShortLit: Literal = Literal(negativeShort) + + val positiveInt: Int = Short.MaxValue + 1 + val negativeInt: Int = Short.MinValue - 1 + + val positiveIntLit: Literal = Literal(positiveInt) + val negativeIntLit: Literal = Literal(negativeInt) + + val positiveLong: Long = Int.MaxValue + 1L + val negativeLong: Long = Int.MinValue - 1L + + val positiveLongLit: Literal = Literal(positiveLong) + val negativeLongLit: Literal = Literal(negativeLong) +} diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathFunctionsSuite.scala index 9fcb548af6..033792eee6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathFunctionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MathFunctionsSuite.scala @@ -30,6 +30,8 @@ import org.apache.spark.sql.types._ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { + import IntegralLiteralTestUtils._ + /** * Used for testing leaf math expressions. * @@ -293,6 +295,9 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(Bin(l3), java.lang.Long.toBinaryString(123), row) checkEvaluation(Bin(l4), java.lang.Long.toBinaryString(1234), row) checkEvaluation(Bin(l5), java.lang.Long.toBinaryString(-123), row) + + checkEvaluation(Bin(positiveLongLit), java.lang.Long.toBinaryString(positiveLong)) + checkEvaluation(Bin(negativeLongLit), java.lang.Long.toBinaryString(negativeLong)) } test("log2") { @@ -324,6 +329,15 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(ShiftLeft(Literal(21.toLong), Literal(1)), 42.toLong) checkEvaluation(ShiftLeft(Literal(-21.toLong), Literal(1)), -42.toLong) + + checkEvaluation(ShiftLeft(positiveIntLit, positiveIntLit), positiveInt << positiveInt) + checkEvaluation(ShiftLeft(positiveIntLit, negativeIntLit), positiveInt << negativeInt) + checkEvaluation(ShiftLeft(negativeIntLit, positiveIntLit), negativeInt << positiveInt) + checkEvaluation(ShiftLeft(negativeIntLit, negativeIntLit), negativeInt << negativeInt) + checkEvaluation(ShiftLeft(positiveLongLit, positiveIntLit), positiveLong << positiveInt) + checkEvaluation(ShiftLeft(positiveLongLit, negativeIntLit), positiveLong << negativeInt) + checkEvaluation(ShiftLeft(negativeLongLit, positiveIntLit), negativeLong << positiveInt) + checkEvaluation(ShiftLeft(negativeLongLit, negativeIntLit), negativeLong << negativeInt) } test("shift right") { @@ -335,6 +349,15 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(ShiftRight(Literal(42.toLong), Literal(1)), 21.toLong) checkEvaluation(ShiftRight(Literal(-42.toLong), Literal(1)), -21.toLong) + + checkEvaluation(ShiftRight(positiveIntLit, positiveIntLit), positiveInt >> positiveInt) + checkEvaluation(ShiftRight(positiveIntLit, negativeIntLit), positiveInt >> negativeInt) + checkEvaluation(ShiftRight(negativeIntLit, positiveIntLit), negativeInt >> positiveInt) + checkEvaluation(ShiftRight(negativeIntLit, negativeIntLit), negativeInt >> negativeInt) + checkEvaluation(ShiftRight(positiveLongLit, positiveIntLit), positiveLong >> positiveInt) + checkEvaluation(ShiftRight(positiveLongLit, negativeIntLit), positiveLong >> negativeInt) + checkEvaluation(ShiftRight(negativeLongLit, positiveIntLit), negativeLong >> positiveInt) + checkEvaluation(ShiftRight(negativeLongLit, negativeIntLit), negativeLong >> negativeInt) } test("shift right unsigned") { @@ -346,6 +369,23 @@ class MathFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(ShiftRightUnsigned(Literal(42.toLong), Literal(1)), 21.toLong) checkEvaluation(ShiftRightUnsigned(Literal(-42.toLong), Literal(1)), 9223372036854775787L) + + checkEvaluation(ShiftRightUnsigned(positiveIntLit, positiveIntLit), + positiveInt >>> positiveInt) + checkEvaluation(ShiftRightUnsigned(positiveIntLit, negativeIntLit), + positiveInt >>> negativeInt) + checkEvaluation(ShiftRightUnsigned(negativeIntLit, positiveIntLit), + negativeInt >>> positiveInt) + checkEvaluation(ShiftRightUnsigned(negativeIntLit, negativeIntLit), + negativeInt >>> negativeInt) + checkEvaluation(ShiftRightUnsigned(positiveLongLit, positiveIntLit), + positiveLong >>> positiveInt) + checkEvaluation(ShiftRightUnsigned(positiveLongLit, negativeIntLit), + positiveLong >>> negativeInt) + checkEvaluation(ShiftRightUnsigned(negativeLongLit, positiveIntLit), + negativeLong >>> positiveInt) + checkEvaluation(ShiftRightUnsigned(negativeLongLit, negativeIntLit), + negativeLong >>> negativeInt) } test("hex") { |