[SPARK-7188] added python support for math DataFrame functions

Adds support for the math functions for DataFrames in PySpark.

rxin I love Davies.

Author: Burak Yavuz <brkyvz@gmail.com>

Closes #5750 from brkyvz/python-math-udfs and squashes the following commits:

7c4f563 [Burak Yavuz] removed is_math
3c4adde [Burak Yavuz] cleanup imports
d5dca3f [Burak Yavuz] moved math functions to mathfunctions
25e6534 [Burak Yavuz] addressed comments v2.0
d3f7e0f [Burak Yavuz] addressed comments and added tests
7b7d7c4 [Burak Yavuz] remove tests for removed methods
33c2c15 [Burak Yavuz] fixed python style
3ee0c05 [Burak Yavuz] added python functions

2 files changed, 116 insertions, 305 deletions

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
index 84f62bf47f..d901542b7e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
@@ -29,9 +29,6 @@ import org.apache.spark.sql.functions.lit
  * Mathematical Functions available for [[DataFrame]].
  *
  * @groupname double_funcs Functions that require DoubleType as an input
- * @groupname int_funcs Functions that require IntegerType as an input
- * @groupname float_funcs Functions that require FloatType as an input
- * @groupname long_funcs Functions that require LongType as an input
  */
 @Experimental
 // scalastyle:off
@@ -41,522 +38,348 @@ object mathfunctions {
   private[this] implicit def toColumn(expr: Expression): Column = Column(expr)
 
   /**
-   * Computes the sine of the given value.
-   *
-   * @group double_funcs
+   * Computes the cosine inverse of the given value; the returned angle is in the range
+   * 0.0 through pi.
    */
-  def sin(e: Column): Column = Sin(e.expr)
+  def acos(e: Column): Column = Acos(e.expr)
 
   /**
-   * Computes the sine of the given column.
-   *
-   * @group double_funcs
+   * Computes the cosine inverse of the given column; the returned angle is in the range
+   * 0.0 through pi.
    */
-  def sin(columnName: String): Column = sin(Column(columnName))
+  def acos(columnName: String): Column = acos(Column(columnName))
 
   /**
    * Computes the sine inverse of the given value; the returned angle is in the range
    * -pi/2 through pi/2.
-   *
-   * @group double_funcs
    */
   def asin(e: Column): Column = Asin(e.expr)
 
   /**
    * Computes the sine inverse of the given column; the returned angle is in the range
    * -pi/2 through pi/2.
-   *
-   * @group double_funcs
    */
   def asin(columnName: String): Column = asin(Column(columnName))
 
   /**
-   * Computes the hyperbolic sine of the given value.
-   *
-   * @group double_funcs
-   */
-  def sinh(e: Column): Column = Sinh(e.expr)
-
-  /**
-   * Computes the hyperbolic sine of the given column.
-   *
-   * @group double_funcs
+   * Computes the tangent inverse of the given value.
    */
-  def sinh(columnName: String): Column = sinh(Column(columnName))
+  def atan(e: Column): Column = Atan(e.expr)
 
   /**
-   * Computes the cosine of the given value.
-   *
-   * @group double_funcs
+   * Computes the tangent inverse of the given column.
    */
-  def cos(e: Column): Column = Cos(e.expr)
+  def atan(columnName: String): Column = atan(Column(columnName))
 
   /**
-   * Computes the cosine of the given column.
-   *
-   * @group double_funcs
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
    */
-  def cos(columnName: String): Column = cos(Column(columnName))
+  def atan2(l: Column, r: Column): Column = Atan2(l.expr, r.expr)
 
   /**
-   * Computes the cosine inverse of the given value; the returned angle is in the range
-   * 0.0 through pi.
-   *
-   * @group double_funcs
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
    */
-  def acos(e: Column): Column = Acos(e.expr)
+  def atan2(l: Column, rightName: String): Column = atan2(l, Column(rightName))
 
   /**
-   * Computes the cosine inverse of the given column; the returned angle is in the range
-   * 0.0 through pi.
-   *
-   * @group double_funcs
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
    */
-  def acos(columnName: String): Column = acos(Column(columnName))
+  def atan2(leftName: String, r: Column): Column = atan2(Column(leftName), r)
 
   /**
-   * Computes the hyperbolic cosine of the given value.
-   *
-   * @group double_funcs
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
    */
-  def cosh(e: Column): Column = Cosh(e.expr)
+  def atan2(leftName: String, rightName: String): Column =
+    atan2(Column(leftName), Column(rightName))
 
   /**
-   * Computes the hyperbolic cosine of the given column.
-   *
-   * @group double_funcs
-   */
-  def cosh(columnName: String): Column = cosh(Column(columnName))
-  
-  /**
-   * Computes the tangent of the given value.
-   *
-   * @group double_funcs
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
    */
-  def tan(e: Column): Column = Tan(e.expr)
+  def atan2(l: Column, r: Double): Column = atan2(l, lit(r).expr)
 
   /**
-   * Computes the tangent of the given column.
-   *
-   * @group double_funcs
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).=
    */
-  def tan(columnName: String): Column = tan(Column(columnName))
+  def atan2(leftName: String, r: Double): Column = atan2(Column(leftName), r)
 
   /**
-   * Computes the tangent inverse of the given value.
-   *
-   * @group double_funcs
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
    */
-  def atan(e: Column): Column = Atan(e.expr)
+  def atan2(l: Double, r: Column): Column = atan2(lit(l).expr, r)
 
   /**
-   * Computes the tangent inverse of the given column.
-   *
-   * @group double_funcs
+   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+   * polar coordinates (r, theta).
    */
-  def atan(columnName: String): Column = atan(Column(columnName))
+  def atan2(l: Double, rightName: String): Column = atan2(l, Column(rightName))
 
   /**
-   * Computes the hyperbolic tangent of the given value.
-   *
-   * @group double_funcs
+   * Computes the cube-root of the given value.
    */
-  def tanh(e: Column): Column = Tanh(e.expr)
+  def cbrt(e: Column): Column = Cbrt(e.expr)
 
   /**
-   * Computes the hyperbolic tangent of the given column.
-   *
-   * @group double_funcs
+   * Computes the cube-root of the given column.
    */
-  def tanh(columnName: String): Column = tanh(Column(columnName))
+  def cbrt(columnName: String): Column = cbrt(Column(columnName))
 
   /**
-   * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
-   *
-   * @group double_funcs
+   * Computes the ceiling of the given value.
    */
-  def toDeg(e: Column): Column = ToDegrees(e.expr)
+  def ceil(e: Column): Column = Ceil(e.expr)
 
   /**
-   * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
-   *
-   * @group double_funcs
+   * Computes the ceiling of the given column.
    */
-  def toDeg(columnName: String): Column = toDeg(Column(columnName))
+  def ceil(columnName: String): Column = ceil(Column(columnName))
 
   /**
-   * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
-   *
-   * @group double_funcs
+   * Computes the cosine of the given value.
    */
-  def toRad(e: Column): Column = ToRadians(e.expr)
+  def cos(e: Column): Column = Cos(e.expr)
 
   /**
-   * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
-   *
-   * @group double_funcs
+   * Computes the cosine of the given column.
    */
-  def toRad(columnName: String): Column = toRad(Column(columnName))
+  def cos(columnName: String): Column = cos(Column(columnName))
 
   /**
-   * Computes the ceiling of the given value.
-   *
-   * @group double_funcs
+   * Computes the hyperbolic cosine of the given value.
    */
-  def ceil(e: Column): Column = Ceil(e.expr)
+  def cosh(e: Column): Column = Cosh(e.expr)
 
   /**
-   * Computes the ceiling of the given column.
-   *
-   * @group double_funcs
+   * Computes the hyperbolic cosine of the given column.
    */
-  def ceil(columnName: String): Column = ceil(Column(columnName))
+  def cosh(columnName: String): Column = cosh(Column(columnName))
 
   /**
-   * Computes the floor of the given value.
-   *
-   * @group double_funcs
+   * Computes the exponential of the given value.
    */
-  def floor(e: Column): Column = Floor(e.expr)
+  def exp(e: Column): Column = Exp(e.expr)
 
   /**
-   * Computes the floor of the given column.
-   *
-   * @group double_funcs
+   * Computes the exponential of the given column.
    */
-  def floor(columnName: String): Column = floor(Column(columnName))
+  def exp(columnName: String): Column = exp(Column(columnName))
 
   /**
-   * Returns the double value that is closest in value to the argument and
-   * is equal to a mathematical integer.
-   *
-   * @group double_funcs
+   * Computes the exponential of the given value minus one.
    */
-  def rint(e: Column): Column = Rint(e.expr)
+  def expm1(e: Column): Column = Expm1(e.expr)
 
   /**
-   * Returns the double value that is closest in value to the argument and
-   * is equal to a mathematical integer.
-   *
-   * @group double_funcs
+   * Computes the exponential of the given column.
    */
-  def rint(columnName: String): Column = rint(Column(columnName))
+  def expm1(columnName: String): Column = expm1(Column(columnName))
 
   /**
-   * Computes the cube-root of the given value.
-   *
-   * @group double_funcs
+   * Computes the floor of the given value.
    */
-  def cbrt(e: Column): Column = Cbrt(e.expr)
+  def floor(e: Column): Column = Floor(e.expr)
 
   /**
-   * Computes the cube-root of the given column.
-   *
-   * @group double_funcs
+   * Computes the floor of the given column.
    */
-  def cbrt(columnName: String): Column = cbrt(Column(columnName))
+  def floor(columnName: String): Column = floor(Column(columnName))
 
   /**
-   * Computes the signum of the given value.
-   *
-   * @group double_funcs
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
    */
-  def signum(e: Column): Column = Signum(e.expr)
+  def hypot(l: Column, r: Column): Column = Hypot(l.expr, r.expr)
 
   /**
-   * Computes the signum of the given column.
-   *
-   * @group double_funcs
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
    */
-  def signum(columnName: String): Column = signum(Column(columnName))
+  def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName))
 
   /**
-   * Computes the signum of the given value. For IntegerType.
-   *
-   * @group int_funcs
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
    */
-  def isignum(e: Column): Column = ISignum(e.expr)
+  def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r)
 
   /**
-   * Computes the signum of the given column. For IntegerType.
-   *
-   * @group int_funcs
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
    */
-  def isignum(columnName: String): Column = isignum(Column(columnName))
+  def hypot(leftName: String, rightName: String): Column =
+    hypot(Column(leftName), Column(rightName))
 
   /**
-   * Computes the signum of the given value. For FloatType.
-   *
-   * @group float_funcs
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
    */
-  def fsignum(e: Column): Column = FSignum(e.expr)
+  def hypot(l: Column, r: Double): Column = hypot(l, lit(r).expr)
 
   /**
-   * Computes the signum of the given column. For FloatType.
-   *
-   * @group float_funcs
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
    */
-  def fsignum(columnName: String): Column = fsignum(Column(columnName))
+  def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r)
 
   /**
-   * Computes the signum of the given value. For LongType.
-   *
-   * @group long_funcs
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
    */
-  def lsignum(e: Column): Column = LSignum(e.expr)
+  def hypot(l: Double, r: Column): Column = hypot(lit(l).expr, r)
 
   /**
-   * Computes the signum of the given column. For FloatType.
-   *
-   * @group long_funcs
+   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
    */
-  def lsignum(columnName: String): Column = lsignum(Column(columnName))
+  def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
 
   /**
    * Computes the natural logarithm of the given value.
-   *
-   * @group double_funcs
    */
   def log(e: Column): Column = Log(e.expr)
 
   /**
    * Computes the natural logarithm of the given column.
-   *
-   * @group double_funcs
    */
   def log(columnName: String): Column = log(Column(columnName))
 
   /**
    * Computes the logarithm of the given value in Base 10.
-   *
-   * @group double_funcs
    */
   def log10(e: Column): Column = Log10(e.expr)
 
   /**
    * Computes the logarithm of the given value in Base 10.
-   *
-   * @group double_funcs
    */
   def log10(columnName: String): Column = log10(Column(columnName))
 
   /**
    * Computes the natural logarithm of the given value plus one.
-   *
-   * @group double_funcs
    */
   def log1p(e: Column): Column = Log1p(e.expr)
 
   /**
    * Computes the natural logarithm of the given column plus one.
-   *
-   * @group double_funcs
    */
   def log1p(columnName: String): Column = log1p(Column(columnName))
 
   /**
-   * Computes the exponential of the given value.
-   *
-   * @group double_funcs
-   */
-  def exp(e: Column): Column = Exp(e.expr)
-
-  /**
-   * Computes the exponential of the given column.
-   *
-   * @group double_funcs
-   */
-  def exp(columnName: String): Column = exp(Column(columnName))
-
-  /**
-   * Computes the exponential of the given value minus one.
-   *
-   * @group double_funcs
-   */
-  def expm1(e: Column): Column = Expm1(e.expr)
-
-  /**
-   * Computes the exponential of the given column.
-   *
-   * @group double_funcs
-   */
-  def expm1(columnName: String): Column = expm1(Column(columnName))
-
-  /**
    * Returns the value of the first argument raised to the power of the second argument.
-   *
-   * @group double_funcs
    */
   def pow(l: Column, r: Column): Column = Pow(l.expr, r.expr)
 
   /**
    * Returns the value of the first argument raised to the power of the second argument.
-   *
-   * @group double_funcs
    */
   def pow(l: Column, rightName: String): Column = pow(l, Column(rightName))
 
   /**
    * Returns the value of the first argument raised to the power of the second argument.
-   *
-   * @group double_funcs
    */
   def pow(leftName: String, r: Column): Column = pow(Column(leftName), r)
 
   /**
    * Returns the value of the first argument raised to the power of the second argument.
-   *
-   * @group double_funcs
    */
   def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName))
 
   /**
    * Returns the value of the first argument raised to the power of the second argument.
-   *
-   * @group double_funcs
    */
   def pow(l: Column, r: Double): Column = pow(l, lit(r).expr)
 
   /**
    * Returns the value of the first argument raised to the power of the second argument.
-   *
-   * @group double_funcs
    */
   def pow(leftName: String, r: Double): Column = pow(Column(leftName), r)
 
   /**
    * Returns the value of the first argument raised to the power of the second argument.
-   *
-   * @group double_funcs
    */
   def pow(l: Double, r: Column): Column = pow(lit(l).expr, r)
 
   /**
    * Returns the value of the first argument raised to the power of the second argument.
-   *
-   * @group double_funcs
    */
   def pow(l: Double, rightName: String): Column = pow(l, Column(rightName))
 
   /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   *
-   * @group double_funcs
+   * Returns the double value that is closest in value to the argument and
+   * is equal to a mathematical integer.
    */
-  def hypot(l: Column, r: Column): Column = Hypot(l.expr, r.expr)
+  def rint(e: Column): Column = Rint(e.expr)
 
   /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   *
-   * @group double_funcs
+   * Returns the double value that is closest in value to the argument and
+   * is equal to a mathematical integer.
    */
-  def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName))
+  def rint(columnName: String): Column = rint(Column(columnName))
 
   /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   *
-   * @group double_funcs
+   * Computes the signum of the given value.
    */
-  def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r)
+  def signum(e: Column): Column = Signum(e.expr)
 
   /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   *
-   * @group double_funcs
+   * Computes the signum of the given column.
    */
-  def hypot(leftName: String, rightName: String): Column =
-    hypot(Column(leftName), Column(rightName))
+  def signum(columnName: String): Column = signum(Column(columnName))
 
   /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   *
-   * @group double_funcs
+   * Computes the sine of the given value.
    */
-  def hypot(l: Column, r: Double): Column = hypot(l, lit(r).expr)
+  def sin(e: Column): Column = Sin(e.expr)
 
   /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   *
-   * @group double_funcs
+   * Computes the sine of the given column.
    */
-  def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r)
+  def sin(columnName: String): Column = sin(Column(columnName))
 
   /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   *
-   * @group double_funcs
+   * Computes the hyperbolic sine of the given value.
    */
-  def hypot(l: Double, r: Column): Column = hypot(lit(l).expr, r)
+  def sinh(e: Column): Column = Sinh(e.expr)
 
   /**
-   * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-   *
-   * @group double_funcs
+   * Computes the hyperbolic sine of the given column.
    */
-  def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
-
+  def sinh(columnName: String): Column = sinh(Column(columnName))
+  
   /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   *
-   * @group double_funcs
+   * Computes the tangent of the given value.
    */
-  def atan2(l: Column, r: Column): Column = Atan2(l.expr, r.expr)
+  def tan(e: Column): Column = Tan(e.expr)
 
   /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   *
-   * @group double_funcs
+   * Computes the tangent of the given column.
    */
-  def atan2(l: Column, rightName: String): Column = atan2(l, Column(rightName))
-
+  def tan(columnName: String): Column = tan(Column(columnName))
+  
   /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   *
-   * @group double_funcs
+   * Computes the hyperbolic tangent of the given value.
    */
-  def atan2(leftName: String, r: Column): Column = atan2(Column(leftName), r)
+  def tanh(e: Column): Column = Tanh(e.expr)
 
   /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   *
-   * @group double_funcs
+   * Computes the hyperbolic tangent of the given column.
    */
-  def atan2(leftName: String, rightName: String): Column =
-    atan2(Column(leftName), Column(rightName))
+  def tanh(columnName: String): Column = tanh(Column(columnName))
 
   /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   *
-   * @group double_funcs
+   * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
    */
-  def atan2(l: Column, r: Double): Column = atan2(l, lit(r).expr)
+  def toDeg(e: Column): Column = ToDegrees(e.expr)
 
   /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   *
-   * @group double_funcs
+   * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
    */
-  def atan2(leftName: String, r: Double): Column = atan2(Column(leftName), r)
+  def toDeg(columnName: String): Column = toDeg(Column(columnName))
 
   /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   *
-   * @group double_funcs
+   * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
    */
-  def atan2(l: Double, r: Column): Column = atan2(lit(l).expr, r)
+  def toRad(e: Column): Column = ToRadians(e.expr)
 
   /**
-   * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-   * polar coordinates (r, theta).
-   *
-   * @group double_funcs
+   * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
    */
-  def atan2(l: Double, rightName: String): Column = atan2(l, Column(rightName))
+  def toRad(columnName: String): Column = toRad(Column(columnName))
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
index 561553cc92..9e19bb7482 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
@@ -194,18 +194,6 @@ class MathExpressionsSuite extends QueryTest {
     testOneToOneMathFunction[Double](signum, math.signum)
   }
 
-  test("isignum") {
-    testOneToOneMathFunction[Int](isignum, math.signum)
-  }
-
-  test("fsignum") {
-    testOneToOneMathFunction[Float](fsignum, math.signum)
-  }
-
-  test("lsignum") {
-    testOneToOneMathFunction[Long](lsignum, math.signum)
-  }
-
   test("pow") {
     testTwoToOneMathFunction(pow, pow, math.pow)
   }