diff options
author | 0x0FFF <programmerag@gmail.com> | 2015-09-11 15:19:04 -0700 |
---|---|---|
committer | Davies Liu <davies.liu@gmail.com> | 2015-09-11 15:19:04 -0700 |
commit | c34fc19765bdf55365cdce78d9ba11b220b73bb6 (patch) | |
tree | 0562a581bc5d7d9f79ea174754065dfdcbefafdc /python/pyspark/sql/column.py | |
parent | d74c6a143cbd060c25bf14a8d306841b3ec55d03 (diff) | |
download | spark-c34fc19765bdf55365cdce78d9ba11b220b73bb6.tar.gz spark-c34fc19765bdf55365cdce78d9ba11b220b73bb6.tar.bz2 spark-c34fc19765bdf55365cdce78d9ba11b220b73bb6.zip |
[SPARK-9014] [SQL] Allow Python spark API to use built-in exponential operator
This PR addresses (SPARK-9014)[https://issues.apache.org/jira/browse/SPARK-9014]
Added functionality: `Column` object in Python now supports exponential operator `**`
Example:
```
from pyspark.sql import *
df = sqlContext.createDataFrame([Row(a=2)])
df.select(3**df.a,df.a**3,df.a**df.a).collect()
```
Outputs:
```
[Row(POWER(3.0, a)=9.0, POWER(a, 3.0)=8.0, POWER(a, a)=4.0)]
```
Author: 0x0FFF <programmerag@gmail.com>
Closes #8658 from 0x0FFF/SPARK-9014.
Diffstat (limited to 'python/pyspark/sql/column.py')
-rw-r--r-- | python/pyspark/sql/column.py | 13 |
1 files changed, 13 insertions, 0 deletions
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py index 573f65f5bf..9ca8e1f264 100644 --- a/python/pyspark/sql/column.py +++ b/python/pyspark/sql/column.py @@ -91,6 +91,17 @@ def _func_op(name, doc=''): return _ +def _bin_func_op(name, reverse=False, doc="binary function"): + def _(self, other): + sc = SparkContext._active_spark_context + fn = getattr(sc._jvm.functions, name) + jc = other._jc if isinstance(other, Column) else _create_column_from_literal(other) + njc = fn(self._jc, jc) if not reverse else fn(jc, self._jc) + return Column(njc) + _.__doc__ = doc + return _ + + def _bin_op(name, doc="binary operator"): """ Create a method for given binary operator """ @@ -151,6 +162,8 @@ class Column(object): __rdiv__ = _reverse_op("divide") __rtruediv__ = _reverse_op("divide") __rmod__ = _reverse_op("mod") + __pow__ = _bin_func_op("pow") + __rpow__ = _bin_func_op("pow", reverse=True) # logistic operators __eq__ = _bin_op("equalTo") |