diff options
| author | JD <jd@csh.rit.edu> | 2015-07-25 00:34:59 -0700 |
|---|---|---|
| committer | Reynold Xin <rxin@databricks.com> | 2015-07-25 00:34:59 -0700 |
| commit | 723db13e0688bf20e2a5f02ad170397c3a287712 (patch) | |
| tree | f2c4d9315ff02e64814731d398c0f9bad5a9e937 /python | |
| parent | 19bcd6ab12bf355bc5d774905ec7fe3b5fc8e0e2 (diff) | |
| download | spark-723db13e0688bf20e2a5f02ad170397c3a287712.tar.gz spark-723db13e0688bf20e2a5f02ad170397c3a287712.tar.bz2 spark-723db13e0688bf20e2a5f02ad170397c3a287712.zip | |
[Spark-8668][SQL] Adding expr to functions
Author: JD <jd@csh.rit.edu>
Author: Joseph Batchik <josephbatchik@gmail.com>
Closes #7606 from JDrit/expr and squashes the following commits:
ad7f607 [Joseph Batchik] fixing python linter error
9d6daea [Joseph Batchik] removed order by per @rxin's comment
707d5c6 [Joseph Batchik] Added expr to fuctions.py
79df83c [JD] added example to the docs
b89eec8 [JD] moved function up as per @rxin's comment
4960909 [JD] updated per @JoshRosen's comment
2cb329c [JD] updated per @rxin's comment
9a9ad0c [JD] removing unused import
6dc26d0 [JD] removed split
7f2222c [JD] Adding expr function as per SPARK-8668
Diffstat (limited to 'python')
| -rw-r--r-- | python/pyspark/sql/functions.py | 10 | ||||
| -rw-r--r-- | python/pyspark/sql/tests.py | 7 |
2 files changed, 17 insertions, 0 deletions
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 719e623a1a..d930f7db25 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -541,6 +541,16 @@ def sparkPartitionId(): return Column(sc._jvm.functions.sparkPartitionId()) +def expr(str): + """Parses the expression string into the column that it represents + + >>> df.select(expr("length(name)")).collect() + [Row('length(name)=5), Row('length(name)=3)] + """ + sc = SparkContext._active_spark_context + return Column(sc._jvm.functions.expr(str)) + + @ignore_unicode_prefix @since(1.5) def length(col): diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py index ea821f486f..5aa6135dc1 100644 --- a/python/pyspark/sql/tests.py +++ b/python/pyspark/sql/tests.py @@ -846,6 +846,13 @@ class SQLTests(ReusedPySparkTestCase): result = df.select(functions.bitwiseNOT(df.b)).collect()[0].asDict() self.assertEqual(~75, result['~b']) + def test_expr(self): + from pyspark.sql import functions + row = Row(a="length string", b=75) + df = self.sqlCtx.createDataFrame([row]) + result = df.select(functions.expr("length(a)")).collect()[0].asDict() + self.assertEqual(13, result["'length(a)"]) + def test_replace(self): schema = StructType([ StructField("name", StringType(), True), |