diff options
| author | Jian Feng <jzhang.chs@gmail.com> | 2015-09-21 23:36:41 -0700 |
|---|---|---|
| committer | Reynold Xin <rxin@databricks.com> | 2015-09-21 23:36:41 -0700 |
| commit | 0180b849dbaf191826231eda7dfaaf146a19602b (patch) | |
| tree | e04903f9aed018f9891760f0d5c7c00c8b6c7e35 /python/pyspark/sql/functions.py | |
| parent | bf20d6c9f9e478a5de24b45bbafd4dd89666c4cf (diff) | |
| download | spark-0180b849dbaf191826231eda7dfaaf146a19602b.tar.gz spark-0180b849dbaf191826231eda7dfaaf146a19602b.tar.bz2 spark-0180b849dbaf191826231eda7dfaaf146a19602b.zip | |
[SPARK-10577] [PYSPARK] DataFrame hint for broadcast join
https://issues.apache.org/jira/browse/SPARK-10577
Author: Jian Feng <jzhang.chs@gmail.com>
Closes #8801 from Jianfeng-chs/master.
Diffstat (limited to 'python/pyspark/sql/functions.py')
| -rw-r--r-- | python/pyspark/sql/functions.py | 9 |
1 files changed, 9 insertions, 0 deletions
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 26b8662718..fa04f4cd83 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -29,6 +29,7 @@ from pyspark.rdd import _prepare_for_python_RDD, ignore_unicode_prefix from pyspark.serializers import PickleSerializer, AutoBatchedSerializer from pyspark.sql.types import StringType from pyspark.sql.column import Column, _to_java_column, _to_seq +from pyspark.sql.dataframe import DataFrame def _create_function(name, doc=""): @@ -189,6 +190,14 @@ def approxCountDistinct(col, rsd=None): return Column(jc) +@since(1.6) +def broadcast(df): + """Marks a DataFrame as small enough for use in broadcast joins.""" + + sc = SparkContext._active_spark_context + return DataFrame(sc._jvm.functions.broadcast(df._jdf), df.sql_ctx) + + @since(1.4) def coalesce(*cols): """Returns the first column that is not null. |