diff options
Diffstat (limited to 'python/pyspark/sql')
| -rw-r--r-- | python/pyspark/sql/dataframe.py | 8 | ||||
| -rw-r--r-- | python/pyspark/sql/functions.py | 8 | ||||
| -rw-r--r-- | python/pyspark/sql/group.py | 2 | ||||
| -rw-r--r-- | python/pyspark/sql/session.py | 13 | ||||
| -rw-r--r-- | python/pyspark/sql/types.py | 4 |
5 files changed, 19 insertions, 16 deletions
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index e44b01bba9..a0ac7a9342 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -1045,10 +1045,10 @@ class DataFrame(object): :func:`drop_duplicates` is an alias for :func:`dropDuplicates`. >>> from pyspark.sql import Row - >>> df = sc.parallelize([ \ - Row(name='Alice', age=5, height=80), \ - Row(name='Alice', age=5, height=80), \ - Row(name='Alice', age=10, height=80)]).toDF() + >>> df = sc.parallelize([ \\ + ... Row(name='Alice', age=5, height=80), \\ + ... Row(name='Alice', age=5, height=80), \\ + ... Row(name='Alice', age=10, height=80)]).toDF() >>> df.dropDuplicates().show() +---+------+-----+ |age|height| name| diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 7a7345170c..92d709ee40 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -1550,8 +1550,8 @@ def translate(srcCol, matching, replace): The translate will happen when any character in the string matching with the character in the `matching`. - >>> spark.createDataFrame([('translate',)], ['a']).select(translate('a', "rnlt", "123")\ - .alias('r')).collect() + >>> spark.createDataFrame([('translate',)], ['a']).select(translate('a', "rnlt", "123") \\ + ... .alias('r')).collect() [Row(r=u'1a2s3ae')] """ sc = SparkContext._active_spark_context @@ -1670,8 +1670,8 @@ def get_json_object(col, path): >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] >>> df = spark.createDataFrame(data, ("key", "jstring")) - >>> df.select(df.key, get_json_object(df.jstring, '$.f1').alias("c0"), \ - get_json_object(df.jstring, '$.f2').alias("c1") ).collect() + >>> df.select(df.key, get_json_object(df.jstring, '$.f1').alias("c0"), \\ + ... get_json_object(df.jstring, '$.f2').alias("c1") ).collect() [Row(key=u'1', c0=u'value1', c1=u'value2'), Row(key=u'2', c0=u'value12', c1=None)] """ sc = SparkContext._active_spark_context diff --git a/python/pyspark/sql/group.py b/python/pyspark/sql/group.py index a423206554..f2092f9c63 100644 --- a/python/pyspark/sql/group.py +++ b/python/pyspark/sql/group.py @@ -179,10 +179,12 @@ class GroupedData(object): :param values: List of values that will be translated to columns in the output DataFrame. # Compute the sum of earnings for each year by course with each course as a separate column + >>> df4.groupBy("year").pivot("course", ["dotNET", "Java"]).sum("earnings").collect() [Row(year=2012, dotNET=15000, Java=20000), Row(year=2013, dotNET=48000, Java=30000)] # Or without specifying column values (less efficient) + >>> df4.groupBy("year").pivot("course").sum("earnings").collect() [Row(year=2012, Java=20000, dotNET=15000), Row(year=2013, Java=30000, dotNET=48000)] """ diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 55f86a16f5..a360fbefa4 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -66,12 +66,11 @@ class SparkSession(object): tables, execute SQL over tables, cache tables, and read parquet files. To create a SparkSession, use the following builder pattern: - >>> spark = SparkSession.builder \ - .master("local") \ - .appName("Word Count") \ - .config("spark.some.config.option", "some-value") \ - .getOrCreate() - + >>> spark = SparkSession.builder \\ + ... .master("local") \\ + ... .appName("Word Count") \\ + ... .config("spark.some.config.option", "some-value") \\ + ... .getOrCreate() """ class Builder(object): @@ -87,11 +86,13 @@ class SparkSession(object): both :class:`SparkConf` and :class:`SparkSession`'s own configuration. For an existing SparkConf, use `conf` parameter. + >>> from pyspark.conf import SparkConf >>> SparkSession.builder.config(conf=SparkConf()) <pyspark.sql.session... For a (key, value) pair, you can omit parameter names. + >>> SparkSession.builder.config("spark.some.config.option", "some-value") <pyspark.sql.session... diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index a3679873e1..eea80684e2 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -486,8 +486,8 @@ class StructType(DataType): DataType object. >>> struct1 = StructType().add("f1", StringType(), True).add("f2", StringType(), True, None) - >>> struct2 = StructType([StructField("f1", StringType(), True),\ - StructField("f2", StringType(), True, None)]) + >>> struct2 = StructType([StructField("f1", StringType(), True), \\ + ... StructField("f2", StringType(), True, None)]) >>> struct1 == struct2 True >>> struct1 = StructType().add(StructField("f1", StringType(), True)) |