<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>pyspark.sql.dataframe — PySpark 1.5.0 documentation</title>
<link rel="stylesheet" href="../../../_static/nature.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../../../',
VERSION: '1.5.0',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../_static/doctools.js"></script>
<link rel="top" title="PySpark 1.5.0 documentation" href="../../../index.html" />
<link rel="up" title="Module code" href="../../index.html" />
</head>
<body role="document">
<div class="related" role="navigation" aria-label="related navigation">
<h3>Navigation</h3>
<ul>
<li class="nav-item nav-item-0"><a href="../../../index.html">PySpark 1.5.0 documentation</a> »</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for pyspark.sql.dataframe</h1><div class="highlight"><pre>
<span class="c">#</span>
<span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
<span class="c"># this work for additional information regarding copyright ownership.</span>
<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
<span class="c"># (the "License"); you may not use this file except in compliance with</span>
<span class="c"># the License. You may obtain a copy of the License at</span>
<span class="c">#</span>
<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c">#</span>
<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c"># See the License for the specific language governing permissions and</span>
<span class="c"># limitations under the License.</span>
<span class="c">#</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">warnings</span>
<span class="kn">import</span> <span class="nn">random</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version</span> <span class="o">>=</span> <span class="s">'3'</span><span class="p">:</span>
<span class="nb">basestring</span> <span class="o">=</span> <span class="nb">unicode</span> <span class="o">=</span> <span class="nb">str</span>
<span class="nb">long</span> <span class="o">=</span> <span class="nb">int</span>
<span class="kn">from</span> <span class="nn">functools</span> <span class="kn">import</span> <span class="nb">reduce</span>
<span class="k">else</span><span class="p">:</span>
<span class="kn">from</span> <span class="nn">itertools</span> <span class="kn">import</span> <span class="n">imap</span> <span class="k">as</span> <span class="nb">map</span>
<span class="kn">from</span> <span class="nn">pyspark.rdd</span> <span class="kn">import</span> <span class="n">RDD</span><span class="p">,</span> <span class="n">_load_from_socket</span><span class="p">,</span> <span class="n">ignore_unicode_prefix</span>
<span class="kn">from</span> <span class="nn">pyspark.serializers</span> <span class="kn">import</span> <span class="n">BatchedSerializer</span><span class="p">,</span> <span class="n">PickleSerializer</span><span class="p">,</span> <span class="n">UTF8Deserializer</span>
<span class="kn">from</span> <span class="nn">pyspark.storagelevel</span> <span class="kn">import</span> <span class="n">StorageLevel</span>
<span class="kn">from</span> <span class="nn">pyspark.traceback_utils</span> <span class="kn">import</span> <span class="n">SCCallSiteSync</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="kn">import</span> <span class="n">since</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.types</span> <span class="kn">import</span> <span class="n">_parse_datatype_json_string</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.column</span> <span class="kn">import</span> <span class="n">Column</span><span class="p">,</span> <span class="n">_to_seq</span><span class="p">,</span> <span class="n">_to_java_column</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.readwriter</span> <span class="kn">import</span> <span class="n">DataFrameWriter</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.types</span> <span class="kn">import</span> <span class="o">*</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s">"DataFrame"</span><span class="p">,</span> <span class="s">"SchemaRDD"</span><span class="p">,</span> <span class="s">"DataFrameNaFunctions"</span><span class="p">,</span> <span class="s">"DataFrameStatFunctions"</span><span class="p">]</span>
<div class="viewcode-block" id="DataFrame"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame">[docs]</a><span class="k">class</span> <span class="nc">DataFrame</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="sd">"""A distributed collection of data grouped into named columns.</span>
<span class="sd"> A :class:`DataFrame` is equivalent to a relational table in Spark SQL,</span>
<span class="sd"> and can be created using various functions in :class:`SQLContext`::</span>
<span class="sd"> people = sqlContext.read.parquet("...")</span>
<span class="sd"> Once created, it can be manipulated using the various domain-specific-language</span>
<span class="sd"> (DSL) functions defined in: :class:`DataFrame`, :class:`Column`.</span>
<span class="sd"> To select a column from the data frame, use the apply method::</span>
<span class="sd"> ageCol = people.age</span>
<span class="sd"> A more concrete example::</span>
<span class="sd"> # To create DataFrame using SQLContext</span>
<span class="sd"> people = sqlContext.read.parquet("...")</span>
<span class="sd"> department = sqlContext.read.parquet("...")</span>
<span class="sd"> people.filter(people.age > 30).join(department, people.deptId == department.id)) \</span>
<span class="sd"> .groupBy(department.name, "gender").agg({"salary": "avg", "age": "max"})</span>
<span class="sd"> .. note:: Experimental</span>
<span class="sd"> .. versionadded:: 1.3</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jdf</span><span class="p">,</span> <span class="n">sql_ctx</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span> <span class="o">=</span> <span class="n">jdf</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span> <span class="o">=</span> <span class="n">sql_ctx</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_sc</span> <span class="o">=</span> <span class="n">sql_ctx</span> <span class="ow">and</span> <span class="n">sql_ctx</span><span class="o">.</span><span class="n">_sc</span>
<span class="bp">self</span><span class="o">.</span><span class="n">is_cached</span> <span class="o">=</span> <span class="bp">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_schema</span> <span class="o">=</span> <span class="bp">None</span> <span class="c"># initialized lazily</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_lazy_rdd</span> <span class="o">=</span> <span class="bp">None</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">rdd</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns the content as an :class:`pyspark.RDD` of :class:`Row`.</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_lazy_rdd</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">jrdd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">javaToPython</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_lazy_rdd</span> <span class="o">=</span> <span class="n">RDD</span><span class="p">(</span><span class="n">jrdd</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">BatchedSerializer</span><span class="p">(</span><span class="n">PickleSerializer</span><span class="p">()))</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_lazy_rdd</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="s">"1.3.1"</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">na</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns a :class:`DataFrameNaFunctions` for handling missing values.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrameNaFunctions</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">stat</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns a :class:`DataFrameStatFunctions` for statistic functions.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrameStatFunctions</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.toJSON"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.toJSON">[docs]</a> <span class="k">def</span> <span class="nf">toJSON</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">use_unicode</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
<span class="sd">"""Converts a :class:`DataFrame` into a :class:`RDD` of string.</span>
<span class="sd"> Each row is turned into a JSON document as one element in the returned RDD.</span>
<span class="sd"> >>> df.toJSON().first()</span>
<span class="sd"> u'{"age":2,"name":"Alice"}'</span>
<span class="sd"> """</span>
<span class="n">rdd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">toJSON</span><span class="p">()</span>
<span class="k">return</span> <span class="n">RDD</span><span class="p">(</span><span class="n">rdd</span><span class="o">.</span><span class="n">toJavaRDD</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">UTF8Deserializer</span><span class="p">(</span><span class="n">use_unicode</span><span class="p">))</span>
</div>
<div class="viewcode-block" id="DataFrame.saveAsParquetFile"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.saveAsParquetFile">[docs]</a> <span class="k">def</span> <span class="nf">saveAsParquetFile</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
<span class="sd">"""Saves the contents as a Parquet file, preserving the schema.</span>
<span class="sd"> .. note:: Deprecated in 1.4, use :func:`DataFrameWriter.parquet` instead.</span>
<span class="sd"> """</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s">"saveAsParquetFile is deprecated. Use write.parquet() instead."</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">saveAsParquetFile</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.registerTempTable"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.registerTempTable">[docs]</a> <span class="k">def</span> <span class="nf">registerTempTable</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
<span class="sd">"""Registers this RDD as a temporary table using the given name.</span>
<span class="sd"> The lifetime of this temporary table is tied to the :class:`SQLContext`</span>
<span class="sd"> that was used to create this :class:`DataFrame`.</span>
<span class="sd"> >>> df.registerTempTable("people")</span>
<span class="sd"> >>> df2 = sqlContext.sql("select * from people")</span>
<span class="sd"> >>> sorted(df.collect()) == sorted(df2.collect())</span>
<span class="sd"> True</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">registerTempTable</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DataFrame.registerAsTable"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.registerAsTable">[docs]</a> <span class="k">def</span> <span class="nf">registerAsTable</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> .. note:: Deprecated in 1.4, use :func:`registerTempTable` instead.</span>
<span class="sd"> """</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s">"Use registerTempTable instead of registerAsTable."</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">registerTempTable</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DataFrame.insertInto"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.insertInto">[docs]</a> <span class="k">def</span> <span class="nf">insertInto</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tableName</span><span class="p">,</span> <span class="n">overwrite</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""Inserts the contents of this :class:`DataFrame` into the specified table.</span>
<span class="sd"> .. note:: Deprecated in 1.4, use :func:`DataFrameWriter.insertInto` instead.</span>
<span class="sd"> """</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s">"insertInto is deprecated. Use write.insertInto() instead."</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">insertInto</span><span class="p">(</span><span class="n">tableName</span><span class="p">,</span> <span class="n">overwrite</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DataFrame.saveAsTable"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.saveAsTable">[docs]</a> <span class="k">def</span> <span class="nf">saveAsTable</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tableName</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s">"error"</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">):</span>
<span class="sd">"""Saves the contents of this :class:`DataFrame` to a data source as a table.</span>
<span class="sd"> .. note:: Deprecated in 1.4, use :func:`DataFrameWriter.saveAsTable` instead.</span>
<span class="sd"> """</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s">"insertInto is deprecated. Use write.saveAsTable() instead."</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">saveAsTable</span><span class="p">(</span><span class="n">tableName</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">mode</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.save"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.save">[docs]</a> <span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s">"error"</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">):</span>
<span class="sd">"""Saves the contents of the :class:`DataFrame` to a data source.</span>
<span class="sd"> .. note:: Deprecated in 1.4, use :func:`DataFrameWriter.save` instead.</span>
<span class="sd"> """</span>
<span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s">"insertInto is deprecated. Use write.save() instead."</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">mode</span><span class="p">,</span> <span class="o">**</span><span class="n">options</span><span class="p">)</span>
</div>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">write</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Interface for saving the content of the :class:`DataFrame` out into external storage.</span>
<span class="sd"> :return: :class:`DataFrameWriter`</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrameWriter</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">schema</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns the schema of this :class:`DataFrame` as a :class:`types.StructType`.</span>
<span class="sd"> >>> df.schema</span>
<span class="sd"> StructType(List(StructField(age,IntegerType,true),StructField(name,StringType,true)))</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schema</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_schema</span> <span class="o">=</span> <span class="n">_parse_datatype_json_string</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">json</span><span class="p">())</span>
<span class="k">except</span> <span class="ne">AttributeError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
<span class="s">"Unable to parse datatype from schema. </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">e</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schema</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.printSchema"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.printSchema">[docs]</a> <span class="k">def</span> <span class="nf">printSchema</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Prints out the schema in the tree format.</span>
<span class="sd"> >>> df.printSchema()</span>
<span class="sd"> root</span>
<span class="sd"> |-- age: integer (nullable = true)</span>
<span class="sd"> |-- name: string (nullable = true)</span>
<span class="sd"> <BLANKLINE></span>
<span class="sd"> """</span>
<span class="k">print</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">schema</span><span class="p">()</span><span class="o">.</span><span class="n">treeString</span><span class="p">())</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.explain"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.explain">[docs]</a> <span class="k">def</span> <span class="nf">explain</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">extended</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""Prints the (logical and physical) plans to the console for debugging purpose.</span>
<span class="sd"> :param extended: boolean, default ``False``. If ``False``, prints only the physical plan.</span>
<span class="sd"> >>> df.explain()</span>
<span class="sd"> Scan PhysicalRDD[age#0,name#1]</span>
<span class="sd"> >>> df.explain(True)</span>
<span class="sd"> == Parsed Logical Plan ==</span>
<span class="sd"> ...</span>
<span class="sd"> == Analyzed Logical Plan ==</span>
<span class="sd"> ...</span>
<span class="sd"> == Optimized Logical Plan ==</span>
<span class="sd"> ...</span>
<span class="sd"> == Physical Plan ==</span>
<span class="sd"> ...</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">extended</span><span class="p">:</span>
<span class="k">print</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">queryExecution</span><span class="p">()</span><span class="o">.</span><span class="n">toString</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">print</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">queryExecution</span><span class="p">()</span><span class="o">.</span><span class="n">executedPlan</span><span class="p">()</span><span class="o">.</span><span class="n">toString</span><span class="p">())</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.isLocal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.isLocal">[docs]</a> <span class="k">def</span> <span class="nf">isLocal</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns ``True`` if the :func:`collect` and :func:`take` methods can be run locally</span>
<span class="sd"> (without any Spark executors).</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">isLocal</span><span class="p">()</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.show"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.show">[docs]</a> <span class="k">def</span> <span class="nf">show</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span> <span class="n">truncate</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
<span class="sd">"""Prints the first ``n`` rows to the console.</span>
<span class="sd"> :param n: Number of rows to show.</span>
<span class="sd"> :param truncate: Whether truncate long strings and align cells right.</span>
<span class="sd"> >>> df</span>
<span class="sd"> DataFrame[age: int, name: string]</span>
<span class="sd"> >>> df.show()</span>
<span class="sd"> +---+-----+</span>
<span class="sd"> |age| name|</span>
<span class="sd"> +---+-----+</span>
<span class="sd"> | 2|Alice|</span>
<span class="sd"> | 5| Bob|</span>
<span class="sd"> +---+-----+</span>
<span class="sd"> """</span>
<span class="k">print</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">showString</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">truncate</span><span class="p">))</span>
</div>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"DataFrame[</span><span class="si">%s</span><span class="s">]"</span> <span class="o">%</span> <span class="p">(</span><span class="s">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s">"</span><span class="si">%s</span><span class="s">: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dtypes</span><span class="p">))</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.count"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.count">[docs]</a> <span class="k">def</span> <span class="nf">count</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns the number of rows in this :class:`DataFrame`.</span>
<span class="sd"> >>> df.count()</span>
<span class="sd"> 2</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">count</span><span class="p">())</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.collect"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.collect">[docs]</a> <span class="k">def</span> <span class="nf">collect</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns all the records as a list of :class:`Row`.</span>
<span class="sd"> >>> df.collect()</span>
<span class="sd"> [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
<span class="sd"> """</span>
<span class="k">with</span> <span class="n">SCCallSiteSync</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">)</span> <span class="k">as</span> <span class="n">css</span><span class="p">:</span>
<span class="n">port</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">PythonRDD</span><span class="o">.</span><span class="n">collectAndServe</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">javaToPython</span><span class="p">()</span><span class="o">.</span><span class="n">rdd</span><span class="p">())</span>
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="n">_load_from_socket</span><span class="p">(</span><span class="n">port</span><span class="p">,</span> <span class="n">BatchedSerializer</span><span class="p">(</span><span class="n">PickleSerializer</span><span class="p">())))</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.limit"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.limit">[docs]</a> <span class="k">def</span> <span class="nf">limit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num</span><span class="p">):</span>
<span class="sd">"""Limits the result count to the number specified.</span>
<span class="sd"> >>> df.limit(1).collect()</span>
<span class="sd"> [Row(age=2, name=u'Alice')]</span>
<span class="sd"> >>> df.limit(0).collect()</span>
<span class="sd"> []</span>
<span class="sd"> """</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="n">num</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.take"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.take">[docs]</a> <span class="k">def</span> <span class="nf">take</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num</span><span class="p">):</span>
<span class="sd">"""Returns the first ``num`` rows as a :class:`list` of :class:`Row`.</span>
<span class="sd"> >>> df.take(2)</span>
<span class="sd"> [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">limit</span><span class="p">(</span><span class="n">num</span><span class="p">)</span><span class="o">.</span><span class="n">collect</span><span class="p">()</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.map"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.map">[docs]</a> <span class="k">def</span> <span class="nf">map</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">):</span>
<span class="sd">""" Returns a new :class:`RDD` by applying a the ``f`` function to each :class:`Row`.</span>
<span class="sd"> This is a shorthand for ``df.rdd.map()``.</span>
<span class="sd"> >>> df.map(lambda p: p.name).collect()</span>
<span class="sd"> [u'Alice', u'Bob']</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rdd</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.flatMap"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.flatMap">[docs]</a> <span class="k">def</span> <span class="nf">flatMap</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">):</span>
<span class="sd">""" Returns a new :class:`RDD` by first applying the ``f`` function to each :class:`Row`,</span>
<span class="sd"> and then flattening the results.</span>
<span class="sd"> This is a shorthand for ``df.rdd.flatMap()``.</span>
<span class="sd"> >>> df.flatMap(lambda p: p.name).collect()</span>
<span class="sd"> [u'A', u'l', u'i', u'c', u'e', u'B', u'o', u'b']</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rdd</span><span class="o">.</span><span class="n">flatMap</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.mapPartitions"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.mapPartitions">[docs]</a> <span class="k">def</span> <span class="nf">mapPartitions</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">,</span> <span class="n">preservesPartitioning</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`RDD` by applying the ``f`` function to each partition.</span>
<span class="sd"> This is a shorthand for ``df.rdd.mapPartitions()``.</span>
<span class="sd"> >>> rdd = sc.parallelize([1, 2, 3, 4], 4)</span>
<span class="sd"> >>> def f(iterator): yield 1</span>
<span class="sd"> >>> rdd.mapPartitions(f).sum()</span>
<span class="sd"> 4</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rdd</span><span class="o">.</span><span class="n">mapPartitions</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">preservesPartitioning</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.foreach"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.foreach">[docs]</a> <span class="k">def</span> <span class="nf">foreach</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">):</span>
<span class="sd">"""Applies the ``f`` function to all :class:`Row` of this :class:`DataFrame`.</span>
<span class="sd"> This is a shorthand for ``df.rdd.foreach()``.</span>
<span class="sd"> >>> def f(person):</span>
<span class="sd"> ... print(person.name)</span>
<span class="sd"> >>> df.foreach(f)</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rdd</span><span class="o">.</span><span class="n">foreach</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.foreachPartition"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.foreachPartition">[docs]</a> <span class="k">def</span> <span class="nf">foreachPartition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">):</span>
<span class="sd">"""Applies the ``f`` function to each partition of this :class:`DataFrame`.</span>
<span class="sd"> This a shorthand for ``df.rdd.foreachPartition()``.</span>
<span class="sd"> >>> def f(people):</span>
<span class="sd"> ... for person in people:</span>
<span class="sd"> ... print(person.name)</span>
<span class="sd"> >>> df.foreachPartition(f)</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">rdd</span><span class="o">.</span><span class="n">foreachPartition</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.cache"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.cache">[docs]</a> <span class="k">def</span> <span class="nf">cache</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">""" Persists with the default storage level (C{MEMORY_ONLY_SER}).</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">is_cached</span> <span class="o">=</span> <span class="bp">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">cache</span><span class="p">()</span>
<span class="k">return</span> <span class="bp">self</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.persist"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.persist">[docs]</a> <span class="k">def</span> <span class="nf">persist</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">storageLevel</span><span class="o">=</span><span class="n">StorageLevel</span><span class="o">.</span><span class="n">MEMORY_ONLY_SER</span><span class="p">):</span>
<span class="sd">"""Sets the storage level to persist its values across operations</span>
<span class="sd"> after the first time it is computed. This can only be used to assign</span>
<span class="sd"> a new storage level if the RDD does not have a storage level set yet.</span>
<span class="sd"> If no storage level is specified defaults to (C{MEMORY_ONLY_SER}).</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">is_cached</span> <span class="o">=</span> <span class="bp">True</span>
<span class="n">javaStorageLevel</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="o">.</span><span class="n">_getJavaStorageLevel</span><span class="p">(</span><span class="n">storageLevel</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">persist</span><span class="p">(</span><span class="n">javaStorageLevel</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.unpersist"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.unpersist">[docs]</a> <span class="k">def</span> <span class="nf">unpersist</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">blocking</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
<span class="sd">"""Marks the :class:`DataFrame` as non-persistent, and remove all blocks for it from</span>
<span class="sd"> memory and disk.</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">is_cached</span> <span class="o">=</span> <span class="bp">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">unpersist</span><span class="p">(</span><span class="n">blocking</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.coalesce"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.coalesce">[docs]</a> <span class="k">def</span> <span class="nf">coalesce</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numPartitions</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a new :class:`DataFrame` that has exactly `numPartitions` partitions.</span>
<span class="sd"> Similar to coalesce defined on an :class:`RDD`, this operation results in a</span>
<span class="sd"> narrow dependency, e.g. if you go from 1000 partitions to 100 partitions,</span>
<span class="sd"> there will not be a shuffle, instead each of the 100 new partitions will</span>
<span class="sd"> claim 10 of the current partitions.</span>
<span class="sd"> >>> df.coalesce(1).rdd.getNumPartitions()</span>
<span class="sd"> 1</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">coalesce</span><span class="p">(</span><span class="n">numPartitions</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.repartition"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.repartition">[docs]</a> <span class="k">def</span> <span class="nf">repartition</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numPartitions</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`DataFrame` that has exactly ``numPartitions`` partitions.</span>
<span class="sd"> >>> df.repartition(10).rdd.getNumPartitions()</span>
<span class="sd"> 10</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">repartition</span><span class="p">(</span><span class="n">numPartitions</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.distinct"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.distinct">[docs]</a> <span class="k">def</span> <span class="nf">distinct</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`DataFrame` containing the distinct rows in this :class:`DataFrame`.</span>
<span class="sd"> >>> df.distinct().count()</span>
<span class="sd"> 2</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">distinct</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.sample"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.sample">[docs]</a> <span class="k">def</span> <span class="nf">sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">withReplacement</span><span class="p">,</span> <span class="n">fraction</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Returns a sampled subset of this :class:`DataFrame`.</span>
<span class="sd"> >>> df.sample(False, 0.5, 42).count()</span>
<span class="sd"> 1</span>
<span class="sd"> """</span>
<span class="k">assert</span> <span class="n">fraction</span> <span class="o">>=</span> <span class="mf">0.0</span><span class="p">,</span> <span class="s">"Negative fraction value: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">fraction</span>
<span class="n">seed</span> <span class="o">=</span> <span class="n">seed</span> <span class="k">if</span> <span class="n">seed</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="k">else</span> <span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">sys</span><span class="o">.</span><span class="n">maxsize</span><span class="p">)</span>
<span class="n">rdd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="n">withReplacement</span><span class="p">,</span> <span class="n">fraction</span><span class="p">,</span> <span class="nb">long</span><span class="p">(</span><span class="n">seed</span><span class="p">))</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.5</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.sampleBy"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.sampleBy">[docs]</a> <span class="k">def</span> <span class="nf">sampleBy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">fractions</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a stratified sample without replacement based on the</span>
<span class="sd"> fraction given on each stratum.</span>
<span class="sd"> :param col: column that defines strata</span>
<span class="sd"> :param fractions:</span>
<span class="sd"> sampling fraction for each stratum. If a stratum is not</span>
<span class="sd"> specified, we treat its fraction as zero.</span>
<span class="sd"> :param seed: random seed</span>
<span class="sd"> :return: a new DataFrame that represents the stratified sample</span>
<span class="sd"> >>> from pyspark.sql.functions import col</span>
<span class="sd"> >>> dataset = sqlContext.range(0, 100).select((col("id") % 3).alias("key"))</span>
<span class="sd"> >>> sampled = dataset.sampleBy("key", fractions={0: 0.1, 1: 0.2}, seed=0)</span>
<span class="sd"> >>> sampled.groupBy("key").count().orderBy("key").show()</span>
<span class="sd"> +---+-----+</span>
<span class="sd"> |key|count|</span>
<span class="sd"> +---+-----+</span>
<span class="sd"> | 0| 3|</span>
<span class="sd"> | 1| 8|</span>
<span class="sd"> +---+-----+</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"col must be a string, but got </span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">col</span><span class="p">))</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">fractions</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"fractions must be a dict but got </span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">fractions</span><span class="p">))</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">fractions</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"key must be float, int, long, or string, but got </span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">k</span><span class="p">))</span>
<span class="n">fractions</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
<span class="n">seed</span> <span class="o">=</span> <span class="n">seed</span> <span class="k">if</span> <span class="n">seed</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="k">else</span> <span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">sys</span><span class="o">.</span><span class="n">maxsize</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">sampleBy</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jmap</span><span class="p">(</span><span class="n">fractions</span><span class="p">),</span> <span class="n">seed</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.randomSplit"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.randomSplit">[docs]</a> <span class="k">def</span> <span class="nf">randomSplit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">weights</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Randomly splits this :class:`DataFrame` with the provided weights.</span>
<span class="sd"> :param weights: list of doubles as weights with which to split the DataFrame. Weights will</span>
<span class="sd"> be normalized if they don't sum up to 1.0.</span>
<span class="sd"> :param seed: The seed for sampling.</span>
<span class="sd"> >>> splits = df4.randomSplit([1.0, 2.0], 24)</span>
<span class="sd"> >>> splits[0].count()</span>
<span class="sd"> 1</span>
<span class="sd"> >>> splits[1].count()</span>
<span class="sd"> 3</span>
<span class="sd"> """</span>
<span class="k">for</span> <span class="n">w</span> <span class="ow">in</span> <span class="n">weights</span><span class="p">:</span>
<span class="k">if</span> <span class="n">w</span> <span class="o"><</span> <span class="mf">0.0</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Weights must be positive. Found weight value: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">w</span><span class="p">)</span>
<span class="n">seed</span> <span class="o">=</span> <span class="n">seed</span> <span class="k">if</span> <span class="n">seed</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="k">else</span> <span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">sys</span><span class="o">.</span><span class="n">maxsize</span><span class="p">)</span>
<span class="n">rdd_array</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">randomSplit</span><span class="p">(</span><span class="n">_to_seq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">weights</span><span class="p">),</span> <span class="nb">long</span><span class="p">(</span><span class="n">seed</span><span class="p">))</span>
<span class="k">return</span> <span class="p">[</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">rdd</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span> <span class="k">for</span> <span class="n">rdd</span> <span class="ow">in</span> <span class="n">rdd_array</span><span class="p">]</span>
</div>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">dtypes</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns all column names and their data types as a list.</span>
<span class="sd"> >>> df.dtypes</span>
<span class="sd"> [('age', 'int'), ('name', 'string')]</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="p">[(</span><span class="nb">str</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">),</span> <span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="o">.</span><span class="n">simpleString</span><span class="p">())</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">fields</span><span class="p">]</span>
<span class="nd">@property</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">columns</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns all column names as a list.</span>
<span class="sd"> >>> df.columns</span>
<span class="sd"> ['age', 'name']</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">fields</span><span class="p">]</span>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.alias"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.alias">[docs]</a> <span class="k">def</span> <span class="nf">alias</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">alias</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`DataFrame` with an alias set.</span>
<span class="sd"> >>> from pyspark.sql.functions import *</span>
<span class="sd"> >>> df_as1 = df.alias("df_as1")</span>
<span class="sd"> >>> df_as2 = df.alias("df_as2")</span>
<span class="sd"> >>> joined_df = df_as1.join(df_as2, col("df_as1.name") == col("df_as2.name"), 'inner')</span>
<span class="sd"> >>> joined_df.select(col("df_as1.name"), col("df_as2.name"), col("df_as2.age")).collect()</span>
<span class="sd"> [Row(name=u'Alice', name=u'Alice', age=2), Row(name=u'Bob', name=u'Bob', age=5)]</span>
<span class="sd"> """</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">alias</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">),</span> <span class="s">"alias should be a string"</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="p">,</span> <span class="s">"as"</span><span class="p">)(</span><span class="n">alias</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.join"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.join">[docs]</a> <span class="k">def</span> <span class="nf">join</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Joins with another :class:`DataFrame`, using the given join expression.</span>
<span class="sd"> The following performs a full outer join between ``df1`` and ``df2``.</span>
<span class="sd"> :param other: Right side of the join</span>
<span class="sd"> :param on: a string for join column name, a list of column names,</span>
<span class="sd"> , a join expression (Column) or a list of Columns.</span>
<span class="sd"> If `on` is a string or a list of string indicating the name of the join column(s),</span>
<span class="sd"> the column(s) must exist on both sides, and this performs an inner equi-join.</span>
<span class="sd"> :param how: str, default 'inner'.</span>
<span class="sd"> One of `inner`, `outer`, `left_outer`, `right_outer`, `semijoin`.</span>
<span class="sd"> >>> df.join(df2, df.name == df2.name, 'outer').select(df.name, df2.height).collect()</span>
<span class="sd"> [Row(name=None, height=80), Row(name=u'Alice', height=None), Row(name=u'Bob', height=85)]</span>
<span class="sd"> >>> cond = [df.name == df3.name, df.age == df3.age]</span>
<span class="sd"> >>> df.join(df3, cond, 'outer').select(df.name, df3.age).collect()</span>
<span class="sd"> [Row(name=u'Bob', age=5), Row(name=u'Alice', age=2)]</span>
<span class="sd"> >>> df.join(df2, 'name').select(df.name, df2.height).collect()</span>
<span class="sd"> [Row(name=u'Bob', height=85)]</span>
<span class="sd"> >>> df.join(df4, ['name', 'age']).select(df.name, df.age).collect()</span>
<span class="sd"> [Row(name=u'Bob', age=5)]</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">on</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">on</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">on</span> <span class="o">=</span> <span class="p">[</span><span class="n">on</span><span class="p">]</span>
<span class="k">if</span> <span class="n">on</span> <span class="ow">is</span> <span class="bp">None</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="n">on</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_jdf</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">on</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">on</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">on</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">Column</span><span class="p">),</span> <span class="s">"on should be Column or list of Column"</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">on</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
<span class="n">on</span> <span class="o">=</span> <span class="nb">reduce</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">__and__</span><span class="p">(</span><span class="n">y</span><span class="p">),</span> <span class="n">on</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">on</span> <span class="o">=</span> <span class="n">on</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">how</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_jdf</span><span class="p">,</span> <span class="n">on</span><span class="o">.</span><span class="n">_jc</span><span class="p">,</span> <span class="s">"inner"</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">how</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">),</span> <span class="s">"how should be basestring"</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_jdf</span><span class="p">,</span> <span class="n">on</span><span class="o">.</span><span class="n">_jc</span><span class="p">,</span> <span class="n">how</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.sort"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.sort">[docs]</a> <span class="k">def</span> <span class="nf">sort</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`DataFrame` sorted by the specified column(s).</span>
<span class="sd"> :param cols: list of :class:`Column` or column names to sort by.</span>
<span class="sd"> :param ascending: boolean or list of boolean (default True).</span>
<span class="sd"> Sort ascending vs. descending. Specify list for multiple sort orders.</span>
<span class="sd"> If a list is specified, length of the list must equal length of the `cols`.</span>
<span class="sd"> >>> df.sort(df.age.desc()).collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
<span class="sd"> >>> df.sort("age", ascending=False).collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
<span class="sd"> >>> df.orderBy(df.age.desc()).collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
<span class="sd"> >>> from pyspark.sql.functions import *</span>
<span class="sd"> >>> df.sort(asc("age")).collect()</span>
<span class="sd"> [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
<span class="sd"> >>> df.orderBy(desc("age"), "name").collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
<span class="sd"> >>> df.orderBy(["age", "name"], ascending=[0, 1]).collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob'), Row(age=2, name=u'Alice')]</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">cols</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"should sort by at least one column"</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cols</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">cols</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">jcols</span> <span class="o">=</span> <span class="p">[</span><span class="n">_to_java_column</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cols</span><span class="p">]</span>
<span class="n">ascending</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s">'ascending'</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ascending</span><span class="p">,</span> <span class="p">(</span><span class="nb">bool</span><span class="p">,</span> <span class="nb">int</span><span class="p">)):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">ascending</span><span class="p">:</span>
<span class="n">jcols</span> <span class="o">=</span> <span class="p">[</span><span class="n">jc</span><span class="o">.</span><span class="n">desc</span><span class="p">()</span> <span class="k">for</span> <span class="n">jc</span> <span class="ow">in</span> <span class="n">jcols</span><span class="p">]</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ascending</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">jcols</span> <span class="o">=</span> <span class="p">[</span><span class="n">jc</span> <span class="k">if</span> <span class="n">asc</span> <span class="k">else</span> <span class="n">jc</span><span class="o">.</span><span class="n">desc</span><span class="p">()</span>
<span class="k">for</span> <span class="n">asc</span><span class="p">,</span> <span class="n">jc</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">ascending</span><span class="p">,</span> <span class="n">jcols</span><span class="p">)]</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"ascending can only be boolean or list, but got </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">ascending</span><span class="p">))</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">jcols</span><span class="p">))</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="n">orderBy</span> <span class="o">=</span> <span class="n">sort</span>
<span class="k">def</span> <span class="nf">_jseq</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cols</span><span class="p">,</span> <span class="n">converter</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Return a JVM Seq of Columns from a list of Column or names"""</span>
<span class="k">return</span> <span class="n">_to_seq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">cols</span><span class="p">,</span> <span class="n">converter</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_jmap</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">jm</span><span class="p">):</span>
<span class="sd">"""Return a JVM Scala Map from a dict"""</span>
<span class="k">return</span> <span class="n">_to_scala_map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">jm</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_jcols</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">):</span>
<span class="sd">"""Return a JVM Seq of Columns from a list of Column or column names</span>
<span class="sd"> If `cols` has only one list in it, cols[0] will be used as the list.</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cols</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">cols</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">cols</span><span class="p">,</span> <span class="n">_to_java_column</span><span class="p">)</span>
<span class="nd">@since</span><span class="p">(</span><span class="s">"1.3.1"</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.describe"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.describe">[docs]</a> <span class="k">def</span> <span class="nf">describe</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">):</span>
<span class="sd">"""Computes statistics for numeric columns.</span>
<span class="sd"> This include count, mean, stddev, min, and max. If no columns are</span>
<span class="sd"> given, this function computes statistics for all numerical columns.</span>
<span class="sd"> .. note:: This function is meant for exploratory data analysis, as we make no \</span>
<span class="sd"> guarantee about the backward compatibility of the schema of the resulting DataFrame.</span>
<span class="sd"> >>> df.describe().show()</span>
<span class="sd"> +-------+---+</span>
<span class="sd"> |summary|age|</span>
<span class="sd"> +-------+---+</span>
<span class="sd"> | count| 2|</span>
<span class="sd"> | mean|3.5|</span>
<span class="sd"> | stddev|1.5|</span>
<span class="sd"> | min| 2|</span>
<span class="sd"> | max| 5|</span>
<span class="sd"> +-------+---+</span>
<span class="sd"> >>> df.describe(['age', 'name']).show()</span>
<span class="sd"> +-------+---+-----+</span>
<span class="sd"> |summary|age| name|</span>
<span class="sd"> +-------+---+-----+</span>
<span class="sd"> | count| 2| 2|</span>
<span class="sd"> | mean|3.5| null|</span>
<span class="sd"> | stddev|1.5| null|</span>
<span class="sd"> | min| 2|Alice|</span>
<span class="sd"> | max| 5| Bob|</span>
<span class="sd"> +-------+---+-----+</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">cols</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">cols</span> <span class="o">=</span> <span class="n">cols</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">describe</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">cols</span><span class="p">))</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.head"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.head">[docs]</a> <span class="k">def</span> <span class="nf">head</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">n</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Returns the first ``n`` rows.</span>
<span class="sd"> :param n: int, default 1. Number of rows to return.</span>
<span class="sd"> :return: If n is greater than 1, return a list of :class:`Row`.</span>
<span class="sd"> If n is 1, return a single Row.</span>
<span class="sd"> >>> df.head()</span>
<span class="sd"> Row(age=2, name=u'Alice')</span>
<span class="sd"> >>> df.head(1)</span>
<span class="sd"> [Row(age=2, name=u'Alice')]</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">n</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">rs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">head</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="k">return</span> <span class="n">rs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="n">rs</span> <span class="k">else</span> <span class="bp">None</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.first"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.first">[docs]</a> <span class="k">def</span> <span class="nf">first</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns the first row as a :class:`Row`.</span>
<span class="sd"> >>> df.first()</span>
<span class="sd"> Row(age=2, name=u'Alice')</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
<span class="sd">"""Returns the column as a :class:`Column`.</span>
<span class="sd"> >>> df.select(df['age']).collect()</span>
<span class="sd"> [Row(age=2), Row(age=5)]</span>
<span class="sd"> >>> df[ ["name", "age"]].collect()</span>
<span class="sd"> [Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]</span>
<span class="sd"> >>> df[ df.age > 3 ].collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob')]</span>
<span class="sd"> >>> df[df[0] > 3].collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob')]</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="n">jc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">Column</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="o">*</span><span class="n">item</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
<span class="n">jc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">columns</span><span class="p">[</span><span class="n">item</span><span class="p">])</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"unexpected item type: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">item</span><span class="p">))</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
<span class="sd">"""Returns the :class:`Column` denoted by ``name``.</span>
<span class="sd"> >>> df.select(df.age).collect()</span>
<span class="sd"> [Row(age=2), Row(age=5)]</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span>
<span class="s">"'</span><span class="si">%s</span><span class="s">' object has no attribute '</span><span class="si">%s</span><span class="s">'"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">,</span> <span class="n">name</span><span class="p">))</span>
<span class="n">jc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Column</span><span class="p">(</span><span class="n">jc</span><span class="p">)</span>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.select"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.select">[docs]</a> <span class="k">def</span> <span class="nf">select</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">):</span>
<span class="sd">"""Projects a set of expressions and returns a new :class:`DataFrame`.</span>
<span class="sd"> :param cols: list of column names (string) or expressions (:class:`Column`).</span>
<span class="sd"> If one of the column names is '*', that column is expanded to include all columns</span>
<span class="sd"> in the current DataFrame.</span>
<span class="sd"> >>> df.select('*').collect()</span>
<span class="sd"> [Row(age=2, name=u'Alice'), Row(age=5, name=u'Bob')]</span>
<span class="sd"> >>> df.select('name', 'age').collect()</span>
<span class="sd"> [Row(name=u'Alice', age=2), Row(name=u'Bob', age=5)]</span>
<span class="sd"> >>> df.select(df.name, (df.age + 10).alias('age')).collect()</span>
<span class="sd"> [Row(name=u'Alice', age=12), Row(name=u'Bob', age=15)]</span>
<span class="sd"> """</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">select</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jcols</span><span class="p">(</span><span class="o">*</span><span class="n">cols</span><span class="p">))</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.selectExpr"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.selectExpr">[docs]</a> <span class="k">def</span> <span class="nf">selectExpr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">expr</span><span class="p">):</span>
<span class="sd">"""Projects a set of SQL expressions and returns a new :class:`DataFrame`.</span>
<span class="sd"> This is a variant of :func:`select` that accepts SQL expressions.</span>
<span class="sd"> >>> df.selectExpr("age * 2", "abs(age)").collect()</span>
<span class="sd"> [Row((age * 2)=4, 'abs(age)=2), Row((age * 2)=10, 'abs(age)=5)]</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">expr</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">expr</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">expr</span> <span class="o">=</span> <span class="n">expr</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">selectExpr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">expr</span><span class="p">))</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.filter"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.filter">[docs]</a> <span class="k">def</span> <span class="nf">filter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">condition</span><span class="p">):</span>
<span class="sd">"""Filters rows using the given condition.</span>
<span class="sd"> :func:`where` is an alias for :func:`filter`.</span>
<span class="sd"> :param condition: a :class:`Column` of :class:`types.BooleanType`</span>
<span class="sd"> or a string of SQL expression.</span>
<span class="sd"> >>> df.filter(df.age > 3).collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob')]</span>
<span class="sd"> >>> df.where(df.age == 2).collect()</span>
<span class="sd"> [Row(age=2, name=u'Alice')]</span>
<span class="sd"> >>> df.filter("age > 3").collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob')]</span>
<span class="sd"> >>> df.where("age = 2").collect()</span>
<span class="sd"> [Row(age=2, name=u'Alice')]</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">condition</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">condition</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">condition</span><span class="p">,</span> <span class="n">Column</span><span class="p">):</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">condition</span><span class="o">.</span><span class="n">_jc</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"condition should be string or Column"</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="n">where</span> <span class="o">=</span> <span class="nb">filter</span>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.groupBy"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.groupBy">[docs]</a> <span class="k">def</span> <span class="nf">groupBy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">):</span>
<span class="sd">"""Groups the :class:`DataFrame` using the specified columns,</span>
<span class="sd"> so we can run aggregation on them. See :class:`GroupedData`</span>
<span class="sd"> for all the available aggregate functions.</span>
<span class="sd"> :func:`groupby` is an alias for :func:`groupBy`.</span>
<span class="sd"> :param cols: list of columns to group by.</span>
<span class="sd"> Each element should be a column name (string) or an expression (:class:`Column`).</span>
<span class="sd"> >>> df.groupBy().avg().collect()</span>
<span class="sd"> [Row(avg(age)=3.5)]</span>
<span class="sd"> >>> df.groupBy('name').agg({'age': 'mean'}).collect()</span>
<span class="sd"> [Row(name=u'Alice', avg(age)=2.0), Row(name=u'Bob', avg(age)=5.0)]</span>
<span class="sd"> >>> df.groupBy(df.name).avg().collect()</span>
<span class="sd"> [Row(name=u'Alice', avg(age)=2.0), Row(name=u'Bob', avg(age)=5.0)]</span>
<span class="sd"> >>> df.groupBy(['name', df.age]).count().collect()</span>
<span class="sd"> [Row(name=u'Bob', age=5, count=1), Row(name=u'Alice', age=2, count=1)]</span>
<span class="sd"> """</span>
<span class="n">jgd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">groupBy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jcols</span><span class="p">(</span><span class="o">*</span><span class="n">cols</span><span class="p">))</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.group</span> <span class="kn">import</span> <span class="n">GroupedData</span>
<span class="k">return</span> <span class="n">GroupedData</span><span class="p">(</span><span class="n">jgd</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.rollup"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.rollup">[docs]</a> <span class="k">def</span> <span class="nf">rollup</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Create a multi-dimensional rollup for the current :class:`DataFrame` using</span>
<span class="sd"> the specified columns, so we can run aggregation on them.</span>
<span class="sd"> >>> df.rollup('name', df.age).count().show()</span>
<span class="sd"> +-----+----+-----+</span>
<span class="sd"> | name| age|count|</span>
<span class="sd"> +-----+----+-----+</span>
<span class="sd"> |Alice|null| 1|</span>
<span class="sd"> | Bob| 5| 1|</span>
<span class="sd"> | Bob|null| 1|</span>
<span class="sd"> | null|null| 2|</span>
<span class="sd"> |Alice| 2| 1|</span>
<span class="sd"> +-----+----+-----+</span>
<span class="sd"> """</span>
<span class="n">jgd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">rollup</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jcols</span><span class="p">(</span><span class="o">*</span><span class="n">cols</span><span class="p">))</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.group</span> <span class="kn">import</span> <span class="n">GroupedData</span>
<span class="k">return</span> <span class="n">GroupedData</span><span class="p">(</span><span class="n">jgd</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.cube"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.cube">[docs]</a> <span class="k">def</span> <span class="nf">cube</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">cols</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Create a multi-dimensional cube for the current :class:`DataFrame` using</span>
<span class="sd"> the specified columns, so we can run aggregation on them.</span>
<span class="sd"> >>> df.cube('name', df.age).count().show()</span>
<span class="sd"> +-----+----+-----+</span>
<span class="sd"> | name| age|count|</span>
<span class="sd"> +-----+----+-----+</span>
<span class="sd"> | null| 2| 1|</span>
<span class="sd"> |Alice|null| 1|</span>
<span class="sd"> | Bob| 5| 1|</span>
<span class="sd"> | Bob|null| 1|</span>
<span class="sd"> | null| 5| 1|</span>
<span class="sd"> | null|null| 2|</span>
<span class="sd"> |Alice| 2| 1|</span>
<span class="sd"> +-----+----+-----+</span>
<span class="sd"> """</span>
<span class="n">jgd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">cube</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jcols</span><span class="p">(</span><span class="o">*</span><span class="n">cols</span><span class="p">))</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.group</span> <span class="kn">import</span> <span class="n">GroupedData</span>
<span class="k">return</span> <span class="n">GroupedData</span><span class="p">(</span><span class="n">jgd</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.agg"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.agg">[docs]</a> <span class="k">def</span> <span class="nf">agg</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">exprs</span><span class="p">):</span>
<span class="sd">""" Aggregate on the entire :class:`DataFrame` without groups</span>
<span class="sd"> (shorthand for ``df.groupBy.agg()``).</span>
<span class="sd"> >>> df.agg({"age": "max"}).collect()</span>
<span class="sd"> [Row(max(age)=5)]</span>
<span class="sd"> >>> from pyspark.sql import functions as F</span>
<span class="sd"> >>> df.agg(F.min(df.age)).collect()</span>
<span class="sd"> [Row(min(age)=2)]</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">groupBy</span><span class="p">()</span><span class="o">.</span><span class="n">agg</span><span class="p">(</span><span class="o">*</span><span class="n">exprs</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.unionAll"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.unionAll">[docs]</a> <span class="k">def</span> <span class="nf">unionAll</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">""" Return a new :class:`DataFrame` containing union of rows in this</span>
<span class="sd"> frame and another frame.</span>
<span class="sd"> This is equivalent to `UNION ALL` in SQL.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">unionAll</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_jdf</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.intersect"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.intersect">[docs]</a> <span class="k">def</span> <span class="nf">intersect</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">""" Return a new :class:`DataFrame` containing rows only in</span>
<span class="sd"> both this frame and another frame.</span>
<span class="sd"> This is equivalent to `INTERSECT` in SQL.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">intersect</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_jdf</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.subtract"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.subtract">[docs]</a> <span class="k">def</span> <span class="nf">subtract</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">""" Return a new :class:`DataFrame` containing rows in this frame</span>
<span class="sd"> but not in another frame.</span>
<span class="sd"> This is equivalent to `EXCEPT` in SQL.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="p">,</span> <span class="s">"except"</span><span class="p">)(</span><span class="n">other</span><span class="o">.</span><span class="n">_jdf</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.dropDuplicates"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.dropDuplicates">[docs]</a> <span class="k">def</span> <span class="nf">dropDuplicates</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Return a new :class:`DataFrame` with duplicate rows removed,</span>
<span class="sd"> optionally only considering certain columns.</span>
<span class="sd"> >>> from pyspark.sql import Row</span>
<span class="sd"> >>> df = sc.parallelize([ \</span>
<span class="sd"> Row(name='Alice', age=5, height=80), \</span>
<span class="sd"> Row(name='Alice', age=5, height=80), \</span>
<span class="sd"> Row(name='Alice', age=10, height=80)]).toDF()</span>
<span class="sd"> >>> df.dropDuplicates().show()</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> |age|height| name|</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> | 5| 80|Alice|</span>
<span class="sd"> | 10| 80|Alice|</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> >>> df.dropDuplicates(['name', 'height']).show()</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> |age|height| name|</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> | 5| 80|Alice|</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">subset</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">dropDuplicates</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">dropDuplicates</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">subset</span><span class="p">))</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="s">"1.3.1"</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.dropna"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.dropna">[docs]</a> <span class="k">def</span> <span class="nf">dropna</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s">'any'</span><span class="p">,</span> <span class="n">thresh</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`DataFrame` omitting rows with null values.</span>
<span class="sd"> :func:`DataFrame.dropna` and :func:`DataFrameNaFunctions.drop` are aliases of each other.</span>
<span class="sd"> :param how: 'any' or 'all'.</span>
<span class="sd"> If 'any', drop a row if it contains any nulls.</span>
<span class="sd"> If 'all', drop a row only if all its values are null.</span>
<span class="sd"> :param thresh: int, default None</span>
<span class="sd"> If specified, drop rows that have less than `thresh` non-null values.</span>
<span class="sd"> This overwrites the `how` parameter.</span>
<span class="sd"> :param subset: optional list of column names to consider.</span>
<span class="sd"> >>> df4.na.drop().show()</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> |age|height| name|</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> | 10| 80|Alice|</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">how</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="n">how</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s">'any'</span><span class="p">,</span> <span class="s">'all'</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"how ('"</span> <span class="o">+</span> <span class="n">how</span> <span class="o">+</span> <span class="s">"') should be 'any' or 'all'"</span><span class="p">)</span>
<span class="k">if</span> <span class="n">subset</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">subset</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">columns</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">subset</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="n">subset</span> <span class="o">=</span> <span class="p">[</span><span class="n">subset</span><span class="p">]</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">subset</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"subset should be a list or tuple of column names"</span><span class="p">)</span>
<span class="k">if</span> <span class="n">thresh</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">thresh</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">subset</span><span class="p">)</span> <span class="k">if</span> <span class="n">how</span> <span class="o">==</span> <span class="s">'any'</span> <span class="k">else</span> <span class="mi">1</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">na</span><span class="p">()</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">thresh</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">subset</span><span class="p">)),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="s">"1.3.1"</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.fillna"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.fillna">[docs]</a> <span class="k">def</span> <span class="nf">fillna</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Replace null values, alias for ``na.fill()``.</span>
<span class="sd"> :func:`DataFrame.fillna` and :func:`DataFrameNaFunctions.fill` are aliases of each other.</span>
<span class="sd"> :param value: int, long, float, string, or dict.</span>
<span class="sd"> Value to replace null values with.</span>
<span class="sd"> If the value is a dict, then `subset` is ignored and `value` must be a mapping</span>
<span class="sd"> from column name (string) to replacement value. The replacement value must be</span>
<span class="sd"> an int, long, float, or string.</span>
<span class="sd"> :param subset: optional list of column names to consider.</span>
<span class="sd"> Columns specified in subset that do not have matching data type are ignored.</span>
<span class="sd"> For example, if `value` is a string, and subset contains a non-string column,</span>
<span class="sd"> then the non-string column is simply ignored.</span>
<span class="sd"> >>> df4.na.fill(50).show()</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> |age|height| name|</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> | 10| 80|Alice|</span>
<span class="sd"> | 5| 50| Bob|</span>
<span class="sd"> | 50| 50| Tom|</span>
<span class="sd"> | 50| 50| null|</span>
<span class="sd"> +---+------+-----+</span>
<span class="sd"> >>> df4.na.fill({'age': 50, 'name': 'unknown'}).show()</span>
<span class="sd"> +---+------+-------+</span>
<span class="sd"> |age|height| name|</span>
<span class="sd"> +---+------+-------+</span>
<span class="sd"> | 10| 80| Alice|</span>
<span class="sd"> | 5| null| Bob|</span>
<span class="sd"> | 50| null| Tom|</span>
<span class="sd"> | 50| null|unknown|</span>
<span class="sd"> +---+------+-------+</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"value should be a float, int, long, string, or dict"</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">)):</span>
<span class="n">value</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">na</span><span class="p">()</span><span class="o">.</span><span class="n">fill</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">subset</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">na</span><span class="p">()</span><span class="o">.</span><span class="n">fill</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">subset</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="n">subset</span> <span class="o">=</span> <span class="p">[</span><span class="n">subset</span><span class="p">]</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">subset</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"subset should be a list or tuple of column names"</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">na</span><span class="p">()</span><span class="o">.</span><span class="n">fill</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">subset</span><span class="p">)),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.replace"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.replace">[docs]</a> <span class="k">def</span> <span class="nf">replace</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">to_replace</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`DataFrame` replacing a value with another value.</span>
<span class="sd"> :func:`DataFrame.replace` and :func:`DataFrameNaFunctions.replace` are</span>
<span class="sd"> aliases of each other.</span>
<span class="sd"> :param to_replace: int, long, float, string, or list.</span>
<span class="sd"> Value to be replaced.</span>
<span class="sd"> If the value is a dict, then `value` is ignored and `to_replace` must be a</span>
<span class="sd"> mapping from column name (string) to replacement value. The value to be</span>
<span class="sd"> replaced must be an int, long, float, or string.</span>
<span class="sd"> :param value: int, long, float, string, or list.</span>
<span class="sd"> Value to use to replace holes.</span>
<span class="sd"> The replacement value must be an int, long, float, or string. If `value` is a</span>
<span class="sd"> list or tuple, `value` should be of the same length with `to_replace`.</span>
<span class="sd"> :param subset: optional list of column names to consider.</span>
<span class="sd"> Columns specified in subset that do not have matching data type are ignored.</span>
<span class="sd"> For example, if `value` is a string, and subset contains a non-string column,</span>
<span class="sd"> then the non-string column is simply ignored.</span>
<span class="sd"> >>> df4.na.replace(10, 20).show()</span>
<span class="sd"> +----+------+-----+</span>
<span class="sd"> | age|height| name|</span>
<span class="sd"> +----+------+-----+</span>
<span class="sd"> | 20| 80|Alice|</span>
<span class="sd"> | 5| null| Bob|</span>
<span class="sd"> |null| null| Tom|</span>
<span class="sd"> |null| null| null|</span>
<span class="sd"> +----+------+-----+</span>
<span class="sd"> >>> df4.na.replace(['Alice', 'Bob'], ['A', 'B'], 'name').show()</span>
<span class="sd"> +----+------+----+</span>
<span class="sd"> | age|height|name|</span>
<span class="sd"> +----+------+----+</span>
<span class="sd"> | 10| 80| A|</span>
<span class="sd"> | 5| null| B|</span>
<span class="sd"> |null| null| Tom|</span>
<span class="sd"> |null| null|null|</span>
<span class="sd"> +----+------+----+</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">,</span> <span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
<span class="s">"to_replace should be a float, int, long, string, list, tuple, or dict"</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">,</span> <span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"value should be a float, int, long, string, list, or tuple"</span><span class="p">)</span>
<span class="n">rep_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">)):</span>
<span class="n">to_replace</span> <span class="o">=</span> <span class="p">[</span><span class="n">to_replace</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
<span class="n">to_replace</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">to_replace</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
<span class="n">value</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">to_replace</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"to_replace and value lists should be of the same length"</span><span class="p">)</span>
<span class="n">rep_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="n">value</span><span class="p">))</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">)):</span>
<span class="n">rep_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">([(</span><span class="n">tr</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span> <span class="k">for</span> <span class="n">tr</span> <span class="ow">in</span> <span class="n">to_replace</span><span class="p">])</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">rep_dict</span> <span class="o">=</span> <span class="n">to_replace</span>
<span class="k">if</span> <span class="n">subset</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">na</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s">'*'</span><span class="p">,</span> <span class="n">rep_dict</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">subset</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="n">subset</span> <span class="o">=</span> <span class="p">[</span><span class="n">subset</span><span class="p">]</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">subset</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"subset should be a list or tuple of column names"</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">na</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jseq</span><span class="p">(</span><span class="n">subset</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jmap</span><span class="p">(</span><span class="n">rep_dict</span><span class="p">)),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.corr"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.corr">[docs]</a> <span class="k">def</span> <span class="nf">corr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Calculates the correlation of two columns of a DataFrame as a double value.</span>
<span class="sd"> Currently only supports the Pearson Correlation Coefficient.</span>
<span class="sd"> :func:`DataFrame.corr` and :func:`DataFrameStatFunctions.corr` are aliases of each other.</span>
<span class="sd"> :param col1: The name of the first column</span>
<span class="sd"> :param col2: The name of the second column</span>
<span class="sd"> :param method: The correlation method. Currently only supports "pearson"</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"col1 should be a string."</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col2</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"col2 should be a string."</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">method</span><span class="p">:</span>
<span class="n">method</span> <span class="o">=</span> <span class="s">"pearson"</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">method</span> <span class="o">==</span> <span class="s">"pearson"</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Currently only the calculation of the Pearson Correlation "</span> <span class="o">+</span>
<span class="s">"coefficient is supported."</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">,</span> <span class="n">method</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.cov"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.cov">[docs]</a> <span class="k">def</span> <span class="nf">cov</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Calculate the sample covariance for the given columns, specified by their names, as a</span>
<span class="sd"> double value. :func:`DataFrame.cov` and :func:`DataFrameStatFunctions.cov` are aliases.</span>
<span class="sd"> :param col1: The name of the first column</span>
<span class="sd"> :param col2: The name of the second column</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"col1 should be a string."</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col2</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"col2 should be a string."</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">cov</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.crosstab"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.crosstab">[docs]</a> <span class="k">def</span> <span class="nf">crosstab</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Computes a pair-wise frequency table of the given columns. Also known as a contingency</span>
<span class="sd"> table. The number of distinct values for each column should be less than 1e4. At most 1e6</span>
<span class="sd"> non-zero pair frequencies will be returned.</span>
<span class="sd"> The first column of each row will be the distinct values of `col1` and the column names</span>
<span class="sd"> will be the distinct values of `col2`. The name of the first column will be `$col1_$col2`.</span>
<span class="sd"> Pairs that have no occurrences will have zero as their counts.</span>
<span class="sd"> :func:`DataFrame.crosstab` and :func:`DataFrameStatFunctions.crosstab` are aliases.</span>
<span class="sd"> :param col1: The name of the first column. Distinct items will make the first item of</span>
<span class="sd"> each row.</span>
<span class="sd"> :param col2: The name of the second column. Distinct items will make the column names</span>
<span class="sd"> of the DataFrame.</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"col1 should be a string."</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col2</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"col2 should be a string."</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">crosstab</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.freqItems"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.freqItems">[docs]</a> <span class="k">def</span> <span class="nf">freqItems</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cols</span><span class="p">,</span> <span class="n">support</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Finding frequent items for columns, possibly with false positives. Using the</span>
<span class="sd"> frequent element count algorithm described in</span>
<span class="sd"> "http://dx.doi.org/10.1145/762471.762473, proposed by Karp, Schenker, and Papadimitriou".</span>
<span class="sd"> :func:`DataFrame.freqItems` and :func:`DataFrameStatFunctions.freqItems` are aliases.</span>
<span class="sd"> .. note:: This function is meant for exploratory data analysis, as we make no \</span>
<span class="sd"> guarantee about the backward compatibility of the schema of the resulting DataFrame.</span>
<span class="sd"> :param cols: Names of the columns to calculate frequent items for as a list or tuple of</span>
<span class="sd"> strings.</span>
<span class="sd"> :param support: The frequency with which to consider an item 'frequent'. Default is 1%.</span>
<span class="sd"> The support must be greater than 1e-4.</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cols</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
<span class="n">cols</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">cols</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cols</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"cols must be a list or tuple of column names as strings."</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">support</span><span class="p">:</span>
<span class="n">support</span> <span class="o">=</span> <span class="mf">0.01</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">freqItems</span><span class="p">(</span><span class="n">_to_seq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sc</span><span class="p">,</span> <span class="n">cols</span><span class="p">),</span> <span class="n">support</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.withColumn"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.withColumn">[docs]</a> <span class="k">def</span> <span class="nf">withColumn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">colName</span><span class="p">,</span> <span class="n">col</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a new :class:`DataFrame` by adding a column or replacing the</span>
<span class="sd"> existing column that has the same name.</span>
<span class="sd"> :param colName: string, name of the new column.</span>
<span class="sd"> :param col: a :class:`Column` expression for the new column.</span>
<span class="sd"> >>> df.withColumn('age2', df.age + 2).collect()</span>
<span class="sd"> [Row(age=2, name=u'Alice', age2=4), Row(age=5, name=u'Bob', age2=7)]</span>
<span class="sd"> """</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">Column</span><span class="p">),</span> <span class="s">"col should be Column"</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">withColumn</span><span class="p">(</span><span class="n">colName</span><span class="p">,</span> <span class="n">col</span><span class="o">.</span><span class="n">_jc</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@ignore_unicode_prefix</span>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.withColumnRenamed"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.withColumnRenamed">[docs]</a> <span class="k">def</span> <span class="nf">withColumnRenamed</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">existing</span><span class="p">,</span> <span class="n">new</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`DataFrame` by renaming an existing column.</span>
<span class="sd"> :param existing: string, name of the existing column to rename.</span>
<span class="sd"> :param col: string, new name of the column.</span>
<span class="sd"> >>> df.withColumnRenamed('age', 'age2').collect()</span>
<span class="sd"> [Row(age2=2, name=u'Alice'), Row(age2=5, name=u'Bob')]</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">withColumnRenamed</span><span class="p">(</span><span class="n">existing</span><span class="p">,</span> <span class="n">new</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.4</span><span class="p">)</span>
<span class="nd">@ignore_unicode_prefix</span>
<div class="viewcode-block" id="DataFrame.drop"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.drop">[docs]</a> <span class="k">def</span> <span class="nf">drop</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">):</span>
<span class="sd">"""Returns a new :class:`DataFrame` that drops the specified column.</span>
<span class="sd"> :param col: a string name of the column to drop, or a</span>
<span class="sd"> :class:`Column` to drop.</span>
<span class="sd"> >>> df.drop('age').collect()</span>
<span class="sd"> [Row(name=u'Alice'), Row(name=u'Bob')]</span>
<span class="sd"> >>> df.drop(df.age).collect()</span>
<span class="sd"> [Row(name=u'Alice'), Row(name=u'Bob')]</span>
<span class="sd"> >>> df.join(df2, df.name == df2.name, 'inner').drop(df.name).collect()</span>
<span class="sd"> [Row(age=5, height=85, name=u'Bob')]</span>
<span class="sd"> >>> df.join(df2, df.name == df2.name, 'inner').drop(df2.name).collect()</span>
<span class="sd"> [Row(age=5, name=u'Bob', height=85)]</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">basestring</span><span class="p">):</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">Column</span><span class="p">):</span>
<span class="n">jdf</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jdf</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">col</span><span class="o">.</span><span class="n">_jc</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"col should be a string or a Column"</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DataFrame</span><span class="p">(</span><span class="n">jdf</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql_ctx</span><span class="p">)</span>
</div>
<span class="nd">@since</span><span class="p">(</span><span class="mf">1.3</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrame.toPandas"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrame.toPandas">[docs]</a> <span class="k">def</span> <span class="nf">toPandas</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns the contents of this :class:`DataFrame` as Pandas ``pandas.DataFrame``.</span>
<span class="sd"> This is only available if Pandas is installed and available.</span>
<span class="sd"> >>> df.toPandas() # doctest: +SKIP</span>
<span class="sd"> age name</span>
<span class="sd"> 0 2 Alice</span>
<span class="sd"> 1 5 Bob</span>
<span class="sd"> """</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="kn">as</span> <span class="nn">pd</span>
<span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="o">.</span><span class="n">from_records</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">collect</span><span class="p">(),</span> <span class="n">columns</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">columns</span><span class="p">)</span>
<span class="c">##########################################################################################</span>
<span class="c"># Pandas compatibility</span>
<span class="c">##########################################################################################</span>
</div>
<span class="n">groupby</span> <span class="o">=</span> <span class="n">groupBy</span>
<span class="n">drop_duplicates</span> <span class="o">=</span> <span class="n">dropDuplicates</span>
<span class="c"># Having SchemaRDD for backward compatibility (for docs)</span></div>
<span class="k">class</span> <span class="nc">SchemaRDD</span><span class="p">(</span><span class="n">DataFrame</span><span class="p">):</span>
<span class="sd">"""SchemaRDD is deprecated, please use :class:`DataFrame`.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">_to_scala_map</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="n">jm</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Convert a dict into a JVM Map.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">sc</span><span class="o">.</span><span class="n">_jvm</span><span class="o">.</span><span class="n">PythonUtils</span><span class="o">.</span><span class="n">toScalaMap</span><span class="p">(</span><span class="n">jm</span><span class="p">)</span>
<div class="viewcode-block" id="DataFrameNaFunctions"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameNaFunctions">[docs]</a><span class="k">class</span> <span class="nc">DataFrameNaFunctions</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="sd">"""Functionality for working with missing data in :class:`DataFrame`.</span>
<span class="sd"> .. versionadded:: 1.4</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">df</span> <span class="o">=</span> <span class="n">df</span>
<div class="viewcode-block" id="DataFrameNaFunctions.drop"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameNaFunctions.drop">[docs]</a> <span class="k">def</span> <span class="nf">drop</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">how</span><span class="o">=</span><span class="s">'any'</span><span class="p">,</span> <span class="n">thresh</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">dropna</span><span class="p">(</span><span class="n">how</span><span class="o">=</span><span class="n">how</span><span class="p">,</span> <span class="n">thresh</span><span class="o">=</span><span class="n">thresh</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="n">subset</span><span class="p">)</span>
</div>
<span class="n">drop</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">dropna</span><span class="o">.</span><span class="n">__doc__</span>
<div class="viewcode-block" id="DataFrameNaFunctions.fill"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameNaFunctions.fill">[docs]</a> <span class="k">def</span> <span class="nf">fill</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="n">value</span><span class="o">=</span><span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="n">subset</span><span class="p">)</span>
</div>
<span class="n">fill</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">fillna</span><span class="o">.</span><span class="n">__doc__</span>
<div class="viewcode-block" id="DataFrameNaFunctions.replace"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameNaFunctions.replace">[docs]</a> <span class="k">def</span> <span class="nf">replace</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">to_replace</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">to_replace</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">subset</span><span class="p">)</span>
</div>
<span class="n">replace</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">replace</span><span class="o">.</span><span class="n">__doc__</span>
</div>
<div class="viewcode-block" id="DataFrameStatFunctions"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions">[docs]</a><span class="k">class</span> <span class="nc">DataFrameStatFunctions</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="sd">"""Functionality for statistic functions with :class:`DataFrame`.</span>
<span class="sd"> .. versionadded:: 1.4</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">df</span> <span class="o">=</span> <span class="n">df</span>
<div class="viewcode-block" id="DataFrameStatFunctions.corr"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.corr">[docs]</a> <span class="k">def</span> <span class="nf">corr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">,</span> <span class="n">method</span><span class="p">)</span>
</div>
<span class="n">corr</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">corr</span><span class="o">.</span><span class="n">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.cov"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.cov">[docs]</a> <span class="k">def</span> <span class="nf">cov</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">cov</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">)</span>
</div>
<span class="n">cov</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">cov</span><span class="o">.</span><span class="n">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.crosstab"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.crosstab">[docs]</a> <span class="k">def</span> <span class="nf">crosstab</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">crosstab</span><span class="p">(</span><span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">)</span>
</div>
<span class="n">crosstab</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">crosstab</span><span class="o">.</span><span class="n">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.freqItems"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.freqItems">[docs]</a> <span class="k">def</span> <span class="nf">freqItems</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cols</span><span class="p">,</span> <span class="n">support</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">freqItems</span><span class="p">(</span><span class="n">cols</span><span class="p">,</span> <span class="n">support</span><span class="p">)</span>
</div>
<span class="n">freqItems</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">freqItems</span><span class="o">.</span><span class="n">__doc__</span>
<div class="viewcode-block" id="DataFrameStatFunctions.sampleBy"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataFrameStatFunctions.sampleBy">[docs]</a> <span class="k">def</span> <span class="nf">sampleBy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">,</span> <span class="n">fractions</span><span class="p">,</span> <span class="n">seed</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">df</span><span class="o">.</span><span class="n">sampleBy</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">fractions</span><span class="p">,</span> <span class="n">seed</span><span class="p">)</span>
</div>
<span class="n">sampleBy</span><span class="o">.</span><span class="n">__doc__</span> <span class="o">=</span> <span class="n">DataFrame</span><span class="o">.</span><span class="n">sampleBy</span><span class="o">.</span><span class="n">__doc__</span>
</div>
<span class="k">def</span> <span class="nf">_test</span><span class="p">():</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.context</span> <span class="kn">import</span> <span class="n">SparkContext</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="kn">import</span> <span class="n">Row</span><span class="p">,</span> <span class="n">SQLContext</span>
<span class="kn">import</span> <span class="nn">pyspark.sql.dataframe</span>
<span class="n">globs</span> <span class="o">=</span> <span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">dataframe</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s">'local[4]'</span><span class="p">,</span> <span class="s">'PythonTest'</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'sc'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'sqlContext'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SQLContext</span><span class="p">(</span><span class="n">sc</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'df'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([(</span><span class="mi">2</span><span class="p">,</span> <span class="s">'Alice'</span><span class="p">),</span> <span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="s">'Bob'</span><span class="p">)])</span>\
<span class="o">.</span><span class="n">toDF</span><span class="p">(</span><span class="n">StructType</span><span class="p">([</span><span class="n">StructField</span><span class="p">(</span><span class="s">'age'</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">()),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">'name'</span><span class="p">,</span> <span class="n">StringType</span><span class="p">())]))</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'df2'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="n">Row</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s">'Tom'</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">80</span><span class="p">),</span> <span class="n">Row</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s">'Bob'</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">85</span><span class="p">)])</span><span class="o">.</span><span class="n">toDF</span><span class="p">()</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'df3'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="n">Row</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s">'Alice'</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="mi">2</span><span class="p">),</span>
<span class="n">Row</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s">'Bob'</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="mi">5</span><span class="p">)])</span><span class="o">.</span><span class="n">toDF</span><span class="p">()</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'df4'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">([</span><span class="n">Row</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s">'Alice'</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">80</span><span class="p">),</span>
<span class="n">Row</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s">'Bob'</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="bp">None</span><span class="p">),</span>
<span class="n">Row</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s">'Tom'</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="bp">None</span><span class="p">),</span>
<span class="n">Row</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">age</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="bp">None</span><span class="p">)])</span><span class="o">.</span><span class="n">toDF</span><span class="p">()</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">(</span>
<span class="n">pyspark</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">dataframe</span><span class="p">,</span> <span class="n">globs</span><span class="o">=</span><span class="n">globs</span><span class="p">,</span>
<span class="n">optionflags</span><span class="o">=</span><span class="n">doctest</span><span class="o">.</span><span class="n">ELLIPSIS</span> <span class="o">|</span> <span class="n">doctest</span><span class="o">.</span><span class="n">NORMALIZE_WHITESPACE</span> <span class="o">|</span> <span class="n">doctest</span><span class="o">.</span><span class="n">REPORT_NDIFF</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'sc'</span><span class="p">]</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span>
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="nb">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
</div>
</div>
</div>
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/spark-logo-hd.png" alt="Logo"/>
</a></p>
<div id="searchbox" style="display: none" role="search">
<h3>Quick search</h3>
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" />
<input type="submit" value="Go" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
<p class="searchtip" style="font-size: 90%">
Enter search terms or a module, class or function name.
</p>
</div>
<script type="text/javascript">$('#searchbox').show(0);</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="related" role="navigation" aria-label="related navigation">
<h3>Navigation</h3>
<ul>
<li class="nav-item nav-item-0"><a href="../../../index.html">PySpark 1.5.0 documentation</a> »</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
</ul>
</div>
<div class="footer" role="contentinfo">
© Copyright .
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.3.1.
</div>
</body>
</html>