<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>pyspark.sql.types — PySpark master documentation</title>
<link rel="stylesheet" href="../../../_static/nature.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../../../',
VERSION: 'master',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../_static/doctools.js"></script>
<link rel="top" title="PySpark master documentation" href="../../../index.html" />
<link rel="up" title="Module code" href="../../index.html" />
</head>
<body role="document">
<div class="related" role="navigation" aria-label="related navigation">
<h3>Navigation</h3>
<ul>
<li class="nav-item nav-item-0"><a href="../../../index.html">PySpark master documentation</a> »</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for pyspark.sql.types</h1><div class="highlight"><pre>
<span class="c">#</span>
<span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
<span class="c"># this work for additional information regarding copyright ownership.</span>
<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
<span class="c"># (the "License"); you may not use this file except in compliance with</span>
<span class="c"># the License. You may obtain a copy of the License at</span>
<span class="c">#</span>
<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c">#</span>
<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c"># See the License for the specific language governing permissions and</span>
<span class="c"># limitations under the License.</span>
<span class="c">#</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">decimal</span>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">datetime</span>
<span class="kn">import</span> <span class="nn">calendar</span>
<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">base64</span>
<span class="kn">from</span> <span class="nn">array</span> <span class="kn">import</span> <span class="n">array</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version</span> <span class="o">>=</span> <span class="s">"3"</span><span class="p">:</span>
<span class="nb">long</span> <span class="o">=</span> <span class="nb">int</span>
<span class="nb">unicode</span> <span class="o">=</span> <span class="nb">str</span>
<span class="kn">from</span> <span class="nn">py4j.protocol</span> <span class="kn">import</span> <span class="n">register_input_converter</span>
<span class="kn">from</span> <span class="nn">py4j.java_gateway</span> <span class="kn">import</span> <span class="n">JavaClass</span>
<span class="kn">from</span> <span class="nn">pyspark.serializers</span> <span class="kn">import</span> <span class="n">CloudPickleSerializer</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span>
<span class="s">"DataType"</span><span class="p">,</span> <span class="s">"NullType"</span><span class="p">,</span> <span class="s">"StringType"</span><span class="p">,</span> <span class="s">"BinaryType"</span><span class="p">,</span> <span class="s">"BooleanType"</span><span class="p">,</span> <span class="s">"DateType"</span><span class="p">,</span>
<span class="s">"TimestampType"</span><span class="p">,</span> <span class="s">"DecimalType"</span><span class="p">,</span> <span class="s">"DoubleType"</span><span class="p">,</span> <span class="s">"FloatType"</span><span class="p">,</span> <span class="s">"ByteType"</span><span class="p">,</span> <span class="s">"IntegerType"</span><span class="p">,</span>
<span class="s">"LongType"</span><span class="p">,</span> <span class="s">"ShortType"</span><span class="p">,</span> <span class="s">"ArrayType"</span><span class="p">,</span> <span class="s">"MapType"</span><span class="p">,</span> <span class="s">"StructField"</span><span class="p">,</span> <span class="s">"StructType"</span><span class="p">]</span>
<div class="viewcode-block" id="DataType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType">[docs]</a><span class="k">class</span> <span class="nc">DataType</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="sd">"""Base class for data types."""</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span>
<span class="k">def</span> <span class="nf">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">__dict__</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="nd">@classmethod</span>
<div class="viewcode-block" id="DataType.typeName"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.typeName">[docs]</a> <span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">__name__</span><span class="p">[:</span><span class="o">-</span><span class="mi">4</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="DataType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">typeName</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="DataType.jsonValue"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.jsonValue">[docs]</a> <span class="k">def</span> <span class="nf">jsonValue</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">typeName</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="DataType.json"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.json">[docs]</a> <span class="k">def</span> <span class="nf">json</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">(),</span>
<span class="n">separators</span><span class="o">=</span><span class="p">(</span><span class="s">','</span><span class="p">,</span> <span class="s">':'</span><span class="p">),</span>
<span class="n">sort_keys</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DataType.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Does this type need to conversion between Python object and internal SQL object.</span>
<span class="sd"> This is used to avoid the unnecessary conversion for ArrayType/MapType/StructType.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="bp">False</span>
</div>
<div class="viewcode-block" id="DataType.toInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.toInternal">[docs]</a> <span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Converts a Python object into an internal SQL object.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">obj</span>
</div>
<div class="viewcode-block" id="DataType.fromInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DataType.fromInternal">[docs]</a> <span class="k">def</span> <span class="nf">fromInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Converts an internal SQL object into a native Python object.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">obj</span>
<span class="c"># This singleton pattern does not work with pickle, you will get</span>
<span class="c"># another object after pickle and unpickle</span></div></div>
<span class="k">class</span> <span class="nc">DataTypeSingleton</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
<span class="sd">"""Metaclass for DataType"""</span>
<span class="n">_instances</span> <span class="o">=</span> <span class="p">{}</span>
<span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">if</span> <span class="n">cls</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">:</span>
<span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="n">cls</span><span class="p">]</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">DataTypeSingleton</span><span class="p">,</span> <span class="n">cls</span><span class="p">)</span><span class="o">.</span><span class="n">__call__</span><span class="p">()</span>
<span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="n">cls</span><span class="p">]</span>
<div class="viewcode-block" id="NullType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.NullType">[docs]</a><span class="k">class</span> <span class="nc">NullType</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span>
<span class="sd">"""Null type.</span>
<span class="sd"> The data type representing None, used for the types that cannot be inferred.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
</div>
<span class="k">class</span> <span class="nc">AtomicType</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span>
<span class="sd">"""An internal type used to represent everything that is not</span>
<span class="sd"> null, UDTs, arrays, structs, and maps."""</span>
<span class="k">class</span> <span class="nc">NumericType</span><span class="p">(</span><span class="n">AtomicType</span><span class="p">):</span>
<span class="sd">"""Numeric data types.</span>
<span class="sd"> """</span>
<span class="k">class</span> <span class="nc">IntegralType</span><span class="p">(</span><span class="n">NumericType</span><span class="p">):</span>
<span class="sd">"""Integral data types.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
<span class="k">class</span> <span class="nc">FractionalType</span><span class="p">(</span><span class="n">NumericType</span><span class="p">):</span>
<span class="sd">"""Fractional data types.</span>
<span class="sd"> """</span>
<div class="viewcode-block" id="StringType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StringType">[docs]</a><span class="k">class</span> <span class="nc">StringType</span><span class="p">(</span><span class="n">AtomicType</span><span class="p">):</span>
<span class="sd">"""String data type.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
</div>
<div class="viewcode-block" id="BinaryType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.BinaryType">[docs]</a><span class="k">class</span> <span class="nc">BinaryType</span><span class="p">(</span><span class="n">AtomicType</span><span class="p">):</span>
<span class="sd">"""Binary (byte array) data type.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
</div>
<div class="viewcode-block" id="BooleanType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.BooleanType">[docs]</a><span class="k">class</span> <span class="nc">BooleanType</span><span class="p">(</span><span class="n">AtomicType</span><span class="p">):</span>
<span class="sd">"""Boolean data type.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
</div>
<div class="viewcode-block" id="DateType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DateType">[docs]</a><span class="k">class</span> <span class="nc">DateType</span><span class="p">(</span><span class="n">AtomicType</span><span class="p">):</span>
<span class="sd">"""Date (datetime.date) data type.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
<span class="n">EPOCH_ORDINAL</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="p">(</span><span class="mi">1970</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">toordinal</span><span class="p">()</span>
<div class="viewcode-block" id="DateType.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DateType.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">True</span>
</div>
<div class="viewcode-block" id="DateType.toInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DateType.toInternal">[docs]</a> <span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">d</span><span class="p">):</span>
<span class="k">if</span> <span class="n">d</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">d</span><span class="o">.</span><span class="n">toordinal</span><span class="p">()</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">EPOCH_ORDINAL</span>
</div>
<div class="viewcode-block" id="DateType.fromInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DateType.fromInternal">[docs]</a> <span class="k">def</span> <span class="nf">fromInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">v</span><span class="p">):</span>
<span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">date</span><span class="o">.</span><span class="n">fromordinal</span><span class="p">(</span><span class="n">v</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">EPOCH_ORDINAL</span><span class="p">)</span>
</div></div>
<div class="viewcode-block" id="TimestampType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.TimestampType">[docs]</a><span class="k">class</span> <span class="nc">TimestampType</span><span class="p">(</span><span class="n">AtomicType</span><span class="p">):</span>
<span class="sd">"""Timestamp (datetime.datetime) data type.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
<div class="viewcode-block" id="TimestampType.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.TimestampType.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">True</span>
</div>
<div class="viewcode-block" id="TimestampType.toInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.TimestampType.toInternal">[docs]</a> <span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dt</span><span class="p">):</span>
<span class="k">if</span> <span class="n">dt</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="n">seconds</span> <span class="o">=</span> <span class="p">(</span><span class="n">calendar</span><span class="o">.</span><span class="n">timegm</span><span class="p">(</span><span class="n">dt</span><span class="o">.</span><span class="n">utctimetuple</span><span class="p">())</span> <span class="k">if</span> <span class="n">dt</span><span class="o">.</span><span class="n">tzinfo</span>
<span class="k">else</span> <span class="n">time</span><span class="o">.</span><span class="n">mktime</span><span class="p">(</span><span class="n">dt</span><span class="o">.</span><span class="n">timetuple</span><span class="p">()))</span>
<span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">seconds</span> <span class="o">*</span> <span class="mf">1e6</span> <span class="o">+</span> <span class="n">dt</span><span class="o">.</span><span class="n">microsecond</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="TimestampType.fromInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.TimestampType.fromInternal">[docs]</a> <span class="k">def</span> <span class="nf">fromInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ts</span><span class="p">):</span>
<span class="k">if</span> <span class="n">ts</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="c"># using int to avoid precision loss in float</span>
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">ts</span> <span class="o">//</span> <span class="mi">1000000</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">microsecond</span><span class="o">=</span><span class="n">ts</span> <span class="o">%</span> <span class="mi">1000000</span><span class="p">)</span>
</div></div>
<div class="viewcode-block" id="DecimalType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DecimalType">[docs]</a><span class="k">class</span> <span class="nc">DecimalType</span><span class="p">(</span><span class="n">FractionalType</span><span class="p">):</span>
<span class="sd">"""Decimal (decimal.Decimal) data type.</span>
<span class="sd"> The DecimalType must have fixed precision (the maximum total number of digits)</span>
<span class="sd"> and scale (the number of digits on the right of dot). For example, (5, 2) can</span>
<span class="sd"> support the value from [-999.99 to 999.99].</span>
<span class="sd"> The precision can be up to 38, the scale must less or equal to precision.</span>
<span class="sd"> When create a DecimalType, the default precision and scale is (10, 0). When infer</span>
<span class="sd"> schema from decimal.Decimal objects, it will be DecimalType(38, 18).</span>
<span class="sd"> :param precision: the maximum total number of digits (default: 10)</span>
<span class="sd"> :param scale: the number of digits on right side of dot. (default: 0)</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">precision</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">scale</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">precision</span> <span class="o">=</span> <span class="n">precision</span>
<span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hasPrecisionInfo</span> <span class="o">=</span> <span class="bp">True</span> <span class="c"># this is public API</span>
<div class="viewcode-block" id="DecimalType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DecimalType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"decimal(</span><span class="si">%d</span><span class="s">,</span><span class="si">%d</span><span class="s">)"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">precision</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DecimalType.jsonValue"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DecimalType.jsonValue">[docs]</a> <span class="k">def</span> <span class="nf">jsonValue</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"decimal(</span><span class="si">%d</span><span class="s">,</span><span class="si">%d</span><span class="s">)"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">precision</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">)</span>
</div>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"DecimalType(</span><span class="si">%d</span><span class="s">,</span><span class="si">%d</span><span class="s">)"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">precision</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DoubleType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.DoubleType">[docs]</a><span class="k">class</span> <span class="nc">DoubleType</span><span class="p">(</span><span class="n">FractionalType</span><span class="p">):</span>
<span class="sd">"""Double data type, representing double precision floats.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
</div>
<div class="viewcode-block" id="FloatType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.FloatType">[docs]</a><span class="k">class</span> <span class="nc">FloatType</span><span class="p">(</span><span class="n">FractionalType</span><span class="p">):</span>
<span class="sd">"""Float data type, representing single precision floats.</span>
<span class="sd"> """</span>
<span class="n">__metaclass__</span> <span class="o">=</span> <span class="n">DataTypeSingleton</span>
</div>
<div class="viewcode-block" id="ByteType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ByteType">[docs]</a><span class="k">class</span> <span class="nc">ByteType</span><span class="p">(</span><span class="n">IntegralType</span><span class="p">):</span>
<span class="sd">"""Byte data type, i.e. a signed integer in a single byte.</span>
<span class="sd"> """</span>
<div class="viewcode-block" id="ByteType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ByteType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'tinyint'</span>
</div></div>
<div class="viewcode-block" id="IntegerType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.IntegerType">[docs]</a><span class="k">class</span> <span class="nc">IntegerType</span><span class="p">(</span><span class="n">IntegralType</span><span class="p">):</span>
<span class="sd">"""Int data type, i.e. a signed 32-bit integer.</span>
<span class="sd"> """</span>
<div class="viewcode-block" id="IntegerType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.IntegerType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'int'</span>
</div></div>
<div class="viewcode-block" id="LongType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.LongType">[docs]</a><span class="k">class</span> <span class="nc">LongType</span><span class="p">(</span><span class="n">IntegralType</span><span class="p">):</span>
<span class="sd">"""Long data type, i.e. a signed 64-bit integer.</span>
<span class="sd"> If the values are beyond the range of [-9223372036854775808, 9223372036854775807],</span>
<span class="sd"> please use :class:`DecimalType`.</span>
<span class="sd"> """</span>
<div class="viewcode-block" id="LongType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.LongType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'bigint'</span>
</div></div>
<div class="viewcode-block" id="ShortType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ShortType">[docs]</a><span class="k">class</span> <span class="nc">ShortType</span><span class="p">(</span><span class="n">IntegralType</span><span class="p">):</span>
<span class="sd">"""Short data type, i.e. a signed 16-bit integer.</span>
<span class="sd"> """</span>
<div class="viewcode-block" id="ShortType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ShortType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'smallint'</span>
</div></div>
<div class="viewcode-block" id="ArrayType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType">[docs]</a><span class="k">class</span> <span class="nc">ArrayType</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span>
<span class="sd">"""Array data type.</span>
<span class="sd"> :param elementType: :class:`DataType` of each element in the array.</span>
<span class="sd"> :param containsNull: boolean, whether the array can contain null (None) values.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">elementType</span><span class="p">,</span> <span class="n">containsNull</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> >>> ArrayType(StringType()) == ArrayType(StringType(), True)</span>
<span class="sd"> True</span>
<span class="sd"> >>> ArrayType(StringType(), False) == ArrayType(StringType())</span>
<span class="sd"> False</span>
<span class="sd"> """</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">elementType</span><span class="p">,</span> <span class="n">DataType</span><span class="p">),</span> <span class="s">"elementType should be DataType"</span>
<span class="bp">self</span><span class="o">.</span><span class="n">elementType</span> <span class="o">=</span> <span class="n">elementType</span>
<span class="bp">self</span><span class="o">.</span><span class="n">containsNull</span> <span class="o">=</span> <span class="n">containsNull</span>
<div class="viewcode-block" id="ArrayType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'array<</span><span class="si">%s</span><span class="s">>'</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">elementType</span><span class="o">.</span><span class="n">simpleString</span><span class="p">()</span>
</div>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"ArrayType(</span><span class="si">%s</span><span class="s">,</span><span class="si">%s</span><span class="s">)"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">elementType</span><span class="p">,</span>
<span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">containsNull</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
<div class="viewcode-block" id="ArrayType.jsonValue"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.jsonValue">[docs]</a> <span class="k">def</span> <span class="nf">jsonValue</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">{</span><span class="s">"type"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">typeName</span><span class="p">(),</span>
<span class="s">"elementType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">elementType</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">(),</span>
<span class="s">"containsNull"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">containsNull</span><span class="p">}</span>
</div>
<span class="nd">@classmethod</span>
<div class="viewcode-block" id="ArrayType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="k">return</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s">"elementType"</span><span class="p">]),</span>
<span class="n">json</span><span class="p">[</span><span class="s">"containsNull"</span><span class="p">])</span>
</div>
<div class="viewcode-block" id="ArrayType.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">elementType</span><span class="o">.</span><span class="n">needConversion</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="ArrayType.toInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.toInternal">[docs]</a> <span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">needConversion</span><span class="p">():</span>
<span class="k">return</span> <span class="n">obj</span>
<span class="k">return</span> <span class="n">obj</span> <span class="ow">and</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">elementType</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">]</span>
</div>
<div class="viewcode-block" id="ArrayType.fromInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.ArrayType.fromInternal">[docs]</a> <span class="k">def</span> <span class="nf">fromInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">needConversion</span><span class="p">():</span>
<span class="k">return</span> <span class="n">obj</span>
<span class="k">return</span> <span class="n">obj</span> <span class="ow">and</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">elementType</span><span class="o">.</span><span class="n">fromInternal</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">]</span>
</div></div>
<div class="viewcode-block" id="MapType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType">[docs]</a><span class="k">class</span> <span class="nc">MapType</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span>
<span class="sd">"""Map data type.</span>
<span class="sd"> :param keyType: :class:`DataType` of the keys in the map.</span>
<span class="sd"> :param valueType: :class:`DataType` of the values in the map.</span>
<span class="sd"> :param valueContainsNull: indicates whether values can contain null (None) values.</span>
<span class="sd"> Keys in a map data type are not allowed to be null (None).</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">keyType</span><span class="p">,</span> <span class="n">valueType</span><span class="p">,</span> <span class="n">valueContainsNull</span><span class="o">=</span><span class="bp">True</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> >>> (MapType(StringType(), IntegerType())</span>
<span class="sd"> ... == MapType(StringType(), IntegerType(), True))</span>
<span class="sd"> True</span>
<span class="sd"> >>> (MapType(StringType(), IntegerType(), False)</span>
<span class="sd"> ... == MapType(StringType(), FloatType()))</span>
<span class="sd"> False</span>
<span class="sd"> """</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">keyType</span><span class="p">,</span> <span class="n">DataType</span><span class="p">),</span> <span class="s">"keyType should be DataType"</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">valueType</span><span class="p">,</span> <span class="n">DataType</span><span class="p">),</span> <span class="s">"valueType should be DataType"</span>
<span class="bp">self</span><span class="o">.</span><span class="n">keyType</span> <span class="o">=</span> <span class="n">keyType</span>
<span class="bp">self</span><span class="o">.</span><span class="n">valueType</span> <span class="o">=</span> <span class="n">valueType</span>
<span class="bp">self</span><span class="o">.</span><span class="n">valueContainsNull</span> <span class="o">=</span> <span class="n">valueContainsNull</span>
<div class="viewcode-block" id="MapType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'map<</span><span class="si">%s</span><span class="s">,</span><span class="si">%s</span><span class="s">>'</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keyType</span><span class="o">.</span><span class="n">simpleString</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueType</span><span class="o">.</span><span class="n">simpleString</span><span class="p">())</span>
</div>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"MapType(</span><span class="si">%s</span><span class="s">,</span><span class="si">%s</span><span class="s">,</span><span class="si">%s</span><span class="s">)"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">keyType</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueType</span><span class="p">,</span>
<span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">valueContainsNull</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
<div class="viewcode-block" id="MapType.jsonValue"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.jsonValue">[docs]</a> <span class="k">def</span> <span class="nf">jsonValue</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">{</span><span class="s">"type"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">typeName</span><span class="p">(),</span>
<span class="s">"keyType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">keyType</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">(),</span>
<span class="s">"valueType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueType</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">(),</span>
<span class="s">"valueContainsNull"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueContainsNull</span><span class="p">}</span>
</div>
<span class="nd">@classmethod</span>
<div class="viewcode-block" id="MapType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="k">return</span> <span class="n">MapType</span><span class="p">(</span><span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s">"keyType"</span><span class="p">]),</span>
<span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s">"valueType"</span><span class="p">]),</span>
<span class="n">json</span><span class="p">[</span><span class="s">"valueContainsNull"</span><span class="p">])</span>
</div>
<div class="viewcode-block" id="MapType.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">keyType</span><span class="o">.</span><span class="n">needConversion</span><span class="p">()</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueType</span><span class="o">.</span><span class="n">needConversion</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="MapType.toInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.toInternal">[docs]</a> <span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">needConversion</span><span class="p">():</span>
<span class="k">return</span> <span class="n">obj</span>
<span class="k">return</span> <span class="n">obj</span> <span class="ow">and</span> <span class="nb">dict</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">keyType</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">k</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueType</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">v</span><span class="p">))</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
</div>
<div class="viewcode-block" id="MapType.fromInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.MapType.fromInternal">[docs]</a> <span class="k">def</span> <span class="nf">fromInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">needConversion</span><span class="p">():</span>
<span class="k">return</span> <span class="n">obj</span>
<span class="k">return</span> <span class="n">obj</span> <span class="ow">and</span> <span class="nb">dict</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">keyType</span><span class="o">.</span><span class="n">fromInternal</span><span class="p">(</span><span class="n">k</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">valueType</span><span class="o">.</span><span class="n">fromInternal</span><span class="p">(</span><span class="n">v</span><span class="p">))</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
</div></div>
<div class="viewcode-block" id="StructField"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField">[docs]</a><span class="k">class</span> <span class="nc">StructField</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span>
<span class="sd">"""A field in :class:`StructType`.</span>
<span class="sd"> :param name: string, name of the field.</span>
<span class="sd"> :param dataType: :class:`DataType` of the field.</span>
<span class="sd"> :param nullable: boolean, whether the field can be null (None) or not.</span>
<span class="sd"> :param metadata: a dict from string to simple type that can be toInternald to JSON automatically</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">dataType</span><span class="p">,</span> <span class="n">nullable</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> >>> (StructField("f1", StringType(), True)</span>
<span class="sd"> ... == StructField("f1", StringType(), True))</span>
<span class="sd"> True</span>
<span class="sd"> >>> (StructField("f1", StringType(), True)</span>
<span class="sd"> ... == StructField("f2", StringType(), True))</span>
<span class="sd"> False</span>
<span class="sd"> """</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">DataType</span><span class="p">),</span> <span class="s">"dataType should be DataType"</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">name</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s">'utf-8'</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">name</span> <span class="o">=</span> <span class="n">name</span>
<span class="bp">self</span><span class="o">.</span><span class="n">dataType</span> <span class="o">=</span> <span class="n">dataType</span>
<span class="bp">self</span><span class="o">.</span><span class="n">nullable</span> <span class="o">=</span> <span class="n">nullable</span>
<span class="bp">self</span><span class="o">.</span><span class="n">metadata</span> <span class="o">=</span> <span class="n">metadata</span> <span class="ow">or</span> <span class="p">{}</span>
<div class="viewcode-block" id="StructField.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'</span><span class="si">%s</span><span class="s">:</span><span class="si">%s</span><span class="s">'</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataType</span><span class="o">.</span><span class="n">simpleString</span><span class="p">())</span>
</div>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"StructField(</span><span class="si">%s</span><span class="s">,</span><span class="si">%s</span><span class="s">,</span><span class="si">%s</span><span class="s">)"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataType</span><span class="p">,</span>
<span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">nullable</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
<div class="viewcode-block" id="StructField.jsonValue"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.jsonValue">[docs]</a> <span class="k">def</span> <span class="nf">jsonValue</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">{</span><span class="s">"name"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
<span class="s">"type"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataType</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">(),</span>
<span class="s">"nullable"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">nullable</span><span class="p">,</span>
<span class="s">"metadata"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">}</span>
</div>
<span class="nd">@classmethod</span>
<div class="viewcode-block" id="StructField.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructField</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s">"name"</span><span class="p">],</span>
<span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s">"type"</span><span class="p">]),</span>
<span class="n">json</span><span class="p">[</span><span class="s">"nullable"</span><span class="p">],</span>
<span class="n">json</span><span class="p">[</span><span class="s">"metadata"</span><span class="p">])</span>
</div>
<div class="viewcode-block" id="StructField.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataType</span><span class="o">.</span><span class="n">needConversion</span><span class="p">()</span>
</div>
<div class="viewcode-block" id="StructField.toInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.toInternal">[docs]</a> <span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataType</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="StructField.fromInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructField.fromInternal">[docs]</a> <span class="k">def</span> <span class="nf">fromInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataType</span><span class="o">.</span><span class="n">fromInternal</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
</div></div>
<div class="viewcode-block" id="StructType"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType">[docs]</a><span class="k">class</span> <span class="nc">StructType</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span>
<span class="sd">"""Struct type, consisting of a list of :class:`StructField`.</span>
<span class="sd"> This is the data type representing a :class:`Row`.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fields</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> >>> struct1 = StructType([StructField("f1", StringType(), True)])</span>
<span class="sd"> >>> struct2 = StructType([StructField("f1", StringType(), True)])</span>
<span class="sd"> >>> struct1 == struct2</span>
<span class="sd"> True</span>
<span class="sd"> >>> struct1 = StructType([StructField("f1", StringType(), True)])</span>
<span class="sd"> >>> struct2 = StructType([StructField("f1", StringType(), True),</span>
<span class="sd"> ... StructField("f2", IntegerType(), False)])</span>
<span class="sd"> >>> struct1 == struct2</span>
<span class="sd"> False</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">fields</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fields</span> <span class="o">=</span> <span class="p">[]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">names</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fields</span> <span class="o">=</span> <span class="n">fields</span>
<span class="bp">self</span><span class="o">.</span><span class="n">names</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">]</span>
<span class="k">assert</span> <span class="nb">all</span><span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">StructField</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">),</span>\
<span class="s">"fields should be a list of StructField"</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_needSerializeAnyField</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">needConversion</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">)</span>
<div class="viewcode-block" id="StructType.add"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.add">[docs]</a> <span class="k">def</span> <span class="nf">add</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field</span><span class="p">,</span> <span class="n">data_type</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">nullable</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">metadata</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Construct a StructType by adding new elements to it to define the schema. The method accepts</span>
<span class="sd"> either:</span>
<span class="sd"> a) A single parameter which is a StructField object.</span>
<span class="sd"> b) Between 2 and 4 parameters as (name, data_type, nullable (optional),</span>
<span class="sd"> metadata(optional). The data_type parameter may be either a String or a</span>
<span class="sd"> DataType object.</span>
<span class="sd"> >>> struct1 = StructType().add("f1", StringType(), True).add("f2", StringType(), True, None)</span>
<span class="sd"> >>> struct2 = StructType([StructField("f1", StringType(), True),\</span>
<span class="sd"> StructField("f2", StringType(), True, None)])</span>
<span class="sd"> >>> struct1 == struct2</span>
<span class="sd"> True</span>
<span class="sd"> >>> struct1 = StructType().add(StructField("f1", StringType(), True))</span>
<span class="sd"> >>> struct2 = StructType([StructField("f1", StringType(), True)])</span>
<span class="sd"> >>> struct1 == struct2</span>
<span class="sd"> True</span>
<span class="sd"> >>> struct1 = StructType().add("f1", "string", True)</span>
<span class="sd"> >>> struct2 = StructType([StructField("f1", StringType(), True)])</span>
<span class="sd"> >>> struct1 == struct2</span>
<span class="sd"> True</span>
<span class="sd"> :param field: Either the name of the field or a StructField object</span>
<span class="sd"> :param data_type: If present, the DataType of the StructField to create</span>
<span class="sd"> :param nullable: Whether the field to add should be nullable (default True)</span>
<span class="sd"> :param metadata: Any additional metadata (default None)</span>
<span class="sd"> :return: a new updated StructType</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field</span><span class="p">,</span> <span class="n">StructField</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">field</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">field</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field</span><span class="p">,</span> <span class="nb">str</span><span class="p">)</span> <span class="ow">and</span> <span class="n">data_type</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Must specify DataType if passing name of struct_field to create."</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data_type</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="n">data_type_f</span> <span class="o">=</span> <span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">data_type</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">data_type_f</span> <span class="o">=</span> <span class="n">data_type</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">StructField</span><span class="p">(</span><span class="n">field</span><span class="p">,</span> <span class="n">data_type_f</span><span class="p">,</span> <span class="n">nullable</span><span class="p">,</span> <span class="n">metadata</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">field</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_needSerializeAnyField</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">needConversion</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span>
</div>
<div class="viewcode-block" id="StructType.simpleString"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.simpleString">[docs]</a> <span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'struct<</span><span class="si">%s</span><span class="s">>'</span> <span class="o">%</span> <span class="p">(</span><span class="s">','</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">simpleString</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">))</span>
</div>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span><span class="s">"StructType(List(</span><span class="si">%s</span><span class="s">))"</span> <span class="o">%</span>
<span class="s">","</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">field</span><span class="p">)</span> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">))</span>
<div class="viewcode-block" id="StructType.jsonValue"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.jsonValue">[docs]</a> <span class="k">def</span> <span class="nf">jsonValue</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">{</span><span class="s">"type"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">typeName</span><span class="p">(),</span>
<span class="s">"fields"</span><span class="p">:</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">]}</span>
</div>
<span class="nd">@classmethod</span>
<div class="viewcode-block" id="StructType.fromJson"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.fromJson">[docs]</a> <span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">([</span><span class="n">StructField</span><span class="o">.</span><span class="n">fromJson</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">json</span><span class="p">[</span><span class="s">"fields"</span><span class="p">]])</span>
</div>
<div class="viewcode-block" id="StructType.needConversion"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.needConversion">[docs]</a> <span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="c"># We need convert Row()/namedtuple into tuple()</span>
<span class="k">return</span> <span class="bp">True</span>
</div>
<div class="viewcode-block" id="StructType.toInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.toInternal">[docs]</a> <span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_needSerializeAnyField</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">n</span><span class="p">))</span> <span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">))</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">tuple</span><span class="p">,</span> <span class="nb">list</span><span class="p">)):</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">,</span> <span class="n">obj</span><span class="p">))</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s">"__dict__"</span><span class="p">):</span>
<span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">n</span><span class="p">))</span> <span class="k">for</span> <span class="n">n</span><span class="p">,</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Unexpected tuple </span><span class="si">%r</span><span class="s"> with StructType"</span> <span class="o">%</span> <span class="n">obj</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">n</span><span class="p">)</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s">"__dict__"</span><span class="p">):</span>
<span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">n</span><span class="p">)</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Unexpected tuple </span><span class="si">%r</span><span class="s"> with StructType"</span> <span class="o">%</span> <span class="n">obj</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="StructType.fromInternal"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.StructType.fromInternal">[docs]</a> <span class="k">def</span> <span class="nf">fromInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">Row</span><span class="p">):</span>
<span class="c"># it's already converted by pickler</span>
<span class="k">return</span> <span class="n">obj</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_needSerializeAnyField</span><span class="p">:</span>
<span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">fromInternal</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fields</span><span class="p">,</span> <span class="n">obj</span><span class="p">)]</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">values</span> <span class="o">=</span> <span class="n">obj</span>
<span class="k">return</span> <span class="n">_create_row</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">names</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span>
</div></div>
<span class="k">class</span> <span class="nc">UserDefinedType</span><span class="p">(</span><span class="n">DataType</span><span class="p">):</span>
<span class="sd">"""User-defined type (UDT).</span>
<span class="sd"> .. note:: WARN: Spark Internal Use Only</span>
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">typeName</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">__name__</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Underlying SQL storage type for this UDT.</span>
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s">"UDT must implement sqlType()."</span><span class="p">)</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> The Python module of the UDT.</span>
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s">"UDT must implement module()."</span><span class="p">)</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> The class name of the paired Scala UDT (could be '', if there</span>
<span class="sd"> is no corresponding one).</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="s">''</span>
<span class="k">def</span> <span class="nf">needConversion</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">True</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">_cachedSqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Cache the sqlType() into class, because it's heavy used in `toInternal`.</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="s">"_cached_sql_type"</span><span class="p">):</span>
<span class="n">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span> <span class="o">=</span> <span class="n">cls</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span>
<span class="k">return</span> <span class="n">cls</span><span class="o">.</span><span class="n">_cached_sql_type</span>
<span class="k">def</span> <span class="nf">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_cachedSqlType</span><span class="p">()</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">serialize</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">fromInternal</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_cachedSqlType</span><span class="p">()</span><span class="o">.</span><span class="n">fromInternal</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Converts the a user-type object into a SQL datum.</span>
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s">"UDT must implement toInternal()."</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">datum</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Converts a SQL datum into a user-type object.</span>
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s">"UDT must implement fromInternal()."</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">'udt'</span>
<span class="k">def</span> <span class="nf">json</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">(),</span> <span class="n">separators</span><span class="o">=</span><span class="p">(</span><span class="s">','</span><span class="p">,</span> <span class="s">':'</span><span class="p">),</span> <span class="n">sort_keys</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">jsonValue</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scalaUDT</span><span class="p">():</span>
<span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">()</span> <span class="o">!=</span> <span class="s">'__main__'</span><span class="p">,</span> <span class="s">'UDT in __main__ cannot work with ScalaUDT'</span>
<span class="n">schema</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">"type"</span><span class="p">:</span> <span class="s">"udt"</span><span class="p">,</span>
<span class="s">"class"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">scalaUDT</span><span class="p">(),</span>
<span class="s">"pyClass"</span><span class="p">:</span> <span class="s">"</span><span class="si">%s</span><span class="s">.</span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__name__</span><span class="p">),</span>
<span class="s">"sqlType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span>
<span class="p">}</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">ser</span> <span class="o">=</span> <span class="n">CloudPickleSerializer</span><span class="p">()</span>
<span class="n">b</span> <span class="o">=</span> <span class="n">ser</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
<span class="n">schema</span> <span class="o">=</span> <span class="p">{</span>
<span class="s">"type"</span><span class="p">:</span> <span class="s">"udt"</span><span class="p">,</span>
<span class="s">"pyClass"</span><span class="p">:</span> <span class="s">"</span><span class="si">%s</span><span class="s">.</span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(),</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__name__</span><span class="p">),</span>
<span class="s">"serializedClass"</span><span class="p">:</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">b</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s">'utf8'</span><span class="p">),</span>
<span class="s">"sqlType"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">sqlType</span><span class="p">()</span><span class="o">.</span><span class="n">jsonValue</span><span class="p">()</span>
<span class="p">}</span>
<span class="k">return</span> <span class="n">schema</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">fromJson</span><span class="p">(</span><span class="n">cls</span><span class="p">,</span> <span class="n">json</span><span class="p">):</span>
<span class="n">pyUDT</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s">"pyClass"</span><span class="p">])</span> <span class="c"># convert unicode to str</span>
<span class="n">split</span> <span class="o">=</span> <span class="n">pyUDT</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s">"."</span><span class="p">)</span>
<span class="n">pyModule</span> <span class="o">=</span> <span class="n">pyUDT</span><span class="p">[:</span><span class="n">split</span><span class="p">]</span>
<span class="n">pyClass</span> <span class="o">=</span> <span class="n">pyUDT</span><span class="p">[</span><span class="n">split</span><span class="o">+</span><span class="mi">1</span><span class="p">:]</span>
<span class="n">m</span> <span class="o">=</span> <span class="nb">__import__</span><span class="p">(</span><span class="n">pyModule</span><span class="p">,</span> <span class="nb">globals</span><span class="p">(),</span> <span class="nb">locals</span><span class="p">(),</span> <span class="p">[</span><span class="n">pyClass</span><span class="p">])</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">pyClass</span><span class="p">):</span>
<span class="n">s</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">json</span><span class="p">[</span><span class="s">'serializedClass'</span><span class="p">]</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s">'utf-8'</span><span class="p">))</span>
<span class="n">UDT</span> <span class="o">=</span> <span class="n">CloudPickleSerializer</span><span class="p">()</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">UDT</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">pyClass</span><span class="p">)</span>
<span class="k">return</span> <span class="n">UDT</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="n">_atomic_types</span> <span class="o">=</span> <span class="p">[</span><span class="n">StringType</span><span class="p">,</span> <span class="n">BinaryType</span><span class="p">,</span> <span class="n">BooleanType</span><span class="p">,</span> <span class="n">DecimalType</span><span class="p">,</span> <span class="n">FloatType</span><span class="p">,</span> <span class="n">DoubleType</span><span class="p">,</span>
<span class="n">ByteType</span><span class="p">,</span> <span class="n">ShortType</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">,</span> <span class="n">LongType</span><span class="p">,</span> <span class="n">DateType</span><span class="p">,</span> <span class="n">TimestampType</span><span class="p">,</span> <span class="n">NullType</span><span class="p">]</span>
<span class="n">_all_atomic_types</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">((</span><span class="n">t</span><span class="o">.</span><span class="n">typeName</span><span class="p">(),</span> <span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">_atomic_types</span><span class="p">)</span>
<span class="n">_all_complex_types</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">((</span><span class="n">v</span><span class="o">.</span><span class="n">typeName</span><span class="p">(),</span> <span class="n">v</span><span class="p">)</span>
<span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="p">[</span><span class="n">ArrayType</span><span class="p">,</span> <span class="n">MapType</span><span class="p">,</span> <span class="n">StructType</span><span class="p">])</span>
<span class="k">def</span> <span class="nf">_parse_datatype_json_string</span><span class="p">(</span><span class="n">json_string</span><span class="p">):</span>
<span class="sd">"""Parses the given data type JSON string.</span>
<span class="sd"> >>> import pickle</span>
<span class="sd"> >>> def check_datatype(datatype):</span>
<span class="sd"> ... pickled = pickle.loads(pickle.dumps(datatype))</span>
<span class="sd"> ... assert datatype == pickled</span>
<span class="sd"> ... scala_datatype = sqlContext._ssql_ctx.parseDataType(datatype.json())</span>
<span class="sd"> ... python_datatype = _parse_datatype_json_string(scala_datatype.json())</span>
<span class="sd"> ... assert datatype == python_datatype</span>
<span class="sd"> >>> for cls in _all_atomic_types.values():</span>
<span class="sd"> ... check_datatype(cls())</span>
<span class="sd"> >>> # Simple ArrayType.</span>
<span class="sd"> >>> simple_arraytype = ArrayType(StringType(), True)</span>
<span class="sd"> >>> check_datatype(simple_arraytype)</span>
<span class="sd"> >>> # Simple MapType.</span>
<span class="sd"> >>> simple_maptype = MapType(StringType(), LongType())</span>
<span class="sd"> >>> check_datatype(simple_maptype)</span>
<span class="sd"> >>> # Simple StructType.</span>
<span class="sd"> >>> simple_structtype = StructType([</span>
<span class="sd"> ... StructField("a", DecimalType(), False),</span>
<span class="sd"> ... StructField("b", BooleanType(), True),</span>
<span class="sd"> ... StructField("c", LongType(), True),</span>
<span class="sd"> ... StructField("d", BinaryType(), False)])</span>
<span class="sd"> >>> check_datatype(simple_structtype)</span>
<span class="sd"> >>> # Complex StructType.</span>
<span class="sd"> >>> complex_structtype = StructType([</span>
<span class="sd"> ... StructField("simpleArray", simple_arraytype, True),</span>
<span class="sd"> ... StructField("simpleMap", simple_maptype, True),</span>
<span class="sd"> ... StructField("simpleStruct", simple_structtype, True),</span>
<span class="sd"> ... StructField("boolean", BooleanType(), False),</span>
<span class="sd"> ... StructField("withMeta", DoubleType(), False, {"name": "age"})])</span>
<span class="sd"> >>> check_datatype(complex_structtype)</span>
<span class="sd"> >>> # Complex ArrayType.</span>
<span class="sd"> >>> complex_arraytype = ArrayType(complex_structtype, True)</span>
<span class="sd"> >>> check_datatype(complex_arraytype)</span>
<span class="sd"> >>> # Complex MapType.</span>
<span class="sd"> >>> complex_maptype = MapType(complex_structtype,</span>
<span class="sd"> ... complex_arraytype, False)</span>
<span class="sd"> >>> check_datatype(complex_maptype)</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">json_string</span><span class="p">))</span>
<span class="n">_FIXED_DECIMAL</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="s">"decimal</span><span class="se">\\</span><span class="s">((</span><span class="se">\\</span><span class="s">d+),(</span><span class="se">\\</span><span class="s">d+)</span><span class="se">\\</span><span class="s">)"</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_parse_datatype_json_value</span><span class="p">(</span><span class="n">json_value</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">json_value</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">if</span> <span class="n">json_value</span> <span class="ow">in</span> <span class="n">_all_atomic_types</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="k">return</span> <span class="n">_all_atomic_types</span><span class="p">[</span><span class="n">json_value</span><span class="p">]()</span>
<span class="k">elif</span> <span class="n">json_value</span> <span class="o">==</span> <span class="s">'decimal'</span><span class="p">:</span>
<span class="k">return</span> <span class="n">DecimalType</span><span class="p">()</span>
<span class="k">elif</span> <span class="n">_FIXED_DECIMAL</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">json_value</span><span class="p">):</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">_FIXED_DECIMAL</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="n">json_value</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DecimalType</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">)),</span> <span class="nb">int</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Could not parse datatype: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">json_value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">tpe</span> <span class="o">=</span> <span class="n">json_value</span><span class="p">[</span><span class="s">"type"</span><span class="p">]</span>
<span class="k">if</span> <span class="n">tpe</span> <span class="ow">in</span> <span class="n">_all_complex_types</span><span class="p">:</span>
<span class="k">return</span> <span class="n">_all_complex_types</span><span class="p">[</span><span class="n">tpe</span><span class="p">]</span><span class="o">.</span><span class="n">fromJson</span><span class="p">(</span><span class="n">json_value</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">tpe</span> <span class="o">==</span> <span class="s">'udt'</span><span class="p">:</span>
<span class="k">return</span> <span class="n">UserDefinedType</span><span class="o">.</span><span class="n">fromJson</span><span class="p">(</span><span class="n">json_value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"not supported type: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">tpe</span><span class="p">)</span>
<span class="c"># Mapping Python types to Spark SQL DataType</span>
<span class="n">_type_mappings</span> <span class="o">=</span> <span class="p">{</span>
<span class="nb">type</span><span class="p">(</span><span class="bp">None</span><span class="p">):</span> <span class="n">NullType</span><span class="p">,</span>
<span class="nb">bool</span><span class="p">:</span> <span class="n">BooleanType</span><span class="p">,</span>
<span class="nb">int</span><span class="p">:</span> <span class="n">LongType</span><span class="p">,</span>
<span class="nb">float</span><span class="p">:</span> <span class="n">DoubleType</span><span class="p">,</span>
<span class="nb">str</span><span class="p">:</span> <span class="n">StringType</span><span class="p">,</span>
<span class="nb">bytearray</span><span class="p">:</span> <span class="n">BinaryType</span><span class="p">,</span>
<span class="n">decimal</span><span class="o">.</span><span class="n">Decimal</span><span class="p">:</span> <span class="n">DecimalType</span><span class="p">,</span>
<span class="n">datetime</span><span class="o">.</span><span class="n">date</span><span class="p">:</span> <span class="n">DateType</span><span class="p">,</span>
<span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="p">:</span> <span class="n">TimestampType</span><span class="p">,</span>
<span class="n">datetime</span><span class="o">.</span><span class="n">time</span><span class="p">:</span> <span class="n">TimestampType</span><span class="p">,</span>
<span class="p">}</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version</span> <span class="o"><</span> <span class="s">"3"</span><span class="p">:</span>
<span class="n">_type_mappings</span><span class="o">.</span><span class="n">update</span><span class="p">({</span>
<span class="nb">unicode</span><span class="p">:</span> <span class="n">StringType</span><span class="p">,</span>
<span class="nb">long</span><span class="p">:</span> <span class="n">LongType</span><span class="p">,</span>
<span class="p">})</span>
<span class="k">def</span> <span class="nf">_infer_type</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
<span class="sd">"""Infer the DataType from obj</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">NullType</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s">'__UDT__'</span><span class="p">):</span>
<span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">__UDT__</span>
<span class="n">dataType</span> <span class="o">=</span> <span class="n">_type_mappings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
<span class="k">if</span> <span class="n">dataType</span> <span class="ow">is</span> <span class="n">DecimalType</span><span class="p">:</span>
<span class="c"># the precision and scale of `obj` may be different from row to row.</span>
<span class="k">return</span> <span class="n">DecimalType</span><span class="p">(</span><span class="mi">38</span><span class="p">,</span> <span class="mi">18</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">dataType</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">dataType</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="k">if</span> <span class="n">key</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span> <span class="ow">and</span> <span class="n">value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">MapType</span><span class="p">(</span><span class="n">_infer_type</span><span class="p">(</span><span class="n">key</span><span class="p">),</span> <span class="n">_infer_type</span><span class="p">(</span><span class="n">value</span><span class="p">),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">MapType</span><span class="p">(</span><span class="n">NullType</span><span class="p">(),</span> <span class="n">NullType</span><span class="p">(),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="n">array</span><span class="p">)):</span>
<span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">:</span>
<span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">_infer_type</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="mi">0</span><span class="p">]),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">NullType</span><span class="p">(),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">return</span> <span class="n">_infer_schema</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">TypeError</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"not supported type: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">_infer_schema</span><span class="p">(</span><span class="n">row</span><span class="p">):</span>
<span class="sd">"""Infer the schema from dict/namedtuple/object"""</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">items</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="p">(</span><span class="nb">tuple</span><span class="p">,</span> <span class="nb">list</span><span class="p">)):</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="s">"__fields__"</span><span class="p">):</span> <span class="c"># Row</span>
<span class="n">items</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="n">__fields__</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">row</span><span class="p">))</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="s">"_fields"</span><span class="p">):</span> <span class="c"># namedtuple</span>
<span class="n">items</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="n">_fields</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">row</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">names</span> <span class="o">=</span> <span class="p">[</span><span class="s">'_</span><span class="si">%d</span><span class="s">'</span> <span class="o">%</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">row</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)]</span>
<span class="n">items</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">row</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">row</span><span class="p">,</span> <span class="s">"__dict__"</span><span class="p">):</span> <span class="c"># object</span>
<span class="n">items</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">row</span><span class="o">.</span><span class="n">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"Can not infer schema for type: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">row</span><span class="p">))</span>
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="n">StructField</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">_infer_type</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="bp">True</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">items</span><span class="p">]</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">(</span><span class="n">fields</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_has_nulltype</span><span class="p">(</span><span class="n">dt</span><span class="p">):</span>
<span class="sd">""" Return whether there is NullType in `dt` or not """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dt</span><span class="p">,</span> <span class="n">StructType</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">any</span><span class="p">(</span><span class="n">_has_nulltype</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">dt</span><span class="o">.</span><span class="n">fields</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dt</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_has_nulltype</span><span class="p">((</span><span class="n">dt</span><span class="o">.</span><span class="n">elementType</span><span class="p">))</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dt</span><span class="p">,</span> <span class="n">MapType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_has_nulltype</span><span class="p">(</span><span class="n">dt</span><span class="o">.</span><span class="n">keyType</span><span class="p">)</span> <span class="ow">or</span> <span class="n">_has_nulltype</span><span class="p">(</span><span class="n">dt</span><span class="o">.</span><span class="n">valueType</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dt</span><span class="p">,</span> <span class="n">NullType</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_merge_type</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">NullType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">b</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">b</span><span class="p">,</span> <span class="n">NullType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">a</span>
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">a</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="nb">type</span><span class="p">(</span><span class="n">b</span><span class="p">):</span>
<span class="c"># TODO: type cast (such as int -> long)</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"Can not merge type </span><span class="si">%s</span><span class="s"> and </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">a</span><span class="p">),</span> <span class="nb">type</span><span class="p">(</span><span class="n">b</span><span class="p">)))</span>
<span class="c"># same type</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">StructType</span><span class="p">):</span>
<span class="n">nfs</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">((</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">b</span><span class="o">.</span><span class="n">fields</span><span class="p">)</span>
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="n">StructField</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">_merge_type</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">,</span> <span class="n">nfs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">NullType</span><span class="p">())))</span>
<span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">a</span><span class="o">.</span><span class="n">fields</span><span class="p">]</span>
<span class="n">names</span> <span class="o">=</span> <span class="nb">set</span><span class="p">([</span><span class="n">f</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">])</span>
<span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">nfs</span><span class="p">:</span>
<span class="k">if</span> <span class="n">n</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
<span class="n">fields</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">StructField</span><span class="p">(</span><span class="n">n</span><span class="p">,</span> <span class="n">nfs</span><span class="p">[</span><span class="n">n</span><span class="p">]))</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">(</span><span class="n">fields</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">_merge_type</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">elementType</span><span class="p">,</span> <span class="n">b</span><span class="o">.</span><span class="n">elementType</span><span class="p">),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">MapType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">MapType</span><span class="p">(</span><span class="n">_merge_type</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">keyType</span><span class="p">,</span> <span class="n">b</span><span class="o">.</span><span class="n">keyType</span><span class="p">),</span>
<span class="n">_merge_type</span><span class="p">(</span><span class="n">a</span><span class="o">.</span><span class="n">valueType</span><span class="p">,</span> <span class="n">b</span><span class="o">.</span><span class="n">valueType</span><span class="p">),</span>
<span class="bp">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">a</span>
<span class="k">def</span> <span class="nf">_need_converter</span><span class="p">(</span><span class="n">dataType</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">StructType</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">True</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_need_converter</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">elementType</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">MapType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_need_converter</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">keyType</span><span class="p">)</span> <span class="ow">or</span> <span class="n">_need_converter</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">valueType</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">NullType</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">True</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">False</span>
<span class="k">def</span> <span class="nf">_create_converter</span><span class="p">(</span><span class="n">dataType</span><span class="p">):</span>
<span class="sd">"""Create an converter to drop the names of fields in obj """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">_need_converter</span><span class="p">(</span><span class="n">dataType</span><span class="p">):</span>
<span class="k">return</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">):</span>
<span class="n">conv</span> <span class="o">=</span> <span class="n">_create_converter</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">elementType</span><span class="p">)</span>
<span class="k">return</span> <span class="k">lambda</span> <span class="n">row</span><span class="p">:</span> <span class="p">[</span><span class="n">conv</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">row</span><span class="p">]</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">MapType</span><span class="p">):</span>
<span class="n">kconv</span> <span class="o">=</span> <span class="n">_create_converter</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">keyType</span><span class="p">)</span>
<span class="n">vconv</span> <span class="o">=</span> <span class="n">_create_converter</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">valueType</span><span class="p">)</span>
<span class="k">return</span> <span class="k">lambda</span> <span class="n">row</span><span class="p">:</span> <span class="nb">dict</span><span class="p">((</span><span class="n">kconv</span><span class="p">(</span><span class="n">k</span><span class="p">),</span> <span class="n">vconv</span><span class="p">(</span><span class="n">v</span><span class="p">))</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">row</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">NullType</span><span class="p">):</span>
<span class="k">return</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="bp">None</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">StructType</span><span class="p">):</span>
<span class="k">return</span> <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span>
<span class="c"># dataType must be StructType</span>
<span class="n">names</span> <span class="o">=</span> <span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">]</span>
<span class="n">converters</span> <span class="o">=</span> <span class="p">[</span><span class="n">_create_converter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">]</span>
<span class="n">convert_fields</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span><span class="n">_need_converter</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">convert_struct</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">tuple</span><span class="p">,</span> <span class="nb">list</span><span class="p">)):</span>
<span class="k">if</span> <span class="n">convert_fields</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">conv</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">conv</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">converters</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">d</span> <span class="o">=</span> <span class="n">obj</span>
<span class="k">elif</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s">"__dict__"</span><span class="p">):</span> <span class="c"># object</span>
<span class="n">d</span> <span class="o">=</span> <span class="n">obj</span><span class="o">.</span><span class="n">__dict__</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"Unexpected obj type: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
<span class="k">if</span> <span class="n">convert_fields</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">([</span><span class="n">conv</span><span class="p">(</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">))</span> <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">conv</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">converters</span><span class="p">)])</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="p">([</span><span class="n">d</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">])</span>
<span class="k">return</span> <span class="n">convert_struct</span>
<span class="n">_BRACKETS</span> <span class="o">=</span> <span class="p">{</span><span class="s">'('</span><span class="p">:</span> <span class="s">')'</span><span class="p">,</span> <span class="s">'['</span><span class="p">:</span> <span class="s">']'</span><span class="p">,</span> <span class="s">'{'</span><span class="p">:</span> <span class="s">'}'</span><span class="p">}</span>
<span class="k">def</span> <span class="nf">_split_schema_abstract</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> split the schema abstract into fields</span>
<span class="sd"> >>> _split_schema_abstract("a b c")</span>
<span class="sd"> ['a', 'b', 'c']</span>
<span class="sd"> >>> _split_schema_abstract("a(a b)")</span>
<span class="sd"> ['a(a b)']</span>
<span class="sd"> >>> _split_schema_abstract("a b[] c{a b}")</span>
<span class="sd"> ['a', 'b[]', 'c{a b}']</span>
<span class="sd"> >>> _split_schema_abstract(" ")</span>
<span class="sd"> []</span>
<span class="sd"> """</span>
<span class="n">r</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">w</span> <span class="o">=</span> <span class="s">''</span>
<span class="n">brackets</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">s</span><span class="p">:</span>
<span class="k">if</span> <span class="n">c</span> <span class="o">==</span> <span class="s">' '</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">brackets</span><span class="p">:</span>
<span class="k">if</span> <span class="n">w</span><span class="p">:</span>
<span class="n">r</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
<span class="n">w</span> <span class="o">=</span> <span class="s">''</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">w</span> <span class="o">+=</span> <span class="n">c</span>
<span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">_BRACKETS</span><span class="p">:</span>
<span class="n">brackets</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">_BRACKETS</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">brackets</span> <span class="ow">or</span> <span class="n">c</span> <span class="o">!=</span> <span class="n">_BRACKETS</span><span class="p">[</span><span class="n">brackets</span><span class="o">.</span><span class="n">pop</span><span class="p">()]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"unexpected "</span> <span class="o">+</span> <span class="n">c</span><span class="p">)</span>
<span class="k">if</span> <span class="n">brackets</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"brackets not closed: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">brackets</span><span class="p">)</span>
<span class="k">if</span> <span class="n">w</span><span class="p">:</span>
<span class="n">r</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
<span class="k">return</span> <span class="n">r</span>
<span class="k">def</span> <span class="nf">_parse_field_abstract</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Parse a field in schema abstract</span>
<span class="sd"> >>> _parse_field_abstract("a")</span>
<span class="sd"> StructField(a,NullType,true)</span>
<span class="sd"> >>> _parse_field_abstract("b(c d)")</span>
<span class="sd"> StructField(b,StructType(...c,NullType,true),StructField(d...</span>
<span class="sd"> >>> _parse_field_abstract("a[]")</span>
<span class="sd"> StructField(a,ArrayType(NullType,true),true)</span>
<span class="sd"> >>> _parse_field_abstract("a{[]}")</span>
<span class="sd"> StructField(a,MapType(NullType,ArrayType(NullType,true),true),true)</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">set</span><span class="p">(</span><span class="n">_BRACKETS</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="o">&</span> <span class="nb">set</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
<span class="n">idx</span> <span class="o">=</span> <span class="nb">min</span><span class="p">((</span><span class="n">s</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">_BRACKETS</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">s</span><span class="p">))</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">s</span><span class="p">[:</span><span class="n">idx</span><span class="p">]</span>
<span class="k">return</span> <span class="n">StructField</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">_parse_schema_abstract</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="n">idx</span><span class="p">:]),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">StructField</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">NullType</span><span class="p">(),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_parse_schema_abstract</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> parse abstract into schema</span>
<span class="sd"> >>> _parse_schema_abstract("a b c")</span>
<span class="sd"> StructType...a...b...c...</span>
<span class="sd"> >>> _parse_schema_abstract("a[b c] b{}")</span>
<span class="sd"> StructType...a,ArrayType...b...c...b,MapType...</span>
<span class="sd"> >>> _parse_schema_abstract("c{} d{a b}")</span>
<span class="sd"> StructType...c,MapType...d,MapType...a...b...</span>
<span class="sd"> >>> _parse_schema_abstract("a b(t)").fields[1]</span>
<span class="sd"> StructField(b,StructType(List(StructField(t,NullType,true))),true)</span>
<span class="sd"> """</span>
<span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">s</span><span class="p">:</span>
<span class="k">return</span> <span class="n">NullType</span><span class="p">()</span>
<span class="k">elif</span> <span class="n">s</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s">'('</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_parse_schema_abstract</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
<span class="k">elif</span> <span class="n">s</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s">'['</span><span class="p">):</span>
<span class="k">return</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">_parse_schema_abstract</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">s</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s">'{'</span><span class="p">):</span>
<span class="k">return</span> <span class="n">MapType</span><span class="p">(</span><span class="n">NullType</span><span class="p">(),</span> <span class="n">_parse_schema_abstract</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span>
<span class="n">parts</span> <span class="o">=</span> <span class="n">_split_schema_abstract</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="n">_parse_field_abstract</span><span class="p">(</span><span class="n">p</span><span class="p">)</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">parts</span><span class="p">]</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">(</span><span class="n">fields</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_infer_schema_type</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">dataType</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Fill the dataType with types inferred from obj</span>
<span class="sd"> >>> schema = _parse_schema_abstract("a b c d")</span>
<span class="sd"> >>> row = (1, 1.0, "str", datetime.date(2014, 10, 10))</span>
<span class="sd"> >>> _infer_schema_type(row, schema)</span>
<span class="sd"> StructType...LongType...DoubleType...StringType...DateType...</span>
<span class="sd"> >>> row = [[1], {"key": (1, 2.0)}]</span>
<span class="sd"> >>> schema = _parse_schema_abstract("a[] b{c d}")</span>
<span class="sd"> >>> _infer_schema_type(row, schema)</span>
<span class="sd"> StructType...a,ArrayType...b,MapType(StringType,...c,LongType...</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">NullType</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_infer_type</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">obj</span><span class="p">:</span>
<span class="k">return</span> <span class="n">NullType</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">):</span>
<span class="n">eType</span> <span class="o">=</span> <span class="n">_infer_schema_type</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">dataType</span><span class="o">.</span><span class="n">elementType</span><span class="p">)</span>
<span class="k">return</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">eType</span><span class="p">,</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">MapType</span><span class="p">):</span>
<span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">()))</span>
<span class="k">return</span> <span class="n">MapType</span><span class="p">(</span><span class="n">_infer_schema_type</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">dataType</span><span class="o">.</span><span class="n">keyType</span><span class="p">),</span>
<span class="n">_infer_schema_type</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">dataType</span><span class="o">.</span><span class="n">valueType</span><span class="p">))</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">StructType</span><span class="p">):</span>
<span class="n">fs</span> <span class="o">=</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">fs</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">obj</span><span class="p">),</span> \
<span class="s">"Obj(</span><span class="si">%s</span><span class="s">) have different length with fields(</span><span class="si">%s</span><span class="s">)"</span> <span class="o">%</span> <span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">fs</span><span class="p">)</span>
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="n">StructField</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">_infer_schema_type</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">),</span> <span class="bp">True</span><span class="p">)</span>
<span class="k">for</span> <span class="n">o</span><span class="p">,</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">fs</span><span class="p">)]</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">(</span><span class="n">fields</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"Unexpected dataType: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">dataType</span><span class="p">))</span>
<span class="n">_acceptable_types</span> <span class="o">=</span> <span class="p">{</span>
<span class="n">BooleanType</span><span class="p">:</span> <span class="p">(</span><span class="nb">bool</span><span class="p">,),</span>
<span class="n">ByteType</span><span class="p">:</span> <span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">),</span>
<span class="n">ShortType</span><span class="p">:</span> <span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">),</span>
<span class="n">IntegerType</span><span class="p">:</span> <span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">),</span>
<span class="n">LongType</span><span class="p">:</span> <span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">),</span>
<span class="n">FloatType</span><span class="p">:</span> <span class="p">(</span><span class="nb">float</span><span class="p">,),</span>
<span class="n">DoubleType</span><span class="p">:</span> <span class="p">(</span><span class="nb">float</span><span class="p">,),</span>
<span class="n">DecimalType</span><span class="p">:</span> <span class="p">(</span><span class="n">decimal</span><span class="o">.</span><span class="n">Decimal</span><span class="p">,),</span>
<span class="n">StringType</span><span class="p">:</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="nb">unicode</span><span class="p">),</span>
<span class="n">BinaryType</span><span class="p">:</span> <span class="p">(</span><span class="nb">bytearray</span><span class="p">,),</span>
<span class="n">DateType</span><span class="p">:</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">date</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="p">),</span>
<span class="n">TimestampType</span><span class="p">:</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="p">,),</span>
<span class="n">ArrayType</span><span class="p">:</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">,</span> <span class="n">array</span><span class="p">),</span>
<span class="n">MapType</span><span class="p">:</span> <span class="p">(</span><span class="nb">dict</span><span class="p">,),</span>
<span class="n">StructType</span><span class="p">:</span> <span class="p">(</span><span class="nb">tuple</span><span class="p">,</span> <span class="nb">list</span><span class="p">),</span>
<span class="p">}</span>
<span class="k">def</span> <span class="nf">_verify_type</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">dataType</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Verify the type of obj against dataType, raise an exception if</span>
<span class="sd"> they do not match.</span>
<span class="sd"> >>> _verify_type(None, StructType([]))</span>
<span class="sd"> >>> _verify_type("", StringType())</span>
<span class="sd"> >>> _verify_type(0, LongType())</span>
<span class="sd"> >>> _verify_type(list(range(3)), ArrayType(ShortType()))</span>
<span class="sd"> >>> _verify_type(set(), ArrayType(StringType())) # doctest: +IGNORE_EXCEPTION_DETAIL</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> TypeError:...</span>
<span class="sd"> >>> _verify_type({}, MapType(StringType(), IntegerType()))</span>
<span class="sd"> >>> _verify_type((), StructType([]))</span>
<span class="sd"> >>> _verify_type([], StructType([]))</span>
<span class="sd"> >>> _verify_type([1], StructType([])) # doctest: +IGNORE_EXCEPTION_DETAIL</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> ValueError:...</span>
<span class="sd"> """</span>
<span class="c"># all objects are nullable</span>
<span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
<span class="k">return</span>
<span class="c"># StringType can work with any types</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">StringType</span><span class="p">):</span>
<span class="k">return</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">UserDefinedType</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="nb">hasattr</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="s">'__UDT__'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">obj</span><span class="o">.</span><span class="n">__UDT__</span> <span class="o">==</span> <span class="n">dataType</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"</span><span class="si">%r</span><span class="s"> is not an instance of type </span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">dataType</span><span class="p">))</span>
<span class="n">_verify_type</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">toInternal</span><span class="p">(</span><span class="n">obj</span><span class="p">),</span> <span class="n">dataType</span><span class="o">.</span><span class="n">sqlType</span><span class="p">())</span>
<span class="k">return</span>
<span class="n">_type</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">dataType</span><span class="p">)</span>
<span class="k">assert</span> <span class="n">_type</span> <span class="ow">in</span> <span class="n">_acceptable_types</span><span class="p">,</span> <span class="s">"unknown datatype: </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">dataType</span>
<span class="k">if</span> <span class="n">_type</span> <span class="ow">is</span> <span class="n">StructType</span><span class="p">:</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">tuple</span><span class="p">,</span> <span class="nb">list</span><span class="p">)):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"StructType can not accept object in type </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="c"># subclass of them can not be fromInternald in JVM</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">_acceptable_types</span><span class="p">[</span><span class="n">_type</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"</span><span class="si">%s</span><span class="s"> can not accept object in type </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">)))</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">):</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">:</span>
<span class="n">_verify_type</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="n">dataType</span><span class="o">.</span><span class="n">elementType</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">MapType</span><span class="p">):</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">_verify_type</span><span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">dataType</span><span class="o">.</span><span class="n">keyType</span><span class="p">)</span>
<span class="n">_verify_type</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">dataType</span><span class="o">.</span><span class="n">valueType</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">dataType</span><span class="p">,</span> <span class="n">StructType</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Length of object (</span><span class="si">%d</span><span class="s">) does not match with "</span>
<span class="s">"length of fields (</span><span class="si">%d</span><span class="s">)"</span> <span class="o">%</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">obj</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">)))</span>
<span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fields</span><span class="p">):</span>
<span class="n">_verify_type</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">dataType</span><span class="p">)</span>
<span class="c"># This is used to unpickle a Row from JVM</span>
<span class="k">def</span> <span class="nf">_create_row_inbound_converter</span><span class="p">(</span><span class="n">dataType</span><span class="p">):</span>
<span class="k">return</span> <span class="k">lambda</span> <span class="o">*</span><span class="n">a</span><span class="p">:</span> <span class="n">dataType</span><span class="o">.</span><span class="n">fromInternal</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_create_row</span><span class="p">(</span><span class="n">fields</span><span class="p">,</span> <span class="n">values</span><span class="p">):</span>
<span class="n">row</span> <span class="o">=</span> <span class="n">Row</span><span class="p">(</span><span class="o">*</span><span class="n">values</span><span class="p">)</span>
<span class="n">row</span><span class="o">.</span><span class="n">__fields__</span> <span class="o">=</span> <span class="n">fields</span>
<span class="k">return</span> <span class="n">row</span>
<div class="viewcode-block" id="Row"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.Row">[docs]</a><span class="k">class</span> <span class="nc">Row</span><span class="p">(</span><span class="nb">tuple</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> A row in L{DataFrame}. The fields in it can be accessed like attributes.</span>
<span class="sd"> Row can be used to create a row object by using named arguments,</span>
<span class="sd"> the fields will be sorted by names.</span>
<span class="sd"> >>> row = Row(name="Alice", age=11)</span>
<span class="sd"> >>> row</span>
<span class="sd"> Row(age=11, name='Alice')</span>
<span class="sd"> >>> row.name, row.age</span>
<span class="sd"> ('Alice', 11)</span>
<span class="sd"> Row also can be used to create another Row like class, then it</span>
<span class="sd"> could be used to create Row objects, such as</span>
<span class="sd"> >>> Person = Row("name", "age")</span>
<span class="sd"> >>> Person</span>
<span class="sd"> <Row(name, age)></span>
<span class="sd"> >>> Person("Alice", 11)</span>
<span class="sd"> Row(name='Alice', age=11)</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="k">if</span> <span class="n">args</span> <span class="ow">and</span> <span class="n">kwargs</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Can not use both args "</span>
<span class="s">"and kwargs to create Row"</span><span class="p">)</span>
<span class="k">if</span> <span class="n">args</span><span class="p">:</span>
<span class="c"># create row class or objects</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">kwargs</span><span class="p">:</span>
<span class="c"># create row objects</span>
<span class="n">names</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">kwargs</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
<span class="n">row</span> <span class="o">=</span> <span class="nb">tuple</span><span class="o">.</span><span class="n">__new__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="p">[</span><span class="n">kwargs</span><span class="p">[</span><span class="n">n</span><span class="p">]</span> <span class="k">for</span> <span class="n">n</span> <span class="ow">in</span> <span class="n">names</span><span class="p">])</span>
<span class="n">row</span><span class="o">.</span><span class="n">__fields__</span> <span class="o">=</span> <span class="n">names</span>
<span class="k">return</span> <span class="n">row</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"No args or kwargs"</span><span class="p">)</span>
<div class="viewcode-block" id="Row.asDict"><a class="viewcode-back" href="../../../pyspark.sql.html#pyspark.sql.Row.asDict">[docs]</a> <span class="k">def</span> <span class="nf">asDict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">recursive</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Return as an dict</span>
<span class="sd"> :param recursive: turns the nested Row as dict (default: False).</span>
<span class="sd"> >>> Row(name="Alice", age=11).asDict() == {'name': 'Alice', 'age': 11}</span>
<span class="sd"> True</span>
<span class="sd"> >>> row = Row(key=1, value=Row(name='a', age=2))</span>
<span class="sd"> >>> row.asDict() == {'key': 1, 'value': Row(age=2, name='a')}</span>
<span class="sd"> True</span>
<span class="sd"> >>> row.asDict(True) == {'key': 1, 'value': {'name': 'a', 'age': 2}}</span>
<span class="sd"> True</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s">"__fields__"</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"Cannot convert a Row class into dict"</span><span class="p">)</span>
<span class="k">if</span> <span class="n">recursive</span><span class="p">:</span>
<span class="k">def</span> <span class="nf">conv</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">Row</span><span class="p">):</span>
<span class="k">return</span> <span class="n">obj</span><span class="o">.</span><span class="n">asDict</span><span class="p">(</span><span class="bp">True</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">return</span> <span class="p">[</span><span class="n">conv</span><span class="p">(</span><span class="n">o</span><span class="p">)</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">]</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">dict</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="n">conv</span><span class="p">(</span><span class="n">v</span><span class="p">))</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">obj</span>
<span class="k">return</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span><span class="p">,</span> <span class="p">(</span><span class="n">conv</span><span class="p">(</span><span class="n">o</span><span class="p">)</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">)))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">dict</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span><span class="p">,</span> <span class="bp">self</span><span class="p">))</span>
<span class="c"># let object acts like class</span></div>
<span class="k">def</span> <span class="nf">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
<span class="sd">"""create new Row object"""</span>
<span class="k">return</span> <span class="n">_create_row</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">args</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
<span class="k">if</span> <span class="n">item</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s">"__"</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="c"># it will be slow when it has many fields,</span>
<span class="c"># but this will not be used in normal cases</span>
<span class="n">idx</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">IndexError</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__setattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
<span class="k">if</span> <span class="n">key</span> <span class="o">!=</span> <span class="s">'__fields__'</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s">"Row is read-only"</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">__dict__</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">value</span>
<span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Returns a tuple so Python knows how to pickle Row."""</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s">"__fields__"</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span><span class="n">_create_row</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">(</span><span class="bp">self</span><span class="p">)))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">tuple</span><span class="o">.</span><span class="n">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Printable representation of Row used in Python REPL."""</span>
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s">"__fields__"</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"Row(</span><span class="si">%s</span><span class="s">)"</span> <span class="o">%</span> <span class="s">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s">"</span><span class="si">%s</span><span class="s">=</span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">__fields__</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">(</span><span class="bp">self</span><span class="p">)))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="s">"<Row(</span><span class="si">%s</span><span class="s">)>"</span> <span class="o">%</span> <span class="s">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
</div>
<span class="k">class</span> <span class="nc">DateConverter</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">can_convert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">date</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">convert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">,</span> <span class="n">gateway_client</span><span class="p">):</span>
<span class="n">Date</span> <span class="o">=</span> <span class="n">JavaClass</span><span class="p">(</span><span class="s">"java.sql.Date"</span><span class="p">,</span> <span class="n">gateway_client</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Date</span><span class="o">.</span><span class="n">valueOf</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s">"%Y-%m-</span><span class="si">%d</span><span class="s">"</span><span class="p">))</span>
<span class="k">class</span> <span class="nc">DatetimeConverter</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">can_convert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">convert</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">,</span> <span class="n">gateway_client</span><span class="p">):</span>
<span class="n">Timestamp</span> <span class="o">=</span> <span class="n">JavaClass</span><span class="p">(</span><span class="s">"java.sql.Timestamp"</span><span class="p">,</span> <span class="n">gateway_client</span><span class="p">)</span>
<span class="k">return</span> <span class="n">Timestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">mktime</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">timetuple</span><span class="p">()))</span> <span class="o">*</span> <span class="mi">1000</span> <span class="o">+</span> <span class="n">obj</span><span class="o">.</span><span class="n">microsecond</span> <span class="o">//</span> <span class="mi">1000</span><span class="p">)</span>
<span class="c"># datetime is a subclass of date, we should register DatetimeConverter first</span>
<span class="n">register_input_converter</span><span class="p">(</span><span class="n">DatetimeConverter</span><span class="p">())</span>
<span class="n">register_input_converter</span><span class="p">(</span><span class="n">DateConverter</span><span class="p">())</span>
<span class="k">def</span> <span class="nf">_test</span><span class="p">():</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="kn">from</span> <span class="nn">pyspark.context</span> <span class="kn">import</span> <span class="n">SparkContext</span>
<span class="kn">from</span> <span class="nn">pyspark.sql</span> <span class="kn">import</span> <span class="n">SQLContext</span>
<span class="n">globs</span> <span class="o">=</span> <span class="nb">globals</span><span class="p">()</span>
<span class="n">sc</span> <span class="o">=</span> <span class="n">SparkContext</span><span class="p">(</span><span class="s">'local[4]'</span><span class="p">,</span> <span class="s">'PythonTest'</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'sc'</span><span class="p">]</span> <span class="o">=</span> <span class="n">sc</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'sqlContext'</span><span class="p">]</span> <span class="o">=</span> <span class="n">SQLContext</span><span class="p">(</span><span class="n">sc</span><span class="p">)</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">(</span><span class="n">globs</span><span class="o">=</span><span class="n">globs</span><span class="p">,</span> <span class="n">optionflags</span><span class="o">=</span><span class="n">doctest</span><span class="o">.</span><span class="n">ELLIPSIS</span><span class="p">)</span>
<span class="n">globs</span><span class="p">[</span><span class="s">'sc'</span><span class="p">]</span><span class="o">.</span><span class="n">stop</span><span class="p">()</span>
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="nb">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
</div>
</div>
</div>
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/spark-logo-hd.png" alt="Logo"/>
</a></p>
<div id="searchbox" style="display: none" role="search">
<h3>Quick search</h3>
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" />
<input type="submit" value="Go" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
<p class="searchtip" style="font-size: 90%">
Enter search terms or a module, class or function name.
</p>
</div>
<script type="text/javascript">$('#searchbox').show(0);</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="related" role="navigation" aria-label="related navigation">
<h3>Navigation</h3>
<ul>
<li class="nav-item nav-item-0"><a href="../../../index.html">PySpark master documentation</a> »</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
</ul>
</div>
<div class="footer" role="contentinfo">
© Copyright .
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.3.1.
</div>
</body>
</html>