<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>pyspark.mllib.linalg — PySpark master documentation</title>
<link rel="stylesheet" href="../../../_static/nature.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../../../',
VERSION: 'master',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
<script type="text/javascript" src="../../../_static/underscore.js"></script>
<script type="text/javascript" src="../../../_static/doctools.js"></script>
<link rel="top" title="PySpark master documentation" href="../../../index.html" />
<link rel="up" title="Module code" href="../../index.html" />
</head>
<body role="document">
<div class="related" role="navigation" aria-label="related navigation">
<h3>Navigation</h3>
<ul>
<li class="nav-item nav-item-0"><a href="../../../index.html">PySpark master documentation</a> »</li>
<li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li>
</ul>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<h1>Source code for pyspark.mllib.linalg</h1><div class="highlight"><pre>
<span class="c">#</span>
<span class="c"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
<span class="c"># contributor license agreements. See the NOTICE file distributed with</span>
<span class="c"># this work for additional information regarding copyright ownership.</span>
<span class="c"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
<span class="c"># (the "License"); you may not use this file except in compliance with</span>
<span class="c"># the License. You may obtain a copy of the License at</span>
<span class="c">#</span>
<span class="c"># http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c">#</span>
<span class="c"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c"># distributed under the License is distributed on an "AS IS" BASIS,</span>
<span class="c"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c"># See the License for the specific language governing permissions and</span>
<span class="c"># limitations under the License.</span>
<span class="c">#</span>
<span class="sd">"""</span>
<span class="sd">MLlib utilities for linear algebra. For dense vectors, MLlib</span>
<span class="sd">uses the NumPy C{array} type, so you can simply pass NumPy arrays</span>
<span class="sd">around. For sparse vectors, users can construct a L{SparseVector}</span>
<span class="sd">object from MLlib or pass SciPy C{scipy.sparse} column vectors if</span>
<span class="sd">SciPy is available in their environment.</span>
<span class="sd">"""</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">array</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version</span> <span class="o">>=</span> <span class="s">'3'</span><span class="p">:</span>
<span class="nb">basestring</span> <span class="o">=</span> <span class="nb">str</span>
<span class="nb">xrange</span> <span class="o">=</span> <span class="nb">range</span>
<span class="kn">import</span> <span class="nn">copyreg</span> <span class="kn">as</span> <span class="nn">copy_reg</span>
<span class="nb">long</span> <span class="o">=</span> <span class="nb">int</span>
<span class="k">else</span><span class="p">:</span>
<span class="kn">from</span> <span class="nn">itertools</span> <span class="kn">import</span> <span class="n">izip</span> <span class="k">as</span> <span class="nb">zip</span>
<span class="kn">import</span> <span class="nn">copy_reg</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">pyspark.sql.types</span> <span class="kn">import</span> <span class="n">UserDefinedType</span><span class="p">,</span> <span class="n">StructField</span><span class="p">,</span> <span class="n">StructType</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">,</span> <span class="n">DoubleType</span><span class="p">,</span> \
<span class="n">IntegerType</span><span class="p">,</span> <span class="n">ByteType</span><span class="p">,</span> <span class="n">BooleanType</span>
<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s">'Vector'</span><span class="p">,</span> <span class="s">'DenseVector'</span><span class="p">,</span> <span class="s">'SparseVector'</span><span class="p">,</span> <span class="s">'Vectors'</span><span class="p">,</span>
<span class="s">'Matrix'</span><span class="p">,</span> <span class="s">'DenseMatrix'</span><span class="p">,</span> <span class="s">'SparseMatrix'</span><span class="p">,</span> <span class="s">'Matrices'</span><span class="p">]</span>
<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">7</span><span class="p">):</span>
<span class="c"># speed up pickling array in Python 2.7</span>
<span class="k">def</span> <span class="nf">fast_pickle_array</span><span class="p">(</span><span class="n">ar</span><span class="p">):</span>
<span class="k">return</span> <span class="n">array</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="p">(</span><span class="n">ar</span><span class="o">.</span><span class="n">typecode</span><span class="p">,</span> <span class="n">ar</span><span class="o">.</span><span class="n">tostring</span><span class="p">())</span>
<span class="n">copy_reg</span><span class="o">.</span><span class="n">pickle</span><span class="p">(</span><span class="n">array</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">fast_pickle_array</span><span class="p">)</span>
<span class="c"># Check whether we have SciPy. MLlib works without it too, but if we have it, some methods,</span>
<span class="c"># such as _dot and _serialize_double_vector, start to support scipy.sparse matrices.</span>
<span class="k">try</span><span class="p">:</span>
<span class="kn">import</span> <span class="nn">scipy.sparse</span>
<span class="n">_have_scipy</span> <span class="o">=</span> <span class="bp">True</span>
<span class="k">except</span><span class="p">:</span>
<span class="c"># No SciPy in environment, but that's okay</span>
<span class="n">_have_scipy</span> <span class="o">=</span> <span class="bp">False</span>
<span class="k">def</span> <span class="nf">_convert_to_vector</span><span class="p">(</span><span class="n">l</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">l</span><span class="p">,</span> <span class="n">Vector</span><span class="p">):</span>
<span class="k">return</span> <span class="n">l</span>
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">l</span><span class="p">)</span> <span class="ow">in</span> <span class="p">(</span><span class="n">array</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">,</span> <span class="nb">xrange</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">_have_scipy</span> <span class="ow">and</span> <span class="n">scipy</span><span class="o">.</span><span class="n">sparse</span><span class="o">.</span><span class="n">issparse</span><span class="p">(</span><span class="n">l</span><span class="p">):</span>
<span class="k">assert</span> <span class="n">l</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="mi">1</span><span class="p">,</span> <span class="s">"Expected column vector"</span>
<span class="n">csc</span> <span class="o">=</span> <span class="n">l</span><span class="o">.</span><span class="n">tocsc</span><span class="p">()</span>
<span class="k">return</span> <span class="n">SparseVector</span><span class="p">(</span><span class="n">l</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">csc</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">csc</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"Cannot convert type </span><span class="si">%s</span><span class="s"> into Vector"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">l</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">_vector_size</span><span class="p">(</span><span class="n">v</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns the size of the vector.</span>
<span class="sd"> >>> _vector_size([1., 2., 3.])</span>
<span class="sd"> 3</span>
<span class="sd"> >>> _vector_size((1., 2., 3.))</span>
<span class="sd"> 3</span>
<span class="sd"> >>> _vector_size(array.array('d', [1., 2., 3.]))</span>
<span class="sd"> 3</span>
<span class="sd"> >>> _vector_size(np.zeros(3))</span>
<span class="sd"> 3</span>
<span class="sd"> >>> _vector_size(np.zeros((3, 1)))</span>
<span class="sd"> 3</span>
<span class="sd"> >>> _vector_size(np.zeros((1, 3)))</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> ValueError: Cannot treat an ndarray of shape (1, 3) as a vector</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">Vector</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="ow">in</span> <span class="p">(</span><span class="n">array</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">,</span> <span class="nb">xrange</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="k">if</span> <span class="n">v</span><span class="o">.</span><span class="n">ndim</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">or</span> <span class="p">(</span><span class="n">v</span><span class="o">.</span><span class="n">ndim</span> <span class="o">==</span> <span class="mi">2</span> <span class="ow">and</span> <span class="n">v</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="mi">1</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Cannot treat an ndarray of shape </span><span class="si">%s</span><span class="s"> as a vector"</span> <span class="o">%</span> <span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="o">.</span><span class="n">shape</span><span class="p">))</span>
<span class="k">elif</span> <span class="n">_have_scipy</span> <span class="ow">and</span> <span class="n">scipy</span><span class="o">.</span><span class="n">sparse</span><span class="o">.</span><span class="n">issparse</span><span class="p">(</span><span class="n">v</span><span class="p">):</span>
<span class="k">assert</span> <span class="n">v</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="mi">1</span><span class="p">,</span> <span class="s">"Expected column vector"</span>
<span class="k">return</span> <span class="n">v</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"Cannot treat type </span><span class="si">%s</span><span class="s"> as a vector"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">v</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">_format_float</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">digits</span><span class="o">=</span><span class="mi">4</span><span class="p">):</span>
<span class="n">s</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="nb">round</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">digits</span><span class="p">))</span>
<span class="k">if</span> <span class="s">'.'</span> <span class="ow">in</span> <span class="n">s</span><span class="p">:</span>
<span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="p">[:</span><span class="n">s</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="s">'.'</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span> <span class="o">+</span> <span class="n">digits</span><span class="p">]</span>
<span class="k">return</span> <span class="n">s</span>
<span class="k">def</span> <span class="nf">_format_float_list</span><span class="p">(</span><span class="n">l</span><span class="p">):</span>
<span class="k">return</span> <span class="p">[</span><span class="n">_format_float</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">l</span><span class="p">]</span>
<span class="k">class</span> <span class="nc">VectorUDT</span><span class="p">(</span><span class="n">UserDefinedType</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> SQL user-defined type (UDT) for Vector.</span>
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">([</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"type"</span><span class="p">,</span> <span class="n">ByteType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"size"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="bp">True</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"indices"</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">IntegerType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span> <span class="bp">True</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"values"</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">DoubleType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span> <span class="bp">True</span><span class="p">)])</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"pyspark.mllib.linalg"</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"org.apache.spark.mllib.linalg.VectorUDT"</span>
<span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">SparseVector</span><span class="p">):</span>
<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">indices</span><span class="p">]</span>
<span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="nb">float</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">values</span><span class="p">]</span>
<span class="k">return</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="n">size</span><span class="p">,</span> <span class="n">indices</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">DenseVector</span><span class="p">):</span>
<span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="nb">float</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">]</span>
<span class="k">return</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">None</span><span class="p">,</span> <span class="bp">None</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"cannot serialize </span><span class="si">%r</span><span class="s"> of type </span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">)))</span>
<span class="k">def</span> <span class="nf">deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">datum</span><span class="p">):</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">datum</span><span class="p">)</span> <span class="o">==</span> <span class="mi">4</span><span class="p">,</span> \
<span class="s">"VectorUDT.deserialize given row with length </span><span class="si">%d</span><span class="s"> but requires 4"</span> <span class="o">%</span> <span class="nb">len</span><span class="p">(</span><span class="n">datum</span><span class="p">)</span>
<span class="n">tpe</span> <span class="o">=</span> <span class="n">datum</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">tpe</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="n">SparseVector</span><span class="p">(</span><span class="n">datum</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">datum</span><span class="p">[</span><span class="mi">2</span><span class="p">],</span> <span class="n">datum</span><span class="p">[</span><span class="mi">3</span><span class="p">])</span>
<span class="k">elif</span> <span class="n">tpe</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="n">datum</span><span class="p">[</span><span class="mi">3</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"do not recognize type </span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="n">tpe</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"vector"</span>
<span class="k">class</span> <span class="nc">MatrixUDT</span><span class="p">(</span><span class="n">UserDefinedType</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> SQL user-defined type (UDT) for Matrix.</span>
<span class="sd"> """</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">sqlType</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="n">StructType</span><span class="p">([</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"type"</span><span class="p">,</span> <span class="n">ByteType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"numRows"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"numCols"</span><span class="p">,</span> <span class="n">IntegerType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"colPtrs"</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">IntegerType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span> <span class="bp">True</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"rowIndices"</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">IntegerType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span> <span class="bp">True</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"values"</span><span class="p">,</span> <span class="n">ArrayType</span><span class="p">(</span><span class="n">DoubleType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">),</span> <span class="bp">True</span><span class="p">),</span>
<span class="n">StructField</span><span class="p">(</span><span class="s">"isTransposed"</span><span class="p">,</span> <span class="n">BooleanType</span><span class="p">(),</span> <span class="bp">False</span><span class="p">)])</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">module</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"pyspark.mllib.linalg"</span>
<span class="nd">@classmethod</span>
<span class="k">def</span> <span class="nf">scalaUDT</span><span class="p">(</span><span class="n">cls</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"org.apache.spark.mllib.linalg.MatrixUDT"</span>
<span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">SparseMatrix</span><span class="p">):</span>
<span class="n">colPtrs</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">colPtrs</span><span class="p">]</span>
<span class="n">rowIndices</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">rowIndices</span><span class="p">]</span>
<span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="nb">float</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">values</span><span class="p">]</span>
<span class="k">return</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="n">numCols</span><span class="p">,</span> <span class="n">colPtrs</span><span class="p">,</span>
<span class="n">rowIndices</span><span class="p">,</span> <span class="n">values</span><span class="p">,</span> <span class="nb">bool</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">))</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">DenseMatrix</span><span class="p">):</span>
<span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="nb">float</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">values</span><span class="p">]</span>
<span class="k">return</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="n">numCols</span><span class="p">,</span> <span class="bp">None</span><span class="p">,</span> <span class="bp">None</span><span class="p">,</span> <span class="n">values</span><span class="p">,</span>
<span class="nb">bool</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"cannot serialize type </span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">)))</span>
<span class="k">def</span> <span class="nf">deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">datum</span><span class="p">):</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">datum</span><span class="p">)</span> <span class="o">==</span> <span class="mi">7</span><span class="p">,</span> \
<span class="s">"MatrixUDT.deserialize given row with length </span><span class="si">%d</span><span class="s"> but requires 7"</span> <span class="o">%</span> <span class="nb">len</span><span class="p">(</span><span class="n">datum</span><span class="p">)</span>
<span class="n">tpe</span> <span class="o">=</span> <span class="n">datum</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="n">tpe</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="n">SparseMatrix</span><span class="p">(</span><span class="o">*</span><span class="n">datum</span><span class="p">[</span><span class="mi">1</span><span class="p">:])</span>
<span class="k">elif</span> <span class="n">tpe</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">DenseMatrix</span><span class="p">(</span><span class="n">datum</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">datum</span><span class="p">[</span><span class="mi">2</span><span class="p">],</span> <span class="n">datum</span><span class="p">[</span><span class="mi">5</span><span class="p">],</span> <span class="n">datum</span><span class="p">[</span><span class="mi">6</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"do not recognize type </span><span class="si">%r</span><span class="s">"</span> <span class="o">%</span> <span class="n">tpe</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">simpleString</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"matrix"</span>
<div class="viewcode-block" id="Vector"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vector">[docs]</a><span class="k">class</span> <span class="nc">Vector</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="n">__UDT__</span> <span class="o">=</span> <span class="n">VectorUDT</span><span class="p">()</span>
<span class="sd">"""</span>
<span class="sd"> Abstract class for DenseVector and SparseVector</span>
<span class="sd"> """</span>
<div class="viewcode-block" id="Vector.toArray"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vector.toArray">[docs]</a> <span class="k">def</span> <span class="nf">toArray</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Convert the vector into an numpy.ndarray</span>
<span class="sd"> :return: numpy.ndarray</span>
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span>
</div></div>
<div class="viewcode-block" id="DenseVector"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseVector">[docs]</a><span class="k">class</span> <span class="nc">DenseVector</span><span class="p">(</span><span class="n">Vector</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> A dense vector represented by a value array. We use numpy array for</span>
<span class="sd"> storage and arithmetics will be delegated to the underlying numpy</span>
<span class="sd"> array.</span>
<span class="sd"> >>> v = Vectors.dense([1.0, 2.0])</span>
<span class="sd"> >>> u = Vectors.dense([3.0, 4.0])</span>
<span class="sd"> >>> v + u</span>
<span class="sd"> DenseVector([4.0, 6.0])</span>
<span class="sd"> >>> 2 - v</span>
<span class="sd"> DenseVector([1.0, 0.0])</span>
<span class="sd"> >>> v / 2</span>
<span class="sd"> DenseVector([0.5, 1.0])</span>
<span class="sd"> >>> v * u</span>
<span class="sd"> DenseVector([3.0, 8.0])</span>
<span class="sd"> >>> u / v</span>
<span class="sd"> DenseVector([3.0, 2.0])</span>
<span class="sd"> >>> u % 2</span>
<span class="sd"> DenseVector([1.0, 0.0])</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ar</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ar</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
<span class="n">ar</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">frombuffer</span><span class="p">(</span><span class="n">ar</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ar</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">):</span>
<span class="n">ar</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">ar</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">if</span> <span class="n">ar</span><span class="o">.</span><span class="n">dtype</span> <span class="o">!=</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">:</span>
<span class="n">ar</span> <span class="o">=</span> <span class="n">ar</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">array</span> <span class="o">=</span> <span class="n">ar</span>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="DenseVector.parse"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseVector.parse">[docs]</a> <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Parse string representation back into the DenseVector.</span>
<span class="sd"> >>> DenseVector.parse(' [ 0.0,1.0,2.0, 3.0]')</span>
<span class="sd"> DenseVector([0.0, 1.0, 2.0, 3.0])</span>
<span class="sd"> """</span>
<span class="n">start</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">'['</span><span class="p">)</span>
<span class="k">if</span> <span class="n">start</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Array should start with '['."</span><span class="p">)</span>
<span class="n">end</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">']'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">end</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Array should end with ']'."</span><span class="p">)</span>
<span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="p">[</span><span class="n">start</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span> <span class="n">end</span><span class="p">]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="nb">float</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">s</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">','</span><span class="p">)]</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Unable to parse values from </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="n">s</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="n">values</span><span class="p">)</span>
</div>
<span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="o">.</span><span class="n">tostring</span><span class="p">(),)</span>
<div class="viewcode-block" id="DenseVector.numNonzeros"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseVector.numNonzeros">[docs]</a> <span class="k">def</span> <span class="nf">numNonzeros</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">count_nonzero</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DenseVector.norm"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseVector.norm">[docs]</a> <span class="k">def</span> <span class="nf">norm</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">p</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Calculte the norm of a DenseVector.</span>
<span class="sd"> >>> a = DenseVector([0, -1, 2, -3])</span>
<span class="sd"> >>> a.norm(2)</span>
<span class="sd"> 3.7...</span>
<span class="sd"> >>> a.norm(1)</span>
<span class="sd"> 6.0</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">p</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DenseVector.dot"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseVector.dot">[docs]</a> <span class="k">def</span> <span class="nf">dot</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Compute the dot product of two Vectors. We support</span>
<span class="sd"> (Numpy array, list, SparseVector, or SciPy sparse)</span>
<span class="sd"> and a target NumPy array that is either 1- or 2-dimensional.</span>
<span class="sd"> Equivalent to calling numpy.dot of the two vectors.</span>
<span class="sd"> >>> dense = DenseVector(array.array('d', [1., 2.]))</span>
<span class="sd"> >>> dense.dot(dense)</span>
<span class="sd"> 5.0</span>
<span class="sd"> >>> dense.dot(SparseVector(2, [0, 1], [2., 1.]))</span>
<span class="sd"> 4.0</span>
<span class="sd"> >>> dense.dot(range(1, 3))</span>
<span class="sd"> 5.0</span>
<span class="sd"> >>> dense.dot(np.array(range(1, 3)))</span>
<span class="sd"> 5.0</span>
<span class="sd"> >>> dense.dot([1.,])</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> >>> dense.dot(np.reshape([1., 2., 3., 4.], (2, 2), order='F'))</span>
<span class="sd"> array([ 5., 11.])</span>
<span class="sd"> >>> dense.dot(np.reshape([1., 2., 3.], (3, 1), order='F'))</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> <span class="o">==</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="k">if</span> <span class="n">other</span><span class="o">.</span><span class="n">ndim</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="s">"dimension mismatch"</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">other</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">_have_scipy</span> <span class="ow">and</span> <span class="n">scipy</span><span class="o">.</span><span class="n">sparse</span><span class="o">.</span><span class="n">issparse</span><span class="p">(</span><span class="n">other</span><span class="p">):</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="s">"dimension mismatch"</span>
<span class="k">return</span> <span class="n">other</span><span class="o">.</span><span class="n">transpose</span><span class="p">()</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="n">_vector_size</span><span class="p">(</span><span class="n">other</span><span class="p">),</span> <span class="s">"dimension mismatch"</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">SparseVector</span><span class="p">):</span>
<span class="k">return</span> <span class="n">other</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">Vector</span><span class="p">):</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">(),</span> <span class="n">other</span><span class="o">.</span><span class="n">toArray</span><span class="p">())</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">(),</span> <span class="n">other</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DenseVector.squared_distance"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseVector.squared_distance">[docs]</a> <span class="k">def</span> <span class="nf">squared_distance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Squared distance of two Vectors.</span>
<span class="sd"> >>> dense1 = DenseVector(array.array('d', [1., 2.]))</span>
<span class="sd"> >>> dense1.squared_distance(dense1)</span>
<span class="sd"> 0.0</span>
<span class="sd"> >>> dense2 = np.array([2., 1.])</span>
<span class="sd"> >>> dense1.squared_distance(dense2)</span>
<span class="sd"> 2.0</span>
<span class="sd"> >>> dense3 = [2., 1.]</span>
<span class="sd"> >>> dense1.squared_distance(dense3)</span>
<span class="sd"> 2.0</span>
<span class="sd"> >>> sparse1 = SparseVector(2, [0, 1], [2., 1.])</span>
<span class="sd"> >>> dense1.squared_distance(sparse1)</span>
<span class="sd"> 2.0</span>
<span class="sd"> >>> dense1.squared_distance([1.,])</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> >>> dense1.squared_distance(SparseVector(1, [0,], [1.,]))</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> """</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="n">_vector_size</span><span class="p">(</span><span class="n">other</span><span class="p">),</span> <span class="s">"dimension mismatch"</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">SparseVector</span><span class="p">):</span>
<span class="k">return</span> <span class="n">other</span><span class="o">.</span><span class="n">squared_distance</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">_have_scipy</span> <span class="ow">and</span> <span class="n">scipy</span><span class="o">.</span><span class="n">sparse</span><span class="o">.</span><span class="n">issparse</span><span class="p">(</span><span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="n">_convert_to_vector</span><span class="p">(</span><span class="n">other</span><span class="p">)</span><span class="o">.</span><span class="n">squared_distance</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">Vector</span><span class="p">):</span>
<span class="n">other</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">toArray</span><span class="p">()</span>
<span class="k">elif</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">):</span>
<span class="n">other</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
<span class="n">diff</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">()</span> <span class="o">-</span> <span class="n">other</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">diff</span><span class="p">,</span> <span class="n">diff</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DenseVector.toArray"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseVector.toArray">[docs]</a> <span class="k">def</span> <span class="nf">toArray</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">array</span>
</div>
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">[</span><span class="n">item</span><span class="p">]</span>
<span class="k">def</span> <span class="nf">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"["</span> <span class="o">+</span> <span class="s">","</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">])</span> <span class="o">+</span> <span class="s">"]"</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="s">"DenseVector([</span><span class="si">%s</span><span class="s">])"</span> <span class="o">%</span> <span class="p">(</span><span class="s">', '</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_format_float</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">DenseVector</span><span class="p">)</span> <span class="ow">and</span> <span class="n">np</span><span class="o">.</span><span class="n">array_equal</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">other</span><span class="o">.</span><span class="n">array</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span> <span class="o">==</span> <span class="n">other</span>
<span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">item</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">_delegate</span><span class="p">(</span><span class="n">op</span><span class="p">):</span>
<span class="k">def</span> <span class="nf">func</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">DenseVector</span><span class="p">):</span>
<span class="n">other</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">array</span>
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">array</span><span class="p">,</span> <span class="n">op</span><span class="p">)(</span><span class="n">other</span><span class="p">))</span>
<span class="k">return</span> <span class="n">func</span>
<span class="n">__neg__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__neg__"</span><span class="p">)</span>
<span class="n">__add__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__add__"</span><span class="p">)</span>
<span class="n">__sub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__sub__"</span><span class="p">)</span>
<span class="n">__mul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__mul__"</span><span class="p">)</span>
<span class="n">__div__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__div__"</span><span class="p">)</span>
<span class="n">__truediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__truediv__"</span><span class="p">)</span>
<span class="n">__mod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__mod__"</span><span class="p">)</span>
<span class="n">__radd__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__radd__"</span><span class="p">)</span>
<span class="n">__rsub__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__rsub__"</span><span class="p">)</span>
<span class="n">__rmul__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__rmul__"</span><span class="p">)</span>
<span class="n">__rdiv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__rdiv__"</span><span class="p">)</span>
<span class="n">__rtruediv__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__rtruediv__"</span><span class="p">)</span>
<span class="n">__rmod__</span> <span class="o">=</span> <span class="n">_delegate</span><span class="p">(</span><span class="s">"__rmod__"</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="SparseVector"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseVector">[docs]</a><span class="k">class</span> <span class="nc">SparseVector</span><span class="p">(</span><span class="n">Vector</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> A simple sparse vector class for passing data to MLlib. Users may</span>
<span class="sd"> alternatively pass SciPy's {scipy.sparse} data types.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Create a sparse vector, using either a dictionary, a list of</span>
<span class="sd"> (index, value) pairs, or two separate arrays of indices and</span>
<span class="sd"> values (sorted by index).</span>
<span class="sd"> :param size: Size of the vector.</span>
<span class="sd"> :param args: Active entries, as a dictionary {index: value, ...},</span>
<span class="sd"> a list of tuples [(index, value), ...], or a list of strictly i</span>
<span class="sd"> ncreasing indices and a list of corresponding values [index, ...],</span>
<span class="sd"> [value, ...]. Inactive entries are treated as zeros.</span>
<span class="sd"> >>> SparseVector(4, {1: 1.0, 3: 5.5})</span>
<span class="sd"> SparseVector(4, {1: 1.0, 3: 5.5})</span>
<span class="sd"> >>> SparseVector(4, [(1, 1.0), (3, 5.5)])</span>
<span class="sd"> SparseVector(4, {1: 1.0, 3: 5.5})</span>
<span class="sd"> >>> SparseVector(4, [1, 3], [1.0, 5.5])</span>
<span class="sd"> SparseVector(4, {1: 1.0, 3: 5.5})</span>
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">size</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">size</span><span class="p">)</span>
<span class="sd">""" Size of the vector. """</span>
<span class="k">assert</span> <span class="mi">1</span> <span class="o"><=</span> <span class="nb">len</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o"><=</span> <span class="mi">2</span><span class="p">,</span> <span class="s">"must pass either 2 or 3 arguments"</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">args</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="n">pairs</span> <span class="o">=</span> <span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">pairs</span><span class="p">)</span> <span class="o">==</span> <span class="nb">dict</span><span class="p">:</span>
<span class="n">pairs</span> <span class="o">=</span> <span class="n">pairs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
<span class="n">pairs</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">pairs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">p</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">pairs</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="sd">""" A list of indices corresponding to active entries. """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">p</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">pairs</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="sd">""" A list of values corresponding to active entries. """</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">bytes</span><span class="p">):</span>
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="nb">bytes</span><span class="p">),</span> <span class="s">"values should be string too"</span>
<span class="k">if</span> <span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">frombuffer</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">frombuffer</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c"># np.frombuffer() doesn't work well with empty string in older version</span>
<span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">),</span> <span class="s">"index and value arrays not same length"</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">xrange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s">"indices array must be sorted"</span><span class="p">)</span>
<div class="viewcode-block" id="SparseVector.numNonzeros"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseVector.numNonzeros">[docs]</a> <span class="k">def</span> <span class="nf">numNonzeros</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">count_nonzero</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="SparseVector.norm"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseVector.norm">[docs]</a> <span class="k">def</span> <span class="nf">norm</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">p</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Calculte the norm of a SparseVector.</span>
<span class="sd"> >>> a = SparseVector(4, [0, 1], [3., -4.])</span>
<span class="sd"> >>> a.norm(1)</span>
<span class="sd"> 7.0</span>
<span class="sd"> >>> a.norm(2)</span>
<span class="sd"> 5.0</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">,</span> <span class="n">p</span><span class="p">)</span>
</div>
<span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span>
<span class="n">SparseVector</span><span class="p">,</span>
<span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="o">.</span><span class="n">tostring</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tostring</span><span class="p">()))</span>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="SparseVector.parse"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseVector.parse">[docs]</a> <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Parse string representation back into the DenseVector.</span>
<span class="sd"> >>> SparseVector.parse(' (4, [0,1 ],[ 4.0,5.0] )')</span>
<span class="sd"> SparseVector(4, {0: 4.0, 1: 5.0})</span>
<span class="sd"> """</span>
<span class="n">start</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">'('</span><span class="p">)</span>
<span class="k">if</span> <span class="n">start</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Tuple should start with '('"</span><span class="p">)</span>
<span class="n">end</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">')'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">start</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Tuple should end with ')'"</span><span class="p">)</span>
<span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="p">[</span><span class="n">start</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span> <span class="n">end</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">size</span> <span class="o">=</span> <span class="n">s</span><span class="p">[:</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">','</span><span class="p">)]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">size</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">size</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Cannot parse size </span><span class="si">%s</span><span class="s">."</span> <span class="o">%</span> <span class="n">size</span><span class="p">)</span>
<span class="n">ind_start</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">'['</span><span class="p">)</span>
<span class="k">if</span> <span class="n">ind_start</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Indices array should start with '['."</span><span class="p">)</span>
<span class="n">ind_end</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">']'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">ind_end</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Indices array should end with ']'"</span><span class="p">)</span>
<span class="n">new_s</span> <span class="o">=</span> <span class="n">s</span><span class="p">[</span><span class="n">ind_start</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span> <span class="n">ind_end</span><span class="p">]</span>
<span class="n">ind_list</span> <span class="o">=</span> <span class="n">new_s</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">','</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">indices</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">ind</span><span class="p">)</span> <span class="k">for</span> <span class="n">ind</span> <span class="ow">in</span> <span class="n">ind_list</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Unable to parse indices from </span><span class="si">%s</span><span class="s">."</span> <span class="o">%</span> <span class="n">new_s</span><span class="p">)</span>
<span class="n">s</span> <span class="o">=</span> <span class="n">s</span><span class="p">[</span><span class="n">ind_end</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">val_start</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">'['</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val_start</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Values array should start with '['."</span><span class="p">)</span>
<span class="n">val_end</span> <span class="o">=</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">']'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">val_end</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Values array should end with ']'."</span><span class="p">)</span>
<span class="n">val_list</span> <span class="o">=</span> <span class="n">s</span><span class="p">[</span><span class="n">val_start</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span> <span class="n">val_end</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s">','</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">values</span> <span class="o">=</span> <span class="p">[</span><span class="nb">float</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="k">for</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">val_list</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Unable to parse values from </span><span class="si">%s</span><span class="s">."</span> <span class="o">%</span> <span class="n">s</span><span class="p">)</span>
<span class="k">return</span> <span class="n">SparseVector</span><span class="p">(</span><span class="n">size</span><span class="p">,</span> <span class="n">indices</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="SparseVector.dot"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseVector.dot">[docs]</a> <span class="k">def</span> <span class="nf">dot</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Dot product with a SparseVector or 1- or 2-dimensional Numpy array.</span>
<span class="sd"> >>> a = SparseVector(4, [1, 3], [3.0, 4.0])</span>
<span class="sd"> >>> a.dot(a)</span>
<span class="sd"> 25.0</span>
<span class="sd"> >>> a.dot(array.array('d', [1., 2., 3., 4.]))</span>
<span class="sd"> 22.0</span>
<span class="sd"> >>> b = SparseVector(4, [2], [1.0])</span>
<span class="sd"> >>> a.dot(b)</span>
<span class="sd"> 0.0</span>
<span class="sd"> >>> a.dot(np.array([[1, 1], [2, 2], [3, 3], [4, 4]]))</span>
<span class="sd"> array([ 22., 22.])</span>
<span class="sd"> >>> a.dot([1., 2., 3.])</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> >>> a.dot(np.array([1., 2.]))</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> >>> a.dot(DenseVector([1., 2.]))</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> >>> a.dot(np.zeros((3, 2)))</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">):</span>
<span class="k">if</span> <span class="n">other</span><span class="o">.</span><span class="n">ndim</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">]:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Cannot call dot with </span><span class="si">%d</span><span class="s">-dimensional array"</span> <span class="o">%</span> <span class="n">other</span><span class="o">.</span><span class="n">ndim</span><span class="p">)</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="s">"dimension mismatch"</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">,</span> <span class="n">other</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">])</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="n">_vector_size</span><span class="p">(</span><span class="n">other</span><span class="p">),</span> <span class="s">"dimension mismatch"</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">DenseVector</span><span class="p">):</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">array</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">SparseVector</span><span class="p">):</span>
<span class="c"># Find out common indices.</span>
<span class="n">self_cmind</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">in1d</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">other</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">assume_unique</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
<span class="n">self_values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">self_cmind</span><span class="p">]</span>
<span class="k">if</span> <span class="n">self_values</span><span class="o">.</span><span class="n">size</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">return</span> <span class="mf">0.0</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">other_cmind</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">in1d</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="n">assume_unique</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">self_values</span><span class="p">,</span> <span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">other_cmind</span><span class="p">])</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">_convert_to_vector</span><span class="p">(</span><span class="n">other</span><span class="p">))</span>
</div>
<div class="viewcode-block" id="SparseVector.squared_distance"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseVector.squared_distance">[docs]</a> <span class="k">def</span> <span class="nf">squared_distance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Squared distance from a SparseVector or 1-dimensional NumPy array.</span>
<span class="sd"> >>> a = SparseVector(4, [1, 3], [3.0, 4.0])</span>
<span class="sd"> >>> a.squared_distance(a)</span>
<span class="sd"> 0.0</span>
<span class="sd"> >>> a.squared_distance(array.array('d', [1., 2., 3., 4.]))</span>
<span class="sd"> 11.0</span>
<span class="sd"> >>> a.squared_distance(np.array([1., 2., 3., 4.]))</span>
<span class="sd"> 11.0</span>
<span class="sd"> >>> b = SparseVector(4, [2], [1.0])</span>
<span class="sd"> >>> a.squared_distance(b)</span>
<span class="sd"> 26.0</span>
<span class="sd"> >>> b.squared_distance(a)</span>
<span class="sd"> 26.0</span>
<span class="sd"> >>> b.squared_distance([1., 2.])</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> >>> b.squared_distance(SparseVector(3, [1,], [1.0,]))</span>
<span class="sd"> Traceback (most recent call last):</span>
<span class="sd"> ...</span>
<span class="sd"> AssertionError: dimension mismatch</span>
<span class="sd"> """</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="n">_vector_size</span><span class="p">(</span><span class="n">other</span><span class="p">),</span> <span class="s">"dimension mismatch"</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">DenseVector</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="ow">and</span> <span class="n">other</span><span class="o">.</span><span class="n">ndim</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="s">"Cannot call squared_distance with </span><span class="si">%d</span><span class="s">-dimensional array"</span> <span class="o">%</span>
<span class="n">other</span><span class="o">.</span><span class="n">ndim</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">DenseVector</span><span class="p">):</span>
<span class="n">other</span> <span class="o">=</span> <span class="n">other</span><span class="o">.</span><span class="n">array</span>
<span class="n">sparse_ind</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">size</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">bool</span><span class="p">)</span>
<span class="n">sparse_ind</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">]</span> <span class="o">=</span> <span class="bp">True</span>
<span class="n">dist</span> <span class="o">=</span> <span class="n">other</span><span class="p">[</span><span class="n">sparse_ind</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">dist</span><span class="p">,</span> <span class="n">dist</span><span class="p">)</span>
<span class="n">other_ind</span> <span class="o">=</span> <span class="n">other</span><span class="p">[</span><span class="o">~</span><span class="n">sparse_ind</span><span class="p">]</span>
<span class="n">result</span> <span class="o">+=</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">other_ind</span><span class="p">,</span> <span class="n">other_ind</span><span class="p">)</span>
<span class="k">return</span> <span class="n">result</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">SparseVector</span><span class="p">):</span>
<span class="n">result</span> <span class="o">=</span> <span class="mf">0.0</span>
<span class="n">i</span><span class="p">,</span> <span class="n">j</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span>
<span class="k">while</span> <span class="n">i</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">)</span> <span class="ow">and</span> <span class="n">j</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">indices</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">j</span><span class="p">]:</span>
<span class="n">diff</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">-</span> <span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">j</span><span class="p">]</span>
<span class="n">result</span> <span class="o">+=</span> <span class="n">diff</span> <span class="o">*</span> <span class="n">diff</span>
<span class="n">i</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="n">j</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o"><</span> <span class="n">other</span><span class="o">.</span><span class="n">indices</span><span class="p">[</span><span class="n">j</span><span class="p">]:</span>
<span class="n">result</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
<span class="n">i</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">result</span> <span class="o">+=</span> <span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">j</span><span class="p">]</span> <span class="o">*</span> <span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">j</span><span class="p">]</span>
<span class="n">j</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">while</span> <span class="n">i</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">):</span>
<span class="n">result</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
<span class="n">i</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">while</span> <span class="n">j</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">indices</span><span class="p">):</span>
<span class="n">result</span> <span class="o">+=</span> <span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">j</span><span class="p">]</span> <span class="o">*</span> <span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">j</span><span class="p">]</span>
<span class="n">j</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">return</span> <span class="n">result</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">squared_distance</span><span class="p">(</span><span class="n">_convert_to_vector</span><span class="p">(</span><span class="n">other</span><span class="p">))</span>
</div>
<div class="viewcode-block" id="SparseVector.toArray"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseVector.toArray">[docs]</a> <span class="k">def</span> <span class="nf">toArray</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a copy of this SparseVector as a 1-dimensional NumPy array.</span>
<span class="sd"> """</span>
<span class="n">arr</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">,),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="n">arr</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span>
<span class="k">return</span> <span class="n">arr</span>
</div>
<span class="k">def</span> <span class="nf">__len__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span>
<span class="k">def</span> <span class="nf">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">inds</span> <span class="o">=</span> <span class="s">"["</span> <span class="o">+</span> <span class="s">","</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">])</span> <span class="o">+</span> <span class="s">"]"</span>
<span class="n">vals</span> <span class="o">=</span> <span class="s">"["</span> <span class="o">+</span> <span class="s">","</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">])</span> <span class="o">+</span> <span class="s">"]"</span>
<span class="k">return</span> <span class="s">"("</span> <span class="o">+</span> <span class="s">","</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">),</span> <span class="n">inds</span><span class="p">,</span> <span class="n">vals</span><span class="p">))</span> <span class="o">+</span> <span class="s">")"</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">inds</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span>
<span class="n">vals</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span>
<span class="n">entries</span> <span class="o">=</span> <span class="s">", "</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="s">"{0}: {1}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">inds</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">_format_float</span><span class="p">(</span><span class="n">vals</span><span class="p">[</span><span class="n">i</span><span class="p">]))</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">xrange</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">inds</span><span class="p">))])</span>
<span class="k">return</span> <span class="s">"SparseVector({0}, {{{1}}})"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">,</span> <span class="n">entries</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Test SparseVectors for equality.</span>
<span class="sd"> >>> v1 = SparseVector(4, [(1, 1.0), (3, 5.5)])</span>
<span class="sd"> >>> v2 = SparseVector(4, [(1, 1.0), (3, 5.5)])</span>
<span class="sd"> >>> v1 == v2</span>
<span class="sd"> True</span>
<span class="sd"> >>> v1 != v2</span>
<span class="sd"> False</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="p">)</span>
<span class="ow">and</span> <span class="n">other</span><span class="o">.</span><span class="n">size</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span>
<span class="ow">and</span> <span class="n">np</span><span class="o">.</span><span class="n">array_equal</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">indices</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span><span class="p">)</span>
<span class="ow">and</span> <span class="n">np</span><span class="o">.</span><span class="n">array_equal</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">values</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">index</span><span class="p">):</span>
<span class="n">inds</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">indices</span>
<span class="n">vals</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
<span class="s">"Indices must be of type integer, got type </span><span class="si">%s</span><span class="s">"</span> <span class="o">%</span> <span class="nb">type</span><span class="p">(</span><span class="n">index</span><span class="p">))</span>
<span class="k">if</span> <span class="n">index</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">index</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span>
<span class="k">if</span> <span class="n">index</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span> <span class="ow">or</span> <span class="n">index</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Index </span><span class="si">%d</span><span class="s"> out of bounds."</span> <span class="o">%</span> <span class="n">index</span><span class="p">)</span>
<span class="n">insert_index</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">searchsorted</span><span class="p">(</span><span class="n">inds</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
<span class="n">row_ind</span> <span class="o">=</span> <span class="n">inds</span><span class="p">[</span><span class="n">insert_index</span><span class="p">]</span>
<span class="k">if</span> <span class="n">row_ind</span> <span class="o">==</span> <span class="n">index</span><span class="p">:</span>
<span class="k">return</span> <span class="n">vals</span><span class="p">[</span><span class="n">insert_index</span><span class="p">]</span>
<span class="k">return</span> <span class="mf">0.</span>
<span class="k">def</span> <span class="nf">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="Vectors"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vectors">[docs]</a><span class="k">class</span> <span class="nc">Vectors</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Factory methods for working with vectors. Note that dense vectors</span>
<span class="sd"> are simply represented as NumPy array objects, so there is no need</span>
<span class="sd"> to covert them for use in MLlib. For sparse vectors, the factory</span>
<span class="sd"> methods in this class create an MLlib-compatible type, or users</span>
<span class="sd"> can pass in SciPy's C{scipy.sparse} column vectors.</span>
<span class="sd"> """</span>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Vectors.sparse"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vectors.sparse">[docs]</a> <span class="k">def</span> <span class="nf">sparse</span><span class="p">(</span><span class="n">size</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Create a sparse vector, using either a dictionary, a list of</span>
<span class="sd"> (index, value) pairs, or two separate arrays of indices and</span>
<span class="sd"> values (sorted by index).</span>
<span class="sd"> :param size: Size of the vector.</span>
<span class="sd"> :param args: Non-zero entries, as a dictionary, list of tupes,</span>
<span class="sd"> or two sorted lists containing indices and values.</span>
<span class="sd"> >>> Vectors.sparse(4, {1: 1.0, 3: 5.5})</span>
<span class="sd"> SparseVector(4, {1: 1.0, 3: 5.5})</span>
<span class="sd"> >>> Vectors.sparse(4, [(1, 1.0), (3, 5.5)])</span>
<span class="sd"> SparseVector(4, {1: 1.0, 3: 5.5})</span>
<span class="sd"> >>> Vectors.sparse(4, [1, 3], [1.0, 5.5])</span>
<span class="sd"> SparseVector(4, {1: 1.0, 3: 5.5})</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">SparseVector</span><span class="p">(</span><span class="n">size</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">)</span>
</div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Vectors.dense"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vectors.dense">[docs]</a> <span class="k">def</span> <span class="nf">dense</span><span class="p">(</span><span class="o">*</span><span class="n">elements</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Create a dense vector of 64-bit floats from a Python list or numbers.</span>
<span class="sd"> >>> Vectors.dense([1, 2, 3])</span>
<span class="sd"> DenseVector([1.0, 2.0, 3.0])</span>
<span class="sd"> >>> Vectors.dense(1.0, 2.0)</span>
<span class="sd"> DenseVector([1.0, 2.0])</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">elements</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">elements</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="p">(</span><span class="nb">float</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">long</span><span class="p">)):</span>
<span class="c"># it's list, numpy.array or other iterable object.</span>
<span class="n">elements</span> <span class="o">=</span> <span class="n">elements</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="n">elements</span><span class="p">)</span>
</div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Vectors.stringify"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vectors.stringify">[docs]</a> <span class="k">def</span> <span class="nf">stringify</span><span class="p">(</span><span class="n">vector</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Converts a vector into a string, which can be recognized by</span>
<span class="sd"> Vectors.parse().</span>
<span class="sd"> >>> Vectors.stringify(Vectors.sparse(2, [1], [1.0]))</span>
<span class="sd"> '(2,[1],[1.0])'</span>
<span class="sd"> >>> Vectors.stringify(Vectors.dense([0.0, 1.0]))</span>
<span class="sd"> '[0.0,1.0]'</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">vector</span><span class="p">)</span>
</div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Vectors.squared_distance"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vectors.squared_distance">[docs]</a> <span class="k">def</span> <span class="nf">squared_distance</span><span class="p">(</span><span class="n">v1</span><span class="p">,</span> <span class="n">v2</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Squared distance between two vectors.</span>
<span class="sd"> a and b can be of type SparseVector, DenseVector, np.ndarray</span>
<span class="sd"> or array.array.</span>
<span class="sd"> >>> a = Vectors.sparse(4, [(0, 1), (3, 4)])</span>
<span class="sd"> >>> b = Vectors.dense([2, 5, 4, 1])</span>
<span class="sd"> >>> a.squared_distance(b)</span>
<span class="sd"> 51.0</span>
<span class="sd"> """</span>
<span class="n">v1</span><span class="p">,</span> <span class="n">v2</span> <span class="o">=</span> <span class="n">_convert_to_vector</span><span class="p">(</span><span class="n">v1</span><span class="p">),</span> <span class="n">_convert_to_vector</span><span class="p">(</span><span class="n">v2</span><span class="p">)</span>
<span class="k">return</span> <span class="n">v1</span><span class="o">.</span><span class="n">squared_distance</span><span class="p">(</span><span class="n">v2</span><span class="p">)</span>
</div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Vectors.norm"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vectors.norm">[docs]</a> <span class="k">def</span> <span class="nf">norm</span><span class="p">(</span><span class="n">vector</span><span class="p">,</span> <span class="n">p</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Find norm of the given vector.</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">_convert_to_vector</span><span class="p">(</span><span class="n">vector</span><span class="p">)</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="n">p</span><span class="p">)</span>
</div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Vectors.parse"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vectors.parse">[docs]</a> <span class="k">def</span> <span class="nf">parse</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
<span class="sd">"""Parse a string representation back into the Vector.</span>
<span class="sd"> >>> Vectors.parse('[2,1,2 ]')</span>
<span class="sd"> DenseVector([2.0, 1.0, 2.0])</span>
<span class="sd"> >>> Vectors.parse(' ( 100, [0], [2])')</span>
<span class="sd"> SparseVector(100, {0: 2.0})</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">'('</span><span class="p">)</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span> <span class="ow">and</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">'['</span><span class="p">)</span> <span class="o">!=</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">DenseVector</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">s</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s">'('</span><span class="p">)</span> <span class="o">!=</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
<span class="k">return</span> <span class="n">SparseVector</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
<span class="s">"Cannot find tokens '[' or '(' from the input string."</span><span class="p">)</span>
</div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Vectors.zeros"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Vectors.zeros">[docs]</a> <span class="k">def</span> <span class="nf">zeros</span><span class="p">(</span><span class="n">size</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DenseVector</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">size</span><span class="p">))</span>
</div></div>
<div class="viewcode-block" id="Matrix"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Matrix">[docs]</a><span class="k">class</span> <span class="nc">Matrix</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="n">__UDT__</span> <span class="o">=</span> <span class="n">MatrixUDT</span><span class="p">()</span>
<span class="sd">"""</span>
<span class="sd"> Represents a local matrix.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">numRows</span> <span class="o">=</span> <span class="n">numRows</span>
<span class="bp">self</span><span class="o">.</span><span class="n">numCols</span> <span class="o">=</span> <span class="n">numCols</span>
<span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span> <span class="o">=</span> <span class="n">isTransposed</span>
<div class="viewcode-block" id="Matrix.toArray"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Matrix.toArray">[docs]</a> <span class="k">def</span> <span class="nf">toArray</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns its elements in a NumPy ndarray.</span>
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span>
</div>
<span class="nd">@staticmethod</span>
<span class="k">def</span> <span class="nf">_convert_to_array</span><span class="p">(</span><span class="n">array_like</span><span class="p">,</span> <span class="n">dtype</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Convert Matrix attributes which are array-like or buffer to array.</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">array_like</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">frombuffer</span><span class="p">(</span><span class="n">array_like</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">dtype</span><span class="p">)</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">array_like</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">dtype</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DenseMatrix"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseMatrix">[docs]</a><span class="k">class</span> <span class="nc">DenseMatrix</span><span class="p">(</span><span class="n">Matrix</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Column-major dense matrix.</span>
<span class="sd"> """</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">values</span><span class="p">,</span> <span class="n">isTransposed</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="n">Matrix</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
<span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">values</span><span class="p">)</span> <span class="o">==</span> <span class="n">numRows</span> <span class="o">*</span> <span class="n">numCols</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="n">values</span>
<span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">DenseMatrix</span><span class="p">,</span> <span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tostring</span><span class="p">(),</span>
<span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Pretty printing of a DenseMatrix</span>
<span class="sd"> >>> dm = DenseMatrix(2, 2, range(4))</span>
<span class="sd"> >>> print(dm)</span>
<span class="sd"> DenseMatrix([[ 0., 2.],</span>
<span class="sd"> [ 1., 3.]])</span>
<span class="sd"> >>> dm = DenseMatrix(2, 2, range(4), isTransposed=True)</span>
<span class="sd"> >>> print(dm)</span>
<span class="sd"> DenseMatrix([[ 0., 1.],</span>
<span class="sd"> [ 2., 3.]])</span>
<span class="sd"> """</span>
<span class="c"># Inspired by __repr__ in scipy matrices.</span>
<span class="n">array_lines</span> <span class="o">=</span> <span class="nb">repr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">())</span><span class="o">.</span><span class="n">splitlines</span><span class="p">()</span>
<span class="c"># We need to adjust six spaces which is the difference in number</span>
<span class="c"># of letters between "DenseMatrix" and "array"</span>
<span class="n">x</span> <span class="o">=</span> <span class="s">'</span><span class="se">\n</span><span class="s">'</span><span class="o">.</span><span class="n">join</span><span class="p">([(</span><span class="s">" "</span> <span class="o">*</span> <span class="mi">6</span> <span class="o">+</span> <span class="n">line</span><span class="p">)</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">array_lines</span><span class="p">[</span><span class="mi">1</span><span class="p">:]])</span>
<span class="k">return</span> <span class="n">array_lines</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s">"array"</span><span class="p">,</span> <span class="s">"DenseMatrix"</span><span class="p">)</span> <span class="o">+</span> <span class="s">"</span><span class="se">\n</span><span class="s">"</span> <span class="o">+</span> <span class="n">x</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Representation of a DenseMatrix</span>
<span class="sd"> >>> dm = DenseMatrix(2, 2, range(4))</span>
<span class="sd"> >>> dm</span>
<span class="sd"> DenseMatrix(2, 2, [0.0, 1.0, 2.0, 3.0], False)</span>
<span class="sd"> """</span>
<span class="c"># If the number of values are less than seventeen then return as it is.</span>
<span class="c"># Else return first eight values and last eight values.</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> <span class="o"><</span> <span class="mi">17</span><span class="p">:</span>
<span class="n">entries</span> <span class="o">=</span> <span class="n">_format_float_list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">entries</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">_format_float_list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[:</span><span class="mi">8</span><span class="p">])</span> <span class="o">+</span>
<span class="p">[</span><span class="s">"..."</span><span class="p">]</span> <span class="o">+</span>
<span class="n">_format_float_list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="o">-</span><span class="mi">8</span><span class="p">:])</span>
<span class="p">)</span>
<span class="n">entries</span> <span class="o">=</span> <span class="s">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">entries</span><span class="p">)</span>
<span class="k">return</span> <span class="s">"DenseMatrix({0}, {1}, [{2}], {3})"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">,</span> <span class="n">entries</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">)</span>
<div class="viewcode-block" id="DenseMatrix.toArray"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseMatrix.toArray">[docs]</a> <span class="k">def</span> <span class="nf">toArray</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Return an numpy.ndarray</span>
<span class="sd"> >>> m = DenseMatrix(2, 2, range(4))</span>
<span class="sd"> >>> m.toArray()</span>
<span class="sd"> array([[ 0., 2.],</span>
<span class="sd"> [ 1., 3.]])</span>
<span class="sd"> """</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">:</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">asfortranarray</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">)))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">),</span> <span class="n">order</span><span class="o">=</span><span class="s">'F'</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="DenseMatrix.toSparse"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.DenseMatrix.toSparse">[docs]</a> <span class="k">def</span> <span class="nf">toSparse</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""Convert to SparseMatrix"""</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">:</span>
<span class="n">values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">(),</span> <span class="n">order</span><span class="o">=</span><span class="s">'F'</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span>
<span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">nonzero</span><span class="p">(</span><span class="n">values</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="n">colCounts</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">bincount</span><span class="p">(</span><span class="n">indices</span> <span class="o">//</span> <span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">)</span>
<span class="n">colPtrs</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">cumsum</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">hstack</span><span class="p">(</span>
<span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">colCounts</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">numCols</span> <span class="o">-</span> <span class="n">colCounts</span><span class="o">.</span><span class="n">size</span><span class="p">))))</span>
<span class="n">values</span> <span class="o">=</span> <span class="n">values</span><span class="p">[</span><span class="n">indices</span><span class="p">]</span>
<span class="n">rowIndices</span> <span class="o">=</span> <span class="n">indices</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">numRows</span>
<span class="k">return</span> <span class="n">SparseMatrix</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">,</span> <span class="n">colPtrs</span><span class="p">,</span> <span class="n">rowIndices</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span>
</div>
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">):</span>
<span class="n">i</span><span class="p">,</span> <span class="n">j</span> <span class="o">=</span> <span class="n">indices</span>
<span class="k">if</span> <span class="n">i</span> <span class="o"><</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">i</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Row index </span><span class="si">%d</span><span class="s"> is out of range [0, </span><span class="si">%d</span><span class="s">)"</span>
<span class="o">%</span> <span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">))</span>
<span class="k">if</span> <span class="n">j</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span> <span class="ow">or</span> <span class="n">j</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Column index </span><span class="si">%d</span><span class="s"> is out of range [0, </span><span class="si">%d</span><span class="s">)"</span>
<span class="o">%</span> <span class="p">(</span><span class="n">j</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">))</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">i</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span> <span class="o">+</span> <span class="n">j</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">i</span> <span class="o">+</span> <span class="n">j</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">]</span>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">DenseMatrix</span><span class="p">)</span> <span class="ow">or</span>
<span class="bp">self</span><span class="o">.</span><span class="n">numRows</span> <span class="o">!=</span> <span class="n">other</span><span class="o">.</span><span class="n">numRows</span> <span class="ow">or</span>
<span class="bp">self</span><span class="o">.</span><span class="n">numCols</span> <span class="o">!=</span> <span class="n">other</span><span class="o">.</span><span class="n">numCols</span><span class="p">):</span>
<span class="k">return</span> <span class="bp">False</span>
<span class="n">self_values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">(),</span> <span class="n">order</span><span class="o">=</span><span class="s">'F'</span><span class="p">)</span>
<span class="n">other_values</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">toArray</span><span class="p">(),</span> <span class="n">order</span><span class="o">=</span><span class="s">'F'</span><span class="p">)</span>
<span class="k">return</span> <span class="nb">all</span><span class="p">(</span><span class="n">self_values</span> <span class="o">==</span> <span class="n">other_values</span><span class="p">)</span>
</div>
<div class="viewcode-block" id="SparseMatrix"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseMatrix">[docs]</a><span class="k">class</span> <span class="nc">SparseMatrix</span><span class="p">(</span><span class="n">Matrix</span><span class="p">):</span>
<span class="sd">"""Sparse Matrix stored in CSC format."""</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">colPtrs</span><span class="p">,</span> <span class="n">rowIndices</span><span class="p">,</span> <span class="n">values</span><span class="p">,</span>
<span class="n">isTransposed</span><span class="o">=</span><span class="bp">False</span><span class="p">):</span>
<span class="n">Matrix</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">isTransposed</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">colPtrs</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">rowIndices</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_convert_to_array</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="o">.</span><span class="n">size</span> <span class="o">!=</span> <span class="n">numRows</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Expected colPtrs of size </span><span class="si">%d</span><span class="s">, got </span><span class="si">%d</span><span class="s">."</span>
<span class="o">%</span> <span class="p">(</span><span class="n">numRows</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="o">.</span><span class="n">size</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="o">.</span><span class="n">size</span> <span class="o">!=</span> <span class="n">numCols</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Expected colPtrs of size </span><span class="si">%d</span><span class="s">, got </span><span class="si">%d</span><span class="s">."</span>
<span class="o">%</span> <span class="p">(</span><span class="n">numCols</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="o">.</span><span class="n">size</span><span class="p">))</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="o">.</span><span class="n">size</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">size</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Expected rowIndices of length </span><span class="si">%d</span><span class="s">, got </span><span class="si">%d</span><span class="s">."</span>
<span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="o">.</span><span class="n">size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">size</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Pretty printing of a SparseMatrix</span>
<span class="sd"> >>> sm1 = SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4])</span>
<span class="sd"> >>> print(sm1)</span>
<span class="sd"> 2 X 2 CSCMatrix</span>
<span class="sd"> (0,0) 2.0</span>
<span class="sd"> (1,0) 3.0</span>
<span class="sd"> (1,1) 4.0</span>
<span class="sd"> >>> sm1 = SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4], True)</span>
<span class="sd"> >>> print(sm1)</span>
<span class="sd"> 2 X 2 CSRMatrix</span>
<span class="sd"> (0,0) 2.0</span>
<span class="sd"> (0,1) 3.0</span>
<span class="sd"> (1,1) 4.0</span>
<span class="sd"> """</span>
<span class="n">spstr</span> <span class="o">=</span> <span class="s">"{0} X {1} "</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">:</span>
<span class="n">spstr</span> <span class="o">+=</span> <span class="s">"CSRMatrix</span><span class="se">\n</span><span class="s">"</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">spstr</span> <span class="o">+=</span> <span class="s">"CSCMatrix</span><span class="se">\n</span><span class="s">"</span>
<span class="n">cur_col</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">smlist</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c"># Display first 16 values.</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> <span class="o"><=</span> <span class="mi">16</span><span class="p">:</span>
<span class="n">zipindval</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">zipindval</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="p">[:</span><span class="mi">16</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[:</span><span class="mi">16</span><span class="p">])</span>
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">rowInd</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">zipindval</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="p">[</span><span class="n">cur_col</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span> <span class="o"><=</span> <span class="n">i</span><span class="p">:</span>
<span class="n">cur_col</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">:</span>
<span class="n">smlist</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s">'({0},{1}) {2}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">cur_col</span><span class="p">,</span> <span class="n">rowInd</span><span class="p">,</span> <span class="n">_format_float</span><span class="p">(</span><span class="n">value</span><span class="p">)))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">smlist</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s">'({0},{1}) {2}'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">rowInd</span><span class="p">,</span> <span class="n">cur_col</span><span class="p">,</span> <span class="n">_format_float</span><span class="p">(</span><span class="n">value</span><span class="p">)))</span>
<span class="n">spstr</span> <span class="o">+=</span> <span class="s">"</span><span class="se">\n</span><span class="s">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">smlist</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> <span class="o">></span> <span class="mi">16</span><span class="p">:</span>
<span class="n">spstr</span> <span class="o">+=</span> <span class="s">"</span><span class="se">\n</span><span class="s">.."</span> <span class="o">*</span> <span class="mi">2</span>
<span class="k">return</span> <span class="n">spstr</span>
<span class="k">def</span> <span class="nf">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Representation of a SparseMatrix</span>
<span class="sd"> >>> sm1 = SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4])</span>
<span class="sd"> >>> sm1</span>
<span class="sd"> SparseMatrix(2, 2, [0, 2, 3], [0, 1, 1], [2.0, 3.0, 4.0], False)</span>
<span class="sd"> """</span>
<span class="n">rowIndices</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="p">)</span>
<span class="n">colPtrs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span> <span class="o"><=</span> <span class="mi">16</span><span class="p">:</span>
<span class="n">values</span> <span class="o">=</span> <span class="n">_format_float_list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">values</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">_format_float_list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[:</span><span class="mi">8</span><span class="p">])</span> <span class="o">+</span>
<span class="p">[</span><span class="s">"..."</span><span class="p">]</span> <span class="o">+</span>
<span class="n">_format_float_list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="o">-</span><span class="mi">8</span><span class="p">:])</span>
<span class="p">)</span>
<span class="n">rowIndices</span> <span class="o">=</span> <span class="n">rowIndices</span><span class="p">[:</span><span class="mi">8</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="s">"..."</span><span class="p">]</span> <span class="o">+</span> <span class="n">rowIndices</span><span class="p">[</span><span class="o">-</span><span class="mi">8</span><span class="p">:]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="p">)</span> <span class="o">></span> <span class="mi">16</span><span class="p">:</span>
<span class="n">colPtrs</span> <span class="o">=</span> <span class="n">colPtrs</span><span class="p">[:</span><span class="mi">8</span><span class="p">]</span> <span class="o">+</span> <span class="p">[</span><span class="s">"..."</span><span class="p">]</span> <span class="o">+</span> <span class="n">colPtrs</span><span class="p">[</span><span class="o">-</span><span class="mi">8</span><span class="p">:]</span>
<span class="n">values</span> <span class="o">=</span> <span class="s">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">values</span><span class="p">)</span>
<span class="n">rowIndices</span> <span class="o">=</span> <span class="s">", "</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">ind</span><span class="p">)</span> <span class="k">for</span> <span class="n">ind</span> <span class="ow">in</span> <span class="n">rowIndices</span><span class="p">])</span>
<span class="n">colPtrs</span> <span class="o">=</span> <span class="s">", "</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="nb">str</span><span class="p">(</span><span class="n">ptr</span><span class="p">)</span> <span class="k">for</span> <span class="n">ptr</span> <span class="ow">in</span> <span class="n">colPtrs</span><span class="p">])</span>
<span class="k">return</span> <span class="s">"SparseMatrix({0}, {1}, [{2}], [{3}], [{4}], {5})"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">,</span> <span class="n">colPtrs</span><span class="p">,</span> <span class="n">rowIndices</span><span class="p">,</span>
<span class="n">values</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">__reduce__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="n">SparseMatrix</span><span class="p">,</span> <span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="o">.</span><span class="n">tostring</span><span class="p">(),</span>
<span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="o">.</span><span class="n">tostring</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="o">.</span><span class="n">tostring</span><span class="p">(),</span>
<span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">indices</span><span class="p">):</span>
<span class="n">i</span><span class="p">,</span> <span class="n">j</span> <span class="o">=</span> <span class="n">indices</span>
<span class="k">if</span> <span class="n">i</span> <span class="o"><</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">i</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Row index </span><span class="si">%d</span><span class="s"> is out of range [0, </span><span class="si">%d</span><span class="s">)"</span>
<span class="o">%</span> <span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">))</span>
<span class="k">if</span> <span class="n">j</span> <span class="o"><</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">j</span> <span class="o">>=</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s">"Column index </span><span class="si">%d</span><span class="s"> is out of range [0, </span><span class="si">%d</span><span class="s">)"</span>
<span class="o">%</span> <span class="p">(</span><span class="n">j</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">))</span>
<span class="c"># If a CSR matrix is given, then the row index should be searched</span>
<span class="c"># for in ColPtrs, and the column index should be searched for in the</span>
<span class="c"># corresponding slice obtained from rowIndices.</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">:</span>
<span class="n">j</span><span class="p">,</span> <span class="n">i</span> <span class="o">=</span> <span class="n">i</span><span class="p">,</span> <span class="n">j</span>
<span class="n">colStart</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="p">[</span><span class="n">j</span><span class="p">]</span>
<span class="n">colEnd</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="p">[</span><span class="n">j</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
<span class="n">nz</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="p">[</span><span class="n">colStart</span><span class="p">:</span> <span class="n">colEnd</span><span class="p">]</span>
<span class="n">ind</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">searchsorted</span><span class="p">(</span><span class="n">nz</span><span class="p">,</span> <span class="n">i</span><span class="p">)</span> <span class="o">+</span> <span class="n">colStart</span>
<span class="k">if</span> <span class="n">ind</span> <span class="o"><</span> <span class="n">colEnd</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="p">[</span><span class="n">ind</span><span class="p">]</span> <span class="o">==</span> <span class="n">i</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">ind</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="mf">0.0</span>
<div class="viewcode-block" id="SparseMatrix.toArray"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseMatrix.toArray">[docs]</a> <span class="k">def</span> <span class="nf">toArray</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Return an numpy.ndarray</span>
<span class="sd"> """</span>
<span class="n">A</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">,</span> <span class="n">order</span><span class="o">=</span><span class="s">'F'</span><span class="p">)</span>
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">xrange</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="o">.</span><span class="n">size</span> <span class="o">-</span> <span class="mi">1</span><span class="p">):</span>
<span class="n">startptr</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="p">[</span><span class="n">k</span><span class="p">]</span>
<span class="n">endptr</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">colPtrs</span><span class="p">[</span><span class="n">k</span> <span class="o">+</span> <span class="mi">1</span><span class="p">]</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">isTransposed</span><span class="p">:</span>
<span class="n">A</span><span class="p">[</span><span class="n">k</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="p">[</span><span class="n">startptr</span><span class="p">:</span><span class="n">endptr</span><span class="p">]]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">startptr</span><span class="p">:</span><span class="n">endptr</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">A</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">rowIndices</span><span class="p">[</span><span class="n">startptr</span><span class="p">:</span><span class="n">endptr</span><span class="p">],</span> <span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">values</span><span class="p">[</span><span class="n">startptr</span><span class="p">:</span><span class="n">endptr</span><span class="p">]</span>
<span class="k">return</span> <span class="n">A</span>
</div>
<div class="viewcode-block" id="SparseMatrix.toDense"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.SparseMatrix.toDense">[docs]</a> <span class="k">def</span> <span class="nf">toDense</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">densevals</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">(),</span> <span class="n">order</span><span class="o">=</span><span class="s">'F'</span><span class="p">)</span>
<span class="k">return</span> <span class="n">DenseMatrix</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">numRows</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">numCols</span><span class="p">,</span> <span class="n">densevals</span><span class="p">)</span>
<span class="c"># TODO: More efficient implementation:</span></div>
<span class="k">def</span> <span class="nf">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">all</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">toArray</span><span class="p">()</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">toArray</span><span class="p">())</span>
</div>
<div class="viewcode-block" id="Matrices"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Matrices">[docs]</a><span class="k">class</span> <span class="nc">Matrices</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Matrices.dense"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Matrices.dense">[docs]</a> <span class="k">def</span> <span class="nf">dense</span><span class="p">(</span><span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">values</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Create a DenseMatrix</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">DenseMatrix</span><span class="p">(</span><span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span>
</div>
<span class="nd">@staticmethod</span>
<div class="viewcode-block" id="Matrices.sparse"><a class="viewcode-back" href="../../../pyspark.mllib.html#pyspark.mllib.linalg.Matrices.sparse">[docs]</a> <span class="k">def</span> <span class="nf">sparse</span><span class="p">(</span><span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">colPtrs</span><span class="p">,</span> <span class="n">rowIndices</span><span class="p">,</span> <span class="n">values</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Create a SparseMatrix</span>
<span class="sd"> """</span>
<span class="k">return</span> <span class="n">SparseMatrix</span><span class="p">(</span><span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">,</span> <span class="n">colPtrs</span><span class="p">,</span> <span class="n">rowIndices</span><span class="p">,</span> <span class="n">values</span><span class="p">)</span>
</div></div>
<span class="k">def</span> <span class="nf">_test</span><span class="p">():</span>
<span class="kn">import</span> <span class="nn">doctest</span>
<span class="p">(</span><span class="n">failure_count</span><span class="p">,</span> <span class="n">test_count</span><span class="p">)</span> <span class="o">=</span> <span class="n">doctest</span><span class="o">.</span><span class="n">testmod</span><span class="p">(</span><span class="n">optionflags</span><span class="o">=</span><span class="n">doctest</span><span class="o">.</span><span class="n">ELLIPSIS</span><span class="p">)</span>
<span class="k">if</span> <span class="n">failure_count</span><span class="p">:</span>
<span class="nb">exit</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s">"__main__"</span><span class="p">:</span>
<span class="n">_test</span><span class="p">()</span>
</pre></div>
</div>
</div>
</div>
<div class="sphinxsidebar" role="navigation" aria-label="main navigation">
<div class="sphinxsidebarwrapper">
<p class="logo"><a href="../../../index.html">
<img class="logo" src="../../../_static/spark-logo-hd.png" alt="Logo"/>
</a></p>
<div id="searchbox" style="display: none" role="search">
<h3>Quick search</h3>
<form class="search" action="../../../search.html" method="get">
<input type="text" name="q" />
<input type="submit" value="Go" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
<p class="searchtip" style="font-size: 90%">
Enter search terms or a module, class or function name.
</p>
</div>
<script type="text/javascript">$('#searchbox').show(0);</script>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="related" role="navigation" aria-label="related navigation">
<h3>Navigation</h3>
<ul>
<li class="nav-item nav-item-0"><a href="../../../index.html">PySpark master documentation</a> »</li>
<li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> »</li>
</ul>
</div>
<div class="footer" role="contentinfo">
© Copyright .
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.3.1.
</div>
</body>
</html>