diff options
| author | Patrick Wendell <pwendell@apache.org> | 2014-09-11 05:00:26 +0000 |
|---|---|---|
| committer | Patrick Wendell <pwendell@apache.org> | 2014-09-11 05:00:26 +0000 |
| commit | 07461d1269cd6d373630c20fb50c2988af5c21f4 (patch) | |
| tree | 04c4f9a3cbe613b3d3c79a8581e6b83babbc3e0b /site/docs/1.1.0/mllib-statistics.html | |
| parent | 46d52fbb9be4b5b90a7a1ee9ce3e943156d190b9 (diff) | |
| download | spark-website-07461d1269cd6d373630c20fb50c2988af5c21f4.tar.gz spark-website-07461d1269cd6d373630c20fb50c2988af5c21f4.tar.bz2 spark-website-07461d1269cd6d373630c20fb50c2988af5c21f4.zip | |
Adding Spark 1.1.0 docs.
Diffstat (limited to 'site/docs/1.1.0/mllib-statistics.html')
| -rw-r--r-- | site/docs/1.1.0/mllib-statistics.html | 604 |
1 files changed, 604 insertions, 0 deletions
diff --git a/site/docs/1.1.0/mllib-statistics.html b/site/docs/1.1.0/mllib-statistics.html new file mode 100644 index 000000000..ae57e57d8 --- /dev/null +++ b/site/docs/1.1.0/mllib-statistics.html @@ -0,0 +1,604 @@ +<!DOCTYPE html> +<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]--> +<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8"> <![endif]--> +<!--[if IE 8]> <html class="no-js lt-ie9"> <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]--> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"> + <title>Basic Statistics - MLlib - Spark 1.1.0 Documentation</title> + <meta name="description" content=""> + + + + <link rel="stylesheet" href="css/bootstrap.min.css"> + <style> + body { + padding-top: 60px; + padding-bottom: 40px; + } + </style> + <meta name="viewport" content="width=device-width"> + <link rel="stylesheet" href="css/bootstrap-responsive.min.css"> + <link rel="stylesheet" href="css/main.css"> + + <script src="js/vendor/modernizr-2.6.1-respond-1.1.0.min.js"></script> + + <link rel="stylesheet" href="css/pygments-default.css"> + + + <!-- Google analytics script --> + <script type="text/javascript"> + var _gaq = _gaq || []; + _gaq.push(['_setAccount', 'UA-32518208-1']); + _gaq.push(['_trackPageview']); + + (function() { + var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; + ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; + var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); + })(); + </script> + + + </head> + <body> + <!--[if lt IE 7]> + <p class="chromeframe">You are using an outdated browser. <a href="http://browsehappy.com/">Upgrade your browser today</a> or <a href="http://www.google.com/chromeframe/?redirect=true">install Google Chrome Frame</a> to better experience this site.</p> + <![endif]--> + + <!-- This code is taken from http://twitter.github.com/bootstrap/examples/hero.html --> + + <div class="navbar navbar-fixed-top" id="topbar"> + <div class="navbar-inner"> + <div class="container"> + <div class="brand"><a href="index.html"> + <img src="img/spark-logo-hd.png" style="height:50px;"/></a><span class="version">1.1.0</span> + </div> + <ul class="nav"> + <!--TODO(andyk): Add class="active" attribute to li some how.--> + <li><a href="index.html">Overview</a></li> + + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Programming Guides<b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="quick-start.html">Quick Start</a></li> + <li><a href="programming-guide.html">Spark Programming Guide</a></li> + <li class="divider"></li> + <li><a href="streaming-programming-guide.html">Spark Streaming</a></li> + <li><a href="sql-programming-guide.html">Spark SQL</a></li> + <li><a href="mllib-guide.html">MLlib (Machine Learning)</a></li> + <li><a href="graphx-programming-guide.html">GraphX (Graph Processing)</a></li> + <li><a href="bagel-programming-guide.html">Bagel (Pregel on Spark)</a></li> + </ul> + </li> + + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">API Docs<b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="api/scala/index.html#org.apache.spark.package">Scaladoc</a></li> + <li><a href="api/java/index.html">Javadoc</a></li> + <li><a href="api/python/index.html">Python API</a></li> + </ul> + </li> + + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Deploying<b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="cluster-overview.html">Overview</a></li> + <li><a href="submitting-applications.html">Submitting Applications</a></li> + <li class="divider"></li> + <li><a href="ec2-scripts.html">Amazon EC2</a></li> + <li><a href="spark-standalone.html">Standalone Mode</a></li> + <li><a href="running-on-mesos.html">Mesos</a></li> + <li><a href="running-on-yarn.html">YARN</a></li> + </ul> + </li> + + <li class="dropdown"> + <a href="api.html" class="dropdown-toggle" data-toggle="dropdown">More<b class="caret"></b></a> + <ul class="dropdown-menu"> + <li><a href="configuration.html">Configuration</a></li> + <li><a href="monitoring.html">Monitoring</a></li> + <li><a href="tuning.html">Tuning Guide</a></li> + <li><a href="job-scheduling.html">Job Scheduling</a></li> + <li><a href="security.html">Security</a></li> + <li><a href="hardware-provisioning.html">Hardware Provisioning</a></li> + <li><a href="hadoop-third-party-distributions.html">3<sup>rd</sup>-Party Hadoop Distros</a></li> + <li class="divider"></li> + <li><a href="building-with-maven.html">Building Spark with Maven</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SPARK/Contributing+to+Spark">Contributing to Spark</a></li> + </ul> + </li> + </ul> + <!--<p class="navbar-text pull-right"><span class="version-text">v1.1.0</span></p>--> + </div> + </div> + </div> + + <div class="container" id="content"> + + <h1 class="title"><a href="mllib-guide.html">MLlib</a> - Basic Statistics</h1> + + + <ul id="markdown-toc"> + <li><a href="#summary-statistics">Summary statistics</a></li> + <li><a href="#correlations">Correlations</a></li> + <li><a href="#stratified-sampling">Stratified sampling</a></li> + <li><a href="#hypothesis-testing">Hypothesis testing</a></li> + <li><a href="#random-data-generation">Random data generation</a></li> +</ul> + +<p><code>\[ +\newcommand{\R}{\mathbb{R}} +\newcommand{\E}{\mathbb{E}} +\newcommand{\x}{\mathbf{x}} +\newcommand{\y}{\mathbf{y}} +\newcommand{\wv}{\mathbf{w}} +\newcommand{\av}{\mathbf{\alpha}} +\newcommand{\bv}{\mathbf{b}} +\newcommand{\N}{\mathbb{N}} +\newcommand{\id}{\mathbf{I}} +\newcommand{\ind}{\mathbf{1}} +\newcommand{\0}{\mathbf{0}} +\newcommand{\unit}{\mathbf{e}} +\newcommand{\one}{\mathbf{1}} +\newcommand{\zero}{\mathbf{0}} +\]</code></p> + +<h2 id="summary-statistics">Summary statistics</h2> + +<p>We provide column summary statistics for <code>RDD[Vector]</code> through the function <code>colStats</code> +available in <code>Statistics</code>.</p> + +<div class="codetabs"> +<div data-lang="scala"> + + <p><a href="api/scala/index.html#org.apache.spark.mllib.stat.Statistics$"><code>colStats()</code></a> returns an instance of +<a href="api/scala/index.html#org.apache.spark.mllib.stat.MultivariateStatisticalSummary"><code>MultivariateStatisticalSummary</code></a>, +which contains the column-wise max, min, mean, variance, and number of nonzeros, as well as the +total count.</p> + + <div class="highlight"><pre><code class="scala"><span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vector</span> +<span class="k">import</span> <span class="nn">org.apache.spark.mllib.stat.</span><span class="o">{</span><span class="nc">MultivariateStatisticalSummary</span><span class="o">,</span> <span class="nc">Statistics</span><span class="o">}</span> + +<span class="k">val</span> <span class="n">observations</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> <span class="c1">// an RDD of Vectors</span> + +<span class="c1">// Compute column summary statistics.</span> +<span class="k">val</span> <span class="n">summary</span><span class="k">:</span> <span class="kt">MultivariateStatisticalSummary</span> <span class="o">=</span> <span class="nc">Statistics</span><span class="o">.</span><span class="n">colStats</span><span class="o">(</span><span class="n">observations</span><span class="o">)</span> +<span class="n">println</span><span class="o">(</span><span class="n">summary</span><span class="o">.</span><span class="n">mean</span><span class="o">)</span> <span class="c1">// a dense vector containing the mean value for each column</span> +<span class="n">println</span><span class="o">(</span><span class="n">summary</span><span class="o">.</span><span class="n">variance</span><span class="o">)</span> <span class="c1">// column-wise variance</span> +<span class="n">println</span><span class="o">(</span><span class="n">summary</span><span class="o">.</span><span class="n">numNonzeros</span><span class="o">)</span> <span class="c1">// number of nonzeros in each column</span> +</code></pre></div> + + </div> + +<div data-lang="java"> + + <p><a href="api/java/org/apache/spark/mllib/stat/Statistics.html"><code>colStats()</code></a> returns an instance of +<a href="api/java/org/apache/spark/mllib/stat/MultivariateStatisticalSummary.html"><code>MultivariateStatisticalSummary</code></a>, +which contains the column-wise max, min, mean, variance, and number of nonzeros, as well as the +total count.</p> + + <div class="highlight"><pre><code class="java"><span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaRDD</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaSparkContext</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.Vector</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.stat.MultivariateStatisticalSummary</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.stat.Statistics</span><span class="o">;</span> + +<span class="n">JavaSparkContext</span> <span class="n">jsc</span> <span class="o">=</span> <span class="o">...</span> + +<span class="n">JavaRDD</span><span class="o"><</span><span class="n">Vector</span><span class="o">></span> <span class="n">mat</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// an RDD of Vectors</span> + +<span class="c1">// Compute column summary statistics.</span> +<span class="n">MultivariateStatisticalSummary</span> <span class="n">summary</span> <span class="o">=</span> <span class="n">Statistics</span><span class="o">.</span><span class="na">colStats</span><span class="o">(</span><span class="n">mat</span><span class="o">.</span><span class="na">rdd</span><span class="o">());</span> +<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">summary</span><span class="o">.</span><span class="na">mean</span><span class="o">());</span> <span class="c1">// a dense vector containing the mean value for each column</span> +<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">summary</span><span class="o">.</span><span class="na">variance</span><span class="o">());</span> <span class="c1">// column-wise variance</span> +<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">summary</span><span class="o">.</span><span class="na">numNonzeros</span><span class="o">());</span> <span class="c1">// number of nonzeros in each column</span> +</code></pre></div> + + </div> + +<div data-lang="python"> + <p><a href="api/python/pyspark.mllib.stat.Statistics-class.html#colStats"><code>colStats()</code></a> returns an instance of +<a href="api/python/pyspark.mllib.stat.MultivariateStatisticalSummary-class.html"><code>MultivariateStatisticalSummary</code></a>, +which contains the column-wise max, min, mean, variance, and number of nonzeros, as well as the +total count.</p> + + <div class="highlight"><pre><code class="python"><span class="kn">from</span> <span class="nn">pyspark.mllib.stat</span> <span class="kn">import</span> <span class="n">Statistics</span> + +<span class="n">sc</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># SparkContext</span> + +<span class="n">mat</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># an RDD of Vectors</span> + +<span class="c"># Compute column summary statistics.</span> +<span class="n">summary</span> <span class="o">=</span> <span class="n">Statistics</span><span class="o">.</span><span class="n">colStats</span><span class="p">(</span><span class="n">mat</span><span class="p">)</span> +<span class="k">print</span> <span class="n">summary</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span> +<span class="k">print</span> <span class="n">summary</span><span class="o">.</span><span class="n">variance</span><span class="p">()</span> +<span class="k">print</span> <span class="n">summary</span><span class="o">.</span><span class="n">numNonzeros</span><span class="p">()</span> +</code></pre></div> + + </div> + +</div> + +<h2 id="correlations">Correlations</h2> + +<p>Calculating the correlation between two series of data is a common operation in Statistics. In MLlib +we provide the flexibility to calculate pairwise correlations among many series. The supported +correlation methods are currently Pearson’s and Spearman’s correlation.</p> + +<div class="codetabs"> +<div data-lang="scala"> + <p><a href="api/scala/index.html#org.apache.spark.mllib.stat.Statistics$"><code>Statistics</code></a> provides methods to +calculate correlations between series. Depending on the type of input, two <code>RDD[Double]</code>s or +an <code>RDD[Vector]</code>, the output will be a <code>Double</code> or the correlation <code>Matrix</code> respectively.</p> + + <div class="highlight"><pre><code class="scala"><span class="k">import</span> <span class="nn">org.apache.spark.SparkContext</span> +<span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg._</span> +<span class="k">import</span> <span class="nn">org.apache.spark.mllib.stat.Statistics</span> + +<span class="k">val</span> <span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span> <span class="o">=</span> <span class="o">...</span> + +<span class="k">val</span> <span class="n">seriesX</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">Double</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> <span class="c1">// a series</span> +<span class="k">val</span> <span class="n">seriesY</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">Double</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> <span class="c1">// must have the same number of partitions and cardinality as seriesX</span> + +<span class="c1">// compute the correlation using Pearson's method. Enter "spearman" for Spearman's method. If a </span> +<span class="c1">// method is not specified, Pearson's method will be used by default. </span> +<span class="k">val</span> <span class="n">correlation</span><span class="k">:</span> <span class="kt">Double</span> <span class="o">=</span> <span class="nc">Statistics</span><span class="o">.</span><span class="n">corr</span><span class="o">(</span><span class="n">seriesX</span><span class="o">,</span> <span class="n">seriesY</span><span class="o">,</span> <span class="s">"pearson"</span><span class="o">)</span> + +<span class="k">val</span> <span class="n">data</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> <span class="c1">// note that each Vector is a row and not a column</span> + +<span class="c1">// calculate the correlation matrix using Pearson's method. Use "spearman" for Spearman's method.</span> +<span class="c1">// If a method is not specified, Pearson's method will be used by default. </span> +<span class="k">val</span> <span class="n">correlMatrix</span><span class="k">:</span> <span class="kt">Matrix</span> <span class="o">=</span> <span class="nc">Statistics</span><span class="o">.</span><span class="n">corr</span><span class="o">(</span><span class="n">data</span><span class="o">,</span> <span class="s">"pearson"</span><span class="o">)</span> +</code></pre></div> + + </div> + +<div data-lang="java"> + <p><a href="api/java/org/apache/spark/mllib/stat/Statistics.html"><code>Statistics</code></a> provides methods to +calculate correlations between series. Depending on the type of input, two <code>JavaDoubleRDD</code>s or +a <code>JavaRDD<Vector></code>, the output will be a <code>Double</code> or the correlation <code>Matrix</code> respectively.</p> + + <div class="highlight"><pre><code class="java"><span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaDoubleRDD</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaSparkContext</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.stat.Statistics</span><span class="o">;</span> + +<span class="n">JavaSparkContext</span> <span class="n">jsc</span> <span class="o">=</span> <span class="o">...</span> + +<span class="n">JavaDoubleRDD</span> <span class="n">seriesX</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// a series</span> +<span class="n">JavaDoubleRDD</span> <span class="n">seriesY</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// must have the same number of partitions and cardinality as seriesX</span> + +<span class="c1">// compute the correlation using Pearson's method. Enter "spearman" for Spearman's method. If a </span> +<span class="c1">// method is not specified, Pearson's method will be used by default. </span> +<span class="n">Double</span> <span class="n">correlation</span> <span class="o">=</span> <span class="n">Statistics</span><span class="o">.</span><span class="na">corr</span><span class="o">(</span><span class="n">seriesX</span><span class="o">.</span><span class="na">srdd</span><span class="o">(),</span> <span class="n">seriesY</span><span class="o">.</span><span class="na">srdd</span><span class="o">(),</span> <span class="s">"pearson"</span><span class="o">);</span> + +<span class="n">JavaRDD</span><span class="o"><</span><span class="n">Vector</span><span class="o">></span> <span class="n">data</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// note that each Vector is a row and not a column</span> + +<span class="c1">// calculate the correlation matrix using Pearson's method. Use "spearman" for Spearman's method.</span> +<span class="c1">// If a method is not specified, Pearson's method will be used by default. </span> +<span class="n">Matrix</span> <span class="n">correlMatrix</span> <span class="o">=</span> <span class="n">Statistics</span><span class="o">.</span><span class="na">corr</span><span class="o">(</span><span class="n">data</span><span class="o">.</span><span class="na">rdd</span><span class="o">(),</span> <span class="s">"pearson"</span><span class="o">);</span> +</code></pre></div> + + </div> + +<div data-lang="python"> + <p><a href="api/python/pyspark.mllib.stat.Statistics-class.html"><code>Statistics</code></a> provides methods to +calculate correlations between series. Depending on the type of input, two <code>RDD[Double]</code>s or +an <code>RDD[Vector]</code>, the output will be a <code>Double</code> or the correlation <code>Matrix</code> respectively.</p> + + <div class="highlight"><pre><code class="python"><span class="kn">from</span> <span class="nn">pyspark.mllib.stat</span> <span class="kn">import</span> <span class="n">Statistics</span> + +<span class="n">sc</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># SparkContext</span> + +<span class="n">seriesX</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># a series</span> +<span class="n">seriesY</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># must have the same number of partitions and cardinality as seriesX</span> + +<span class="c"># Compute the correlation using Pearson's method. Enter "spearman" for Spearman's method. If a </span> +<span class="c"># method is not specified, Pearson's method will be used by default. </span> +<span class="k">print</span> <span class="n">Statistics</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">seriesX</span><span class="p">,</span> <span class="n">seriesY</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s">"pearson"</span><span class="p">)</span> + +<span class="n">data</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># an RDD of Vectors</span> +<span class="c"># calculate the correlation matrix using Pearson's method. Use "spearman" for Spearman's method.</span> +<span class="c"># If a method is not specified, Pearson's method will be used by default. </span> +<span class="k">print</span> <span class="n">Statistics</span><span class="o">.</span><span class="n">corr</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s">"pearson"</span><span class="p">)</span> +</code></pre></div> + + </div> + +</div> + +<h2 id="stratified-sampling">Stratified sampling</h2> + +<p>Unlike the other statistics functions, which reside in MLLib, stratified sampling methods, +<code>sampleByKey</code> and <code>sampleByKeyExact</code>, can be performed on RDD’s of key-value pairs. For stratified +sampling, the keys can be thought of as a label and the value as a specific attribute. For example +the key can be man or woman, or document ids, and the respective values can be the list of ages +of the people in the population or the list of words in the documents. The <code>sampleByKey</code> method +will flip a coin to decide whether an observation will be sampled or not, therefore requires one +pass over the data, and provides an <em>expected</em> sample size. <code>sampleByKeyExact</code> requires significant +more resources than the per-stratum simple random sampling used in <code>sampleByKey</code>, but will provide +the exact sampling size with 99.99% confidence. <code>sampleByKeyExact</code> is currently not supported in +python.</p> + +<div class="codetabs"> +<div data-lang="scala"> + <p><a href="api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions"><code>sampleByKeyExact()</code></a> allows users to +sample exactly $\lceil f_k \cdot n_k \rceil \, \forall k \in K$ items, where $f_k$ is the desired +fraction for key $k$, $n_k$ is the number of key-value pairs for key $k$, and $K$ is the set of +keys. Sampling without replacement requires one additional pass over the RDD to guarantee sample +size, whereas sampling with replacement requires two additional passes.</p> + + <div class="highlight"><pre><code class="scala"><span class="k">import</span> <span class="nn">org.apache.spark.SparkContext</span> +<span class="k">import</span> <span class="nn">org.apache.spark.SparkContext._</span> +<span class="k">import</span> <span class="nn">org.apache.spark.rdd.PairRDDFunctions</span> + +<span class="k">val</span> <span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span> <span class="o">=</span> <span class="o">...</span> + +<span class="k">val</span> <span class="n">data</span> <span class="k">=</span> <span class="o">...</span> <span class="c1">// an RDD[(K, V)] of any key value pairs</span> +<span class="k">val</span> <span class="n">fractions</span><span class="k">:</span> <span class="kt">Map</span><span class="o">[</span><span class="kt">K</span>, <span class="kt">Double</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> <span class="c1">// specify the exact fraction desired from each key</span> + +<span class="c1">// Get an exact sample from each stratum</span> +<span class="k">val</span> <span class="n">approxSample</span> <span class="k">=</span> <span class="n">data</span><span class="o">.</span><span class="n">sampleByKey</span><span class="o">(</span><span class="n">withReplacement</span> <span class="k">=</span> <span class="kc">false</span><span class="o">,</span> <span class="n">fractions</span><span class="o">)</span> +<span class="k">val</span> <span class="n">exactSample</span> <span class="k">=</span> <span class="n">data</span><span class="o">.</span><span class="n">sampleByKeyExact</span><span class="o">(</span><span class="n">withReplacement</span> <span class="k">=</span> <span class="kc">false</span><span class="o">,</span> <span class="n">fractions</span><span class="o">)</span> +</code></pre></div> + + </div> + +<div data-lang="java"> + <p><a href="api/java/org/apache/spark/api/java/JavaPairRDD.html"><code>sampleByKeyExact()</code></a> allows users to +sample exactly $\lceil f_k \cdot n_k \rceil \, \forall k \in K$ items, where $f_k$ is the desired +fraction for key $k$, $n_k$ is the number of key-value pairs for key $k$, and $K$ is the set of +keys. Sampling without replacement requires one additional pass over the RDD to guarantee sample +size, whereas sampling with replacement requires two additional passes.</p> + + <div class="highlight"><pre><code class="java"><span class="kn">import</span> <span class="nn">java.util.Map</span><span class="o">;</span> + +<span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaPairRDD</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaSparkContext</span><span class="o">;</span> + +<span class="n">JavaSparkContext</span> <span class="n">jsc</span> <span class="o">=</span> <span class="o">...</span> + +<span class="n">JavaPairRDD</span><span class="o"><</span><span class="n">K</span><span class="o">,</span> <span class="n">V</span><span class="o">></span> <span class="n">data</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// an RDD of any key value pairs</span> +<span class="n">Map</span><span class="o"><</span><span class="n">K</span><span class="o">,</span> <span class="n">Object</span><span class="o">></span> <span class="n">fractions</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// specify the exact fraction desired from each key</span> + +<span class="c1">// Get an exact sample from each stratum</span> +<span class="n">JavaPairRDD</span><span class="o"><</span><span class="n">K</span><span class="o">,</span> <span class="n">V</span><span class="o">></span> <span class="n">approxSample</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="na">sampleByKey</span><span class="o">(</span><span class="kc">false</span><span class="o">,</span> <span class="n">fractions</span><span class="o">);</span> +<span class="n">JavaPairRDD</span><span class="o"><</span><span class="n">K</span><span class="o">,</span> <span class="n">V</span><span class="o">></span> <span class="n">exactSample</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="na">sampleByKeyExact</span><span class="o">(</span><span class="kc">false</span><span class="o">,</span> <span class="n">fractions</span><span class="o">);</span> +</code></pre></div> + + </div> +<div data-lang="python"> + <p><a href="api/python/pyspark.rdd.RDD-class.html#sampleByKey"><code>sampleByKey()</code></a> allows users to +sample approximately $\lceil f_k \cdot n_k \rceil \, \forall k \in K$ items, where $f_k$ is the +desired fraction for key $k$, $n_k$ is the number of key-value pairs for key $k$, and $K$ is the +set of keys.</p> + + <p><em>Note:</em> <code>sampleByKeyExact()</code> is currently not supported in Python.</p> + + <div class="highlight"><pre><code class="python"><span class="n">sc</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># SparkContext</span> + +<span class="n">data</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># an RDD of any key value pairs</span> +<span class="n">fractions</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># specify the exact fraction desired from each key as a dictionary</span> + +<span class="n">approxSample</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">sampleByKey</span><span class="p">(</span><span class="bp">False</span><span class="p">,</span> <span class="n">fractions</span><span class="p">);</span> +</code></pre></div> + + </div> + +</div> + +<h2 id="hypothesis-testing">Hypothesis testing</h2> + +<p>Hypothesis testing is a powerful tool in statistics to determine whether a result is statistically +significant, whether this result occurred by chance or not. MLlib currently supports Pearson’s +chi-squared ( $\chi^2$) tests for goodness of fit and independence. The input data types determine +whether the goodness of fit or the independence test is conducted. The goodness of fit test requires +an input type of <code>Vector</code>, whereas the independence test requires a <code>Matrix</code> as input.</p> + +<p>MLlib also supports the input type <code>RDD[LabeledPoint]</code> to enable feature selection via chi-squared +independence tests.</p> + +<div class="codetabs"> +<div data-lang="scala"> + <p><a href="api/scala/index.html#org.apache.spark.mllib.stat.Statistics$"><code>Statistics</code></a> provides methods to +run Pearson’s chi-squared tests. The following example demonstrates how to run and interpret +hypothesis tests.</p> + + <div class="highlight"><pre><code class="scala"><span class="k">import</span> <span class="nn">org.apache.spark.SparkContext</span> +<span class="k">import</span> <span class="nn">org.apache.spark.mllib.linalg._</span> +<span class="k">import</span> <span class="nn">org.apache.spark.mllib.regression.LabeledPoint</span> +<span class="k">import</span> <span class="nn">org.apache.spark.mllib.stat.Statistics._</span> + +<span class="k">val</span> <span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span> <span class="o">=</span> <span class="o">...</span> + +<span class="k">val</span> <span class="n">vec</span><span class="k">:</span> <span class="kt">Vector</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// a vector composed of the frequencies of events</span> + +<span class="c1">// compute the goodness of fit. If a second vector to test against is not supplied as a parameter, </span> +<span class="c1">// the test runs against a uniform distribution. </span> +<span class="k">val</span> <span class="n">goodnessOfFitTestResult</span> <span class="k">=</span> <span class="nc">Statistics</span><span class="o">.</span><span class="n">chiSqTest</span><span class="o">(</span><span class="n">vec</span><span class="o">)</span> +<span class="n">println</span><span class="o">(</span><span class="n">goodnessOfFitTestResult</span><span class="o">)</span> <span class="c1">// summary of the test including the p-value, degrees of freedom, </span> + <span class="c1">// test statistic, the method used, and the null hypothesis.</span> + +<span class="k">val</span> <span class="n">mat</span><span class="k">:</span> <span class="kt">Matrix</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// a contingency matrix</span> + +<span class="c1">// conduct Pearson's independence test on the input contingency matrix</span> +<span class="k">val</span> <span class="n">independenceTestResult</span> <span class="k">=</span> <span class="nc">Statistics</span><span class="o">.</span><span class="n">chiSqTest</span><span class="o">(</span><span class="n">mat</span><span class="o">)</span> +<span class="n">println</span><span class="o">(</span><span class="n">independenceTestResult</span><span class="o">)</span> <span class="c1">// summary of the test including the p-value, degrees of freedom...</span> + +<span class="k">val</span> <span class="n">obs</span><span class="k">:</span> <span class="kt">RDD</span><span class="o">[</span><span class="kt">LabeledPoint</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> <span class="c1">// (feature, label) pairs.</span> + +<span class="c1">// The contingency table is constructed from the raw (feature, label) pairs and used to conduct</span> +<span class="c1">// the independence test. Returns an array containing the ChiSquaredTestResult for every feature </span> +<span class="c1">// against the label.</span> +<span class="k">val</span> <span class="n">featureTestResults</span><span class="k">:</span> <span class="kt">Array</span><span class="o">[</span><span class="kt">ChiSqTestResult</span><span class="o">]</span> <span class="k">=</span> <span class="nc">Statistics</span><span class="o">.</span><span class="n">chiSqTest</span><span class="o">(</span><span class="n">obs</span><span class="o">)</span> +<span class="k">var</span> <span class="n">i</span> <span class="k">=</span> <span class="mi">1</span> +<span class="n">featureTestResults</span><span class="o">.</span><span class="n">foreach</span> <span class="o">{</span> <span class="n">result</span> <span class="k">=></span> + <span class="n">println</span><span class="o">(</span><span class="n">s</span><span class="s">"Column $i:\n$result"</span><span class="o">)</span> + <span class="n">i</span> <span class="o">+=</span> <span class="mi">1</span> +<span class="o">}</span> <span class="c1">// summary of the test </span> +</code></pre></div> + + </div> + +<div data-lang="java"> + <p><a href="api/java/org/apache/spark/mllib/stat/Statistics.html"><code>Statistics</code></a> provides methods to +run Pearson’s chi-squared tests. The following example demonstrates how to run and interpret +hypothesis tests.</p> + + <div class="highlight"><pre><code class="java"><span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaRDD</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.api.java.JavaSparkContext</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.linalg.*</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.regression.LabeledPoint</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.stat.Statistics</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.mllib.stat.test.ChiSqTestResult</span><span class="o">;</span> + +<span class="n">JavaSparkContext</span> <span class="n">jsc</span> <span class="o">=</span> <span class="o">...</span> + +<span class="n">Vector</span> <span class="n">vec</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// a vector composed of the frequencies of events</span> + +<span class="c1">// compute the goodness of fit. If a second vector to test against is not supplied as a parameter, </span> +<span class="c1">// the test runs against a uniform distribution. </span> +<span class="n">ChiSqTestResult</span> <span class="n">goodnessOfFitTestResult</span> <span class="o">=</span> <span class="n">Statistics</span><span class="o">.</span><span class="na">chiSqTest</span><span class="o">(</span><span class="n">vec</span><span class="o">);</span> +<span class="c1">// summary of the test including the p-value, degrees of freedom, test statistic, the method used, </span> +<span class="c1">// and the null hypothesis.</span> +<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">goodnessOfFitTestResult</span><span class="o">);</span> + +<span class="n">Matrix</span> <span class="n">mat</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// a contingency matrix</span> + +<span class="c1">// conduct Pearson's independence test on the input contingency matrix</span> +<span class="n">ChiSqTestResult</span> <span class="n">independenceTestResult</span> <span class="o">=</span> <span class="n">Statistics</span><span class="o">.</span><span class="na">chiSqTest</span><span class="o">(</span><span class="n">mat</span><span class="o">);</span> +<span class="c1">// summary of the test including the p-value, degrees of freedom...</span> +<span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">independenceTestResult</span><span class="o">);</span> + +<span class="n">JavaRDD</span><span class="o"><</span><span class="n">LabeledPoint</span><span class="o">></span> <span class="n">obs</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// an RDD of labeled points</span> + +<span class="c1">// The contingency table is constructed from the raw (feature, label) pairs and used to conduct</span> +<span class="c1">// the independence test. Returns an array containing the ChiSquaredTestResult for every feature </span> +<span class="c1">// against the label.</span> +<span class="n">ChiSqTestResult</span><span class="o">[]</span> <span class="n">featureTestResults</span> <span class="o">=</span> <span class="n">Statistics</span><span class="o">.</span><span class="na">chiSqTest</span><span class="o">(</span><span class="n">obs</span><span class="o">.</span><span class="na">rdd</span><span class="o">());</span> +<span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">1</span><span class="o">;</span> +<span class="k">for</span> <span class="o">(</span><span class="n">ChiSqTestResult</span> <span class="n">result</span> <span class="o">:</span> <span class="n">featureTestResults</span><span class="o">)</span> <span class="o">{</span> + <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">"Column "</span> <span class="o">+</span> <span class="n">i</span> <span class="o">+</span> <span class="s">":"</span><span class="o">);</span> + <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="n">result</span><span class="o">);</span> <span class="c1">// summary of the test</span> + <span class="n">i</span><span class="o">++;</span> +<span class="o">}</span> +</code></pre></div> + + </div> + +</div> + +<h2 id="random-data-generation">Random data generation</h2> + +<p>Random data generation is useful for randomized algorithms, prototyping, and performance testing. +MLlib supports generating random RDDs with i.i.d. values drawn from a given distribution: +uniform, standard normal, or Poisson.</p> + +<div class="codetabs"> +<div data-lang="scala"> + <p><a href="api/scala/index.html#org.apache.spark.mllib.random.RandomRDDs"><code>RandomRDDs</code></a> provides factory +methods to generate random double RDDs or vector RDDs. +The following example generates a random double RDD, whose values follows the standard normal +distribution <code>N(0, 1)</code>, and then map it to <code>N(1, 4)</code>.</p> + + <div class="highlight"><pre><code class="scala"><span class="k">import</span> <span class="nn">org.apache.spark.SparkContext</span> +<span class="k">import</span> <span class="nn">org.apache.spark.mllib.random.RandomRDDs._</span> + +<span class="k">val</span> <span class="n">sc</span><span class="k">:</span> <span class="kt">SparkContext</span> <span class="o">=</span> <span class="o">...</span> + +<span class="c1">// Generate a random double RDD that contains 1 million i.i.d. values drawn from the</span> +<span class="c1">// standard normal distribution `N(0, 1)`, evenly distributed in 10 partitions.</span> +<span class="k">val</span> <span class="n">u</span> <span class="k">=</span> <span class="n">normalRDD</span><span class="o">(</span><span class="n">sc</span><span class="o">,</span> <span class="mi">1000000L</span><span class="o">,</span> <span class="mi">10</span><span class="o">)</span> +<span class="c1">// Apply a transform to get a random double RDD following `N(1, 4)`.</span> +<span class="k">val</span> <span class="n">v</span> <span class="k">=</span> <span class="n">u</span><span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="n">x</span> <span class="k">=></span> <span class="mf">1.0</span> <span class="o">+</span> <span class="mf">2.0</span> <span class="o">*</span> <span class="n">x</span><span class="o">)</span> +</code></pre></div> + + </div> + +<div data-lang="java"> + <p><a href="api/java/index.html#org.apache.spark.mllib.random.RandomRDDs"><code>RandomRDDs</code></a> provides factory +methods to generate random double RDDs or vector RDDs. +The following example generates a random double RDD, whose values follows the standard normal +distribution <code>N(0, 1)</code>, and then map it to <code>N(1, 4)</code>.</p> + + <div class="highlight"><pre><code class="java"><span class="kn">import</span> <span class="nn">org.apache.spark.SparkContext</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">org.apache.spark.api.JavaDoubleRDD</span><span class="o">;</span> +<span class="kn">import</span> <span class="nn">static</span> <span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">spark</span><span class="o">.</span><span class="na">mllib</span><span class="o">.</span><span class="na">random</span><span class="o">.</span><span class="na">RandomRDDs</span><span class="o">.*;</span> + +<span class="n">JavaSparkContext</span> <span class="n">jsc</span> <span class="o">=</span> <span class="o">...</span> + +<span class="c1">// Generate a random double RDD that contains 1 million i.i.d. values drawn from the</span> +<span class="c1">// standard normal distribution `N(0, 1)`, evenly distributed in 10 partitions.</span> +<span class="n">JavaDoubleRDD</span> <span class="n">u</span> <span class="o">=</span> <span class="n">normalJavaRDD</span><span class="o">(</span><span class="n">jsc</span><span class="o">,</span> <span class="mi">1000000L</span><span class="o">,</span> <span class="mi">10</span><span class="o">);</span> +<span class="c1">// Apply a transform to get a random double RDD following `N(1, 4)`.</span> +<span class="n">JavaDoubleRDD</span> <span class="n">v</span> <span class="o">=</span> <span class="n">u</span><span class="o">.</span><span class="na">map</span><span class="o">(</span> + <span class="k">new</span> <span class="n">Function</span><span class="o"><</span><span class="n">Double</span><span class="o">,</span> <span class="n">Double</span><span class="o">>()</span> <span class="o">{</span> + <span class="kd">public</span> <span class="n">Double</span> <span class="nf">call</span><span class="o">(</span><span class="n">Double</span> <span class="n">x</span><span class="o">)</span> <span class="o">{</span> + <span class="k">return</span> <span class="mf">1.0</span> <span class="o">+</span> <span class="mf">2.0</span> <span class="o">*</span> <span class="n">x</span><span class="o">;</span> + <span class="o">}</span> + <span class="o">});</span> +</code></pre></div> + + </div> + +<div data-lang="python"> + <p><a href="api/python/pyspark.mllib.random.RandomRDDs-class.html"><code>RandomRDDs</code></a> provides factory +methods to generate random double RDDs or vector RDDs. +The following example generates a random double RDD, whose values follows the standard normal +distribution <code>N(0, 1)</code>, and then map it to <code>N(1, 4)</code>.</p> + + <div class="highlight"><pre><code class="python"><span class="kn">from</span> <span class="nn">pyspark.mllib.random</span> <span class="kn">import</span> <span class="n">RandomRDDs</span> + +<span class="n">sc</span> <span class="o">=</span> <span class="o">...</span> <span class="c"># SparkContext</span> + +<span class="c"># Generate a random double RDD that contains 1 million i.i.d. values drawn from the</span> +<span class="c"># standard normal distribution `N(0, 1)`, evenly distributed in 10 partitions.</span> +<span class="n">u</span> <span class="o">=</span> <span class="n">RandomRDDs</span><span class="o">.</span><span class="n">uniformRDD</span><span class="p">(</span><span class="n">sc</span><span class="p">,</span> <span class="il">1000000L</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span> +<span class="c"># Apply a transform to get a random double RDD following `N(1, 4)`.</span> +<span class="n">v</span> <span class="o">=</span> <span class="n">u</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="mf">1.0</span> <span class="o">+</span> <span class="mf">2.0</span> <span class="o">*</span> <span class="n">x</span><span class="p">)</span> +</code></pre></div> + + </div> + +</div> + + + </div> <!-- /container --> + + <script src="js/vendor/jquery-1.8.0.min.js"></script> + <script src="js/vendor/bootstrap.min.js"></script> + <script src="js/main.js"></script> + + <!-- MathJax Section --> + <script type="text/x-mathjax-config"> + MathJax.Hub.Config({ + TeX: { equationNumbers: { autoNumber: "AMS" } } + }); + </script> + <script> + // Note that we load MathJax this way to work with local file (file://), HTTP and HTTPS. + // We could use "//cdn.mathjax...", but that won't support "file://". + (function(d, script) { + script = d.createElement('script'); + script.type = 'text/javascript'; + script.async = true; + script.onload = function(){ + MathJax.Hub.Config({ + tex2jax: { + inlineMath: [ ["$", "$"], ["\\\\(","\\\\)"] ], + displayMath: [ ["$$","$$"], ["\\[", "\\]"] ], + processEscapes: true, + skipTags: ['script', 'noscript', 'style', 'textarea', 'pre'] + } + }); + }; + script.src = ('https:' == document.location.protocol ? 'https://' : 'http://') + + 'cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML'; + d.getElementsByTagName('head')[0].appendChild(script); + }(document)); + </script> + </body> +</html> |