<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc (version 1.7.0_51) on Wed Sep 16 15:55:11 PDT 2015 -->
<title>SparkContext</title>
<meta name="date" content="2015-09-16">
<link rel="stylesheet" type="text/css" href="../../../stylesheet.css" title="Style">
</head>
<body>
<script type="text/javascript"><!--
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="SparkContext";
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar_top">
<!-- -->
</a><a href="#skip-navbar_top" title="Skip navigation links"></a><a name="navbar_top_firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../index-all.html">Index</a></li>
<li><a href="../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark"><span class="strong">Prev Class</span></a></li>
<li><a href="../../../org/apache/spark/SparkContext.DoubleAccumulatorParam$.html" title="class in org.apache.spark"><span class="strong">Next Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../index.html?org/apache/spark/SparkContext.html" target="_top">Frames</a></li>
<li><a href="SparkContext.html" target="_top">No Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../allclasses-noframe.html">All Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary: </li>
<li><a href="#nested_class_summary">Nested</a> | </li>
<li>Field | </li>
<li><a href="#constructor_summary">Constr</a> | </li>
<li><a href="#method_summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail: </li>
<li>Field | </li>
<li><a href="#constructor_detail">Constr</a> | </li>
<li><a href="#method_detail">Method</a></li>
</ul>
</div>
<a name="skip-navbar_top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="subTitle">org.apache.spark</div>
<h2 title="Class SparkContext" class="title">Class SparkContext</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li>org.apache.spark.SparkContext</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<dl>
<dt>All Implemented Interfaces:</dt>
<dd><a href="../../../org/apache/spark/Logging.html" title="interface in org.apache.spark">Logging</a></dd>
</dl>
<hr>
<br>
<pre>public class <span class="strong">SparkContext</span>
extends java.lang.Object
implements <a href="../../../org/apache/spark/Logging.html" title="interface in org.apache.spark">Logging</a></pre>
<div class="block">Main entry point for Spark functionality. A SparkContext represents the connection to a Spark
cluster, and can be used to create RDDs, accumulators and broadcast variables on that cluster.
<p>
Only one SparkContext may be active per JVM. You must <code>stop()</code> the active SparkContext before
creating a new one. This limitation may eventually be removed; see SPARK-2243 for more details.
<p>
param: config a Spark Config object describing the application configuration. Any settings in
this config overrides the default configs as well as system properties.</div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ======== NESTED CLASS SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="nested_class_summary">
<!-- -->
</a>
<h3>Nested Class Summary</h3>
<table class="overviewSummary" border="0" cellpadding="3" cellspacing="0" summary="Nested Class Summary table, listing nested classes, and an explanation">
<caption><span>Nested Classes</span><span class="tabEnd"> </span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Class and Description</th>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static class </code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.DoubleAccumulatorParam$.html" title="class in org.apache.spark">SparkContext.DoubleAccumulatorParam$</a></strong></code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static class </code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.FloatAccumulatorParam$.html" title="class in org.apache.spark">SparkContext.FloatAccumulatorParam$</a></strong></code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static class </code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.IntAccumulatorParam$.html" title="class in org.apache.spark">SparkContext.IntAccumulatorParam$</a></strong></code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static class </code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.LongAccumulatorParam$.html" title="class in org.apache.spark">SparkContext.LongAccumulatorParam$</a></strong></code> </td>
</tr>
</table>
</li>
</ul>
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor_summary">
<!-- -->
</a>
<h3>Constructor Summary</h3>
<table class="overviewSummary" border="0" cellpadding="3" cellspacing="0" summary="Constructor Summary table, listing constructors, and an explanation">
<caption><span>Constructors</span><span class="tabEnd"> </span></caption>
<tr>
<th class="colOne" scope="col">Constructor and Description</th>
</tr>
<tr class="altColor">
<td class="colOne"><code><strong><a href="../../../org/apache/spark/SparkContext.html#SparkContext()">SparkContext</a></strong>()</code>
<div class="block">Create a SparkContext that loads settings from system properties (for instance, when
launching with ./bin/spark-submit).</div>
</td>
</tr>
<tr class="rowColor">
<td class="colOne"><code><strong><a href="../../../org/apache/spark/SparkContext.html#SparkContext(org.apache.spark.SparkConf)">SparkContext</a></strong>(<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> config)</code> </td>
</tr>
<tr class="altColor">
<td class="colOne"><code><strong><a href="../../../org/apache/spark/SparkContext.html#SparkContext(org.apache.spark.SparkConf, scala.collection.Map)">SparkContext</a></strong>(<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> config,
scala.collection.Map<java.lang.String,scala.collection.Set<<a href="../../../org/apache/spark/scheduler/SplitInfo.html" title="class in org.apache.spark.scheduler">SplitInfo</a>>> preferredNodeLocationData)</code>
<div class="block">:: DeveloperApi ::
Alternative constructor for setting preferred locations where Spark will create executors.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colOne"><code><strong><a href="../../../org/apache/spark/SparkContext.html#SparkContext(java.lang.String, java.lang.String, org.apache.spark.SparkConf)">SparkContext</a></strong>(java.lang.String master,
java.lang.String appName,
<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> conf)</code>
<div class="block">Alternative constructor that allows setting common Spark properties directly</div>
</td>
</tr>
<tr class="altColor">
<td class="colOne"><code><strong><a href="../../../org/apache/spark/SparkContext.html#SparkContext(java.lang.String, java.lang.String, java.lang.String, scala.collection.Seq, scala.collection.Map, scala.collection.Map)">SparkContext</a></strong>(java.lang.String master,
java.lang.String appName,
java.lang.String sparkHome,
scala.collection.Seq<java.lang.String> jars,
scala.collection.Map<java.lang.String,java.lang.String> environment,
scala.collection.Map<java.lang.String,scala.collection.Set<<a href="../../../org/apache/spark/scheduler/SplitInfo.html" title="class in org.apache.spark.scheduler">SplitInfo</a>>> preferredNodeLocationData)</code>
<div class="block">Alternative constructor that allows setting common Spark properties directly</div>
</td>
</tr>
</table>
</li>
</ul>
<!-- ========== METHOD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="method_summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="overviewSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
<caption><span>Methods</span><span class="tabEnd"> </span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr class="altColor">
<td class="colFirst"><code><R,T> <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark">Accumulable</a><R,T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#accumulable(R, org.apache.spark.AccumulableParam)">accumulable</a></strong>(R initialValue,
<a href="../../../org/apache/spark/AccumulableParam.html" title="interface in org.apache.spark">AccumulableParam</a><R,T> param)</code>
<div class="block">Create an <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark"><code>Accumulable</code></a> shared variable, to which tasks can add values
with <code>+=</code>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><R,T> <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark">Accumulable</a><R,T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#accumulable(R, java.lang.String, org.apache.spark.AccumulableParam)">accumulable</a></strong>(R initialValue,
java.lang.String name,
<a href="../../../org/apache/spark/AccumulableParam.html" title="interface in org.apache.spark">AccumulableParam</a><R,T> param)</code>
<div class="block">Create an <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark"><code>Accumulable</code></a> shared variable, with a name for display in the
Spark UI.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><R,T> <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark">Accumulable</a><R,T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#accumulableCollection(R, scala.Function1, scala.reflect.ClassTag)">accumulableCollection</a></strong>(R initialValue,
scala.Function1<R,scala.collection.generic.Growable<T>> evidence$9,
scala.reflect.ClassTag<R> evidence$10)</code>
<div class="block">Create an accumulator from a "mutable collection" type.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/Accumulator.html" title="class in org.apache.spark">Accumulator</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#accumulator(T, org.apache.spark.AccumulatorParam)">accumulator</a></strong>(T initialValue,
<a href="../../../org/apache/spark/AccumulatorParam.html" title="interface in org.apache.spark">AccumulatorParam</a><T> param)</code>
<div class="block">Create an <a href="../../../org/apache/spark/Accumulator.html" title="class in org.apache.spark"><code>Accumulator</code></a> variable of a given type, which tasks can "add"
values to using the <code>+=</code> method.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/Accumulator.html" title="class in org.apache.spark">Accumulator</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#accumulator(T, java.lang.String, org.apache.spark.AccumulatorParam)">accumulator</a></strong>(T initialValue,
java.lang.String name,
<a href="../../../org/apache/spark/AccumulatorParam.html" title="interface in org.apache.spark">AccumulatorParam</a><T> param)</code>
<div class="block">Create an <a href="../../../org/apache/spark/Accumulator.html" title="class in org.apache.spark"><code>Accumulator</code></a> variable of a given type, with a name for display
in the Spark UI.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.collection.mutable.HashMap<java.lang.String,java.lang.Object></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#addedFiles()">addedFiles</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>scala.collection.mutable.HashMap<java.lang.String,java.lang.Object></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#addedJars()">addedJars</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#addFile(java.lang.String)">addFile</a></strong>(java.lang.String path)</code>
<div class="block">Add a file to be downloaded with this Spark job on every node.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#addFile(java.lang.String, boolean)">addFile</a></strong>(java.lang.String path,
boolean recursive)</code>
<div class="block">Add a file to be downloaded with this Spark job on every node.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#addJar(java.lang.String)">addJar</a></strong>(java.lang.String path)</code>
<div class="block">Adds a JAR dependency for all tasks to be executed on this SparkContext in the future.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#addSparkListener(org.apache.spark.scheduler.SparkListener)">addSparkListener</a></strong>(<a href="../../../org/apache/spark/scheduler/SparkListener.html" title="interface in org.apache.spark.scheduler">SparkListener</a> listener)</code>
<div class="block">:: DeveloperApi ::
Register a listener to receive up-calls from events that happen during execution.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.Option<java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#applicationAttemptId()">applicationAttemptId</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#applicationId()">applicationId</a></strong>()</code>
<div class="block">A unique identifier for the Spark application.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#appName()">appName</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<java.lang.String,<a href="../../../org/apache/spark/input/PortableDataStream.html" title="class in org.apache.spark.input">PortableDataStream</a>>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#binaryFiles(java.lang.String, int)">binaryFiles</a></strong>(java.lang.String path,
int minPartitions)</code>
<div class="block">:: Experimental ::</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><byte[]></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#binaryRecords(java.lang.String, int, org.apache.hadoop.conf.Configuration)">binaryRecords</a></strong>(java.lang.String path,
int recordLength,
org.apache.hadoop.conf.Configuration conf)</code>
<div class="block">:: Experimental ::</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static org.apache.spark.WritableConverter<java.lang.Object></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#booleanWritableConverter()">booleanWritableConverter</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static org.apache.hadoop.io.BooleanWritable</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#boolToBoolWritable(boolean)">boolToBoolWritable</a></strong>(boolean b)</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/broadcast/Broadcast.html" title="class in org.apache.spark.broadcast">Broadcast</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#broadcast(T, scala.reflect.ClassTag)">broadcast</a></strong>(T value,
scala.reflect.ClassTag<T> evidence$11)</code>
<div class="block">Broadcast a read-only variable to the cluster, returning a
<a href="../../../org/apache/spark/broadcast/Broadcast.html" title="class in org.apache.spark.broadcast"><code>Broadcast</code></a> object for reading it in distributed functions.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static org.apache.hadoop.io.BytesWritable</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#bytesToBytesWritable(byte[])">bytesToBytesWritable</a></strong>(byte[] aob)</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static org.apache.spark.WritableConverter<byte[]></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#bytesWritableConverter()">bytesWritableConverter</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#cancelAllJobs()">cancelAllJobs</a></strong>()</code>
<div class="block">Cancel all jobs that have been scheduled or are running.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#cancelJobGroup(java.lang.String)">cancelJobGroup</a></strong>(java.lang.String groupId)</code>
<div class="block">Cancel active jobs for the specified group.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.Option<java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#checkpointDir()">checkpointDir</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>protected <T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#checkpointFile(java.lang.String, scala.reflect.ClassTag)">checkpointFile</a></strong>(java.lang.String path,
scala.reflect.ClassTag<T> evidence$5)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#clearCallSite()">clearCallSite</a></strong>()</code>
<div class="block">Clear the thread-local property for overriding the call sites
of actions and RDDs.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#clearFiles()">clearFiles</a></strong>()</code>
<div class="block">Clear the job's list of files added by <code>addFile</code> so that they do not get downloaded to
any new nodes.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#clearJars()">clearJars</a></strong>()</code>
<div class="block">Clear the job's list of JARs added by <code>addJar</code> so that they do not get downloaded to
any new nodes.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#clearJobGroup()">clearJobGroup</a></strong>()</code>
<div class="block">Clear the current thread's job group ID and its description.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>int</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#defaultMinPartitions()">defaultMinPartitions</a></strong>()</code>
<div class="block">Default min number of partitions for Hadoop RDDs when not given by user
Notice that we use math.min so the "defaultMinPartitions" cannot be higher than 2.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>int</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#defaultMinSplits()">defaultMinSplits</a></strong>()</code>
<div class="block">Default min number of partitions for Hadoop RDDs when not given by user</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>int</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#defaultParallelism()">defaultParallelism</a></strong>()</code>
<div class="block">Default level of parallelism to use when not given by user (e.g.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static <a href="../../../org/apache/spark/rdd/DoubleRDDFunctions.html" title="class in org.apache.spark.rdd">DoubleRDDFunctions</a></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#doubleRDDToDoubleRDDFunctions(org.apache.spark.rdd.RDD)">doubleRDDToDoubleRDDFunctions</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><java.lang.Object> rdd)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static org.apache.hadoop.io.DoubleWritable</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#doubleToDoubleWritable(double)">doubleToDoubleWritable</a></strong>(double d)</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static org.apache.spark.WritableConverter<java.lang.Object></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#doubleWritableConverter()">doubleWritableConverter</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#DRIVER_IDENTIFIER()">DRIVER_IDENTIFIER</a></strong>()</code>
<div class="block">Executor id for the driver.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T> <any></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#emptyRDD(scala.reflect.ClassTag)">emptyRDD</a></strong>(scala.reflect.ClassTag<T> evidence$8)</code>
<div class="block">Get an RDD that has no partitions or elements.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.collection.mutable.HashMap<java.lang.String,java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#executorEnvs()">executorEnvs</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#externalBlockStoreFolderName()">externalBlockStoreFolderName</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.collection.Seq<java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#files()">files</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static org.apache.hadoop.io.FloatWritable</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#floatToFloatWritable(float)">floatToFloatWritable</a></strong>(float f)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static org.apache.spark.WritableConverter<java.lang.Object></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#floatWritableConverter()">floatWritableConverter</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>scala.collection.Seq<org.apache.spark.scheduler.Schedulable></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getAllPools()">getAllPools</a></strong>()</code>
<div class="block">:: DeveloperApi ::
Return pools for fair scheduler</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.Option<java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getCheckpointDir()">getCheckpointDir</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getConf()">getConf</a></strong>()</code>
<div class="block">Return a copy of this SparkContext's configuration.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.collection.Map<java.lang.String,scala.Tuple2<java.lang.Object,java.lang.Object>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getExecutorMemoryStatus()">getExecutorMemoryStatus</a></strong>()</code>
<div class="block">Return a map from the slave to the max memory available for caching and the remaining
memory available for caching.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/storage/StorageStatus.html" title="class in org.apache.spark.storage">StorageStatus</a>[]</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getExecutorStorageStatus()">getExecutorStorageStatus</a></strong>()</code>
<div class="block">:: DeveloperApi ::
Return information about blocks stored in all of the slaves</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getLocalProperty(java.lang.String)">getLocalProperty</a></strong>(java.lang.String key)</code>
<div class="block">Get a local property set in this thread, or null if it is missing.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static <a href="../../../org/apache/spark/SparkContext.html" title="class in org.apache.spark">SparkContext</a></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getOrCreate()">getOrCreate</a></strong>()</code>
<div class="block">This function may be used to get or instantiate a SparkContext and register it as a
singleton object.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static <a href="../../../org/apache/spark/SparkContext.html" title="class in org.apache.spark">SparkContext</a></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getOrCreate(org.apache.spark.SparkConf)">getOrCreate</a></strong>(<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> config)</code>
<div class="block">This function may be used to get or instantiate a SparkContext and register it as a
singleton object.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>scala.collection.Map<java.lang.Object,<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><?>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getPersistentRDDs()">getPersistentRDDs</a></strong>()</code>
<div class="block">Returns an immutable map of RDDs that have marked themselves as persistent via cache() call.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.Option<org.apache.spark.scheduler.Schedulable></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getPoolForName(java.lang.String)">getPoolForName</a></strong>(java.lang.String pool)</code>
<div class="block">:: DeveloperApi ::
Return the pool associated with the given name, if one exists</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/storage/RDDInfo.html" title="class in org.apache.spark.storage">RDDInfo</a>[]</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getRDDStorageInfo()">getRDDStorageInfo</a></strong>()</code>
<div class="block">:: DeveloperApi ::
Return information about what RDDs are cached, if they are in mem or on disk, how much space
they take, etc.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.Enumeration.Value</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#getSchedulingMode()">getSchedulingMode</a></strong>()</code>
<div class="block">Return current scheduling mode</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>org.apache.hadoop.conf.Configuration</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#hadoopConfiguration()">hadoopConfiguration</a></strong>()</code>
<div class="block">A default Hadoop Configuration for the Hadoop code (e.g.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#hadoopFile(java.lang.String, java.lang.Class, java.lang.Class, java.lang.Class, int)">hadoopFile</a></strong>(java.lang.String path,
java.lang.Class<? extends org.apache.hadoop.mapred.InputFormat<K,V>> inputFormatClass,
java.lang.Class<K> keyClass,
java.lang.Class<V> valueClass,
int minPartitions)</code>
<div class="block">Get an RDD for a Hadoop file with an arbitrary InputFormat</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><K,V,F extends org.apache.hadoop.mapred.InputFormat<K,V>> <br><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#hadoopFile(java.lang.String, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.reflect.ClassTag)">hadoopFile</a></strong>(java.lang.String path,
scala.reflect.ClassTag<K> km,
scala.reflect.ClassTag<V> vm,
scala.reflect.ClassTag<F> fm)</code>
<div class="block">Smarter version of hadoopFile() that uses class tags to figure out the classes of keys,
values and the InputFormat so that users don't need to pass them directly.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><K,V,F extends org.apache.hadoop.mapred.InputFormat<K,V>> <br><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#hadoopFile(java.lang.String, int, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.reflect.ClassTag)">hadoopFile</a></strong>(java.lang.String path,
int minPartitions,
scala.reflect.ClassTag<K> km,
scala.reflect.ClassTag<V> vm,
scala.reflect.ClassTag<F> fm)</code>
<div class="block">Smarter version of hadoopFile() that uses class tags to figure out the classes of keys,
values and the InputFormat so that users don't need to pass them directly.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#hadoopRDD(org.apache.hadoop.mapred.JobConf, java.lang.Class, java.lang.Class, java.lang.Class, int)">hadoopRDD</a></strong>(org.apache.hadoop.mapred.JobConf conf,
java.lang.Class<? extends org.apache.hadoop.mapred.InputFormat<K,V>> inputFormatClass,
java.lang.Class<K> keyClass,
java.lang.Class<V> valueClass,
int minPartitions)</code>
<div class="block">Get an RDD for a Hadoop-readable dataset from a Hadoop JobConf given its InputFormat and other
necessary info (e.g.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#initLocalProperties()">initLocalProperties</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static org.apache.hadoop.io.IntWritable</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#intToIntWritable(int)">intToIntWritable</a></strong>(int i)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static org.apache.spark.WritableConverter<java.lang.Object></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#intWritableConverter()">intWritableConverter</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>boolean</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#isLocal()">isLocal</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static scala.Option<java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#jarOfClass(java.lang.Class)">jarOfClass</a></strong>(java.lang.Class<?> cls)</code>
<div class="block">Find the JAR from which a given class was loaded, to make it easy for users to pass
their JARs to SparkContext.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static scala.Option<java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#jarOfObject(java.lang.Object)">jarOfObject</a></strong>(java.lang.Object obj)</code>
<div class="block">Find the JAR that contains the class of a particular object, to make it easy for users
to pass their JARs to SparkContext.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>scala.collection.Seq<java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#jars()">jars</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>boolean</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#killExecutor(java.lang.String)">killExecutor</a></strong>(java.lang.String executorId)</code>
<div class="block">:: DeveloperApi ::
Request that the cluster manager kill the specified executor.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>boolean</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#killExecutors(scala.collection.Seq)">killExecutors</a></strong>(scala.collection.Seq<java.lang.String> executorIds)</code>
<div class="block">:: DeveloperApi ::
Request that the cluster manager kill the specified executors.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#LEGACY_DRIVER_IDENTIFIER()">LEGACY_DRIVER_IDENTIFIER</a></strong>()</code>
<div class="block">Legacy version of DRIVER_IDENTIFIER, retained for backwards-compatibility.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>org.apache.spark.scheduler.LiveListenerBus</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#listenerBus()">listenerBus</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>protected java.lang.InheritableThreadLocal<java.util.Properties></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#localProperties()">localProperties</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static org.apache.hadoop.io.LongWritable</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#longToLongWritable(long)">longToLongWritable</a></strong>(long l)</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static org.apache.spark.WritableConverter<java.lang.Object></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#longWritableConverter()">longWritableConverter</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#makeRDD(scala.collection.Seq, int, scala.reflect.ClassTag)">makeRDD</a></strong>(scala.collection.Seq<T> seq,
int numSlices,
scala.reflect.ClassTag<T> evidence$2)</code>
<div class="block">Distribute a local Scala collection to form an RDD.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#makeRDD(scala.collection.Seq, scala.reflect.ClassTag)">makeRDD</a></strong>(scala.collection.Seq<scala.Tuple2<T,scala.collection.Seq<java.lang.String>>> seq,
scala.reflect.ClassTag<T> evidence$3)</code>
<div class="block">Distribute a local Scala collection to form an RDD, with one or more
location preferences (hostnames of Spark nodes) for each object.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#master()">master</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>org.apache.spark.metrics.MetricsSystem</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#metricsSystem()">metricsSystem</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><K,V,F extends org.apache.hadoop.mapreduce.InputFormat<K,V>> <br><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#newAPIHadoopFile(java.lang.String, java.lang.Class, java.lang.Class, java.lang.Class, org.apache.hadoop.conf.Configuration)">newAPIHadoopFile</a></strong>(java.lang.String path,
java.lang.Class<F> fClass,
java.lang.Class<K> kClass,
java.lang.Class<V> vClass,
org.apache.hadoop.conf.Configuration conf)</code>
<div class="block">Get an RDD for a given Hadoop file with an arbitrary new API InputFormat
and extra configuration options to pass to the input format.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><K,V,F extends org.apache.hadoop.mapreduce.InputFormat<K,V>> <br><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#newAPIHadoopFile(java.lang.String, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.reflect.ClassTag)">newAPIHadoopFile</a></strong>(java.lang.String path,
scala.reflect.ClassTag<K> km,
scala.reflect.ClassTag<V> vm,
scala.reflect.ClassTag<F> fm)</code>
<div class="block">Get an RDD for a Hadoop file with an arbitrary new API InputFormat.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><K,V,F extends org.apache.hadoop.mapreduce.InputFormat<K,V>> <br><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#newAPIHadoopRDD(org.apache.hadoop.conf.Configuration, java.lang.Class, java.lang.Class, java.lang.Class)">newAPIHadoopRDD</a></strong>(org.apache.hadoop.conf.Configuration conf,
java.lang.Class<F> fClass,
java.lang.Class<K> kClass,
java.lang.Class<V> vClass)</code>
<div class="block">Get an RDD for a given Hadoop file with an arbitrary new API InputFormat
and extra configuration options to pass to the input format.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static <T> <a href="../../../org/apache/spark/rdd/DoubleRDDFunctions.html" title="class in org.apache.spark.rdd">DoubleRDDFunctions</a></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#numericRDDToDoubleRDDFunctions(org.apache.spark.rdd.RDD, scala.math.Numeric)">numericRDDToDoubleRDDFunctions</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.math.Numeric<T> num)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#objectFile(java.lang.String, int, scala.reflect.ClassTag)">objectFile</a></strong>(java.lang.String path,
int minPartitions,
scala.reflect.ClassTag<T> evidence$4)</code>
<div class="block">Load an RDD saved as a SequenceFile containing serialized objects, with NullWritable keys and
BytesWritable values that contain a serialized partition.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#parallelize(scala.collection.Seq, int, scala.reflect.ClassTag)">parallelize</a></strong>(scala.collection.Seq<T> seq,
int numSlices,
scala.reflect.ClassTag<T> evidence$1)</code>
<div class="block">Distribute a local Scala collection to form an RDD.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><any></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#persistentRdds()">persistentRdds</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>scala.collection.Map<java.lang.String,scala.collection.Set<<a href="../../../org/apache/spark/scheduler/SplitInfo.html" title="class in org.apache.spark.scheduler">SplitInfo</a>>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#preferredNodeLocationData()">preferredNodeLocationData</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><java.lang.Object></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#range(long, long, long, int)">range</a></strong>(long start,
long end,
long step,
int numSlices)</code>
<div class="block">Creates a new RDD[Long] containing elements from <code>start</code> to <code>end</code>(exclusive), increased by
<code>step</code> every element.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#RDD_SCOPE_KEY()">RDD_SCOPE_KEY</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#RDD_SCOPE_NO_OVERRIDE_KEY()">RDD_SCOPE_NO_OVERRIDE_KEY</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static <T> <a href="../../../org/apache/spark/rdd/AsyncRDDActions.html" title="class in org.apache.spark.rdd">AsyncRDDActions</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#rddToAsyncRDDActions(org.apache.spark.rdd.RDD, scala.reflect.ClassTag)">rddToAsyncRDDActions</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.reflect.ClassTag<T> evidence$22)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static <K,V> <a href="../../../org/apache/spark/rdd/OrderedRDDFunctions.html" title="class in org.apache.spark.rdd">OrderedRDDFunctions</a><K,V,scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#rddToOrderedRDDFunctions(org.apache.spark.rdd.RDD, scala.math.Ordering, scala.reflect.ClassTag, scala.reflect.ClassTag)">rddToOrderedRDDFunctions</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> rdd,
scala.math.Ordering<K> evidence$27,
scala.reflect.ClassTag<K> evidence$28,
scala.reflect.ClassTag<V> evidence$29)</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static <K,V> <a href="../../../org/apache/spark/rdd/PairRDDFunctions.html" title="class in org.apache.spark.rdd">PairRDDFunctions</a><K,V></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#rddToPairRDDFunctions(org.apache.spark.rdd.RDD, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.math.Ordering)">rddToPairRDDFunctions</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> rdd,
scala.reflect.ClassTag<K> kt,
scala.reflect.ClassTag<V> vt,
scala.math.Ordering<K> ord)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static <K,V> <a href="../../../org/apache/spark/rdd/SequenceFileRDDFunctions.html" title="class in org.apache.spark.rdd">SequenceFileRDDFunctions</a><K,V></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#rddToSequenceFileRDDFunctions(org.apache.spark.rdd.RDD, scala.Function1, scala.reflect.ClassTag, scala.Function1, scala.reflect.ClassTag)">rddToSequenceFileRDDFunctions</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> rdd,
scala.Function1<K,org.apache.hadoop.io.Writable> evidence$23,
scala.reflect.ClassTag<K> evidence$24,
scala.Function1<V,org.apache.hadoop.io.Writable> evidence$25,
scala.reflect.ClassTag<V> evidence$26)</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>boolean</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#requestExecutors(int)">requestExecutors</a></strong>(int numAdditionalExecutors)</code>
<div class="block">:: DeveloperApi ::
Request an additional number of executors from the cluster manager.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T,U,R> <a href="../../../org/apache/spark/partial/PartialResult.html" title="class in org.apache.spark.partial">PartialResult</a><R></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runApproximateJob(org.apache.spark.rdd.RDD, scala.Function2, , long)">runApproximateJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
<any> evaluator,
long timeout)</code>
<div class="block">:: DeveloperApi ::
Run a job that can return approximate results.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T,U> java.lang.Object</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function1, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> func,
scala.reflect.ClassTag<U> evidence$19)</code>
<div class="block">Run a job on all partitions in an RDD and return the results in an array.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T,U> void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function1, scala.Function2, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> processPartition,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.reflect.ClassTag<U> evidence$21)</code>
<div class="block">Run a job on all partitions in an RDD and pass the results to a handler function.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T,U> java.lang.Object</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function1, scala.collection.Seq, boolean, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
boolean allowLocal,
scala.reflect.ClassTag<U> evidence$17)</code>
<div class="block">Run a job on a given set of partitions of an RDD, but take a function of type
<code>Iterator[T] => U</code> instead of <code>(TaskContext, Iterator[T]) => U</code>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T,U> java.lang.Object</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function1, scala.collection.Seq, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
scala.reflect.ClassTag<U> evidence$14)</code>
<div class="block">Run a job on a given set of partitions of an RDD, but take a function of type
<code>Iterator[T] => U</code> instead of <code>(TaskContext, Iterator[T]) => U</code>.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T,U> java.lang.Object</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.reflect.ClassTag<U> evidence$18)</code>
<div class="block">Run a job on all partitions in an RDD and return the results in an array.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T,U> void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.Function2, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> processPartition,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.reflect.ClassTag<U> evidence$20)</code>
<div class="block">Run a job on all partitions in an RDD and pass the results to a handler function.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T,U> java.lang.Object</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.collection.Seq, boolean, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
boolean allowLocal,
scala.reflect.ClassTag<U> evidence$16)</code>
<div class="block">Run a function on a given set of partitions in an RDD and return the results as an array.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T,U> void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.collection.Seq, boolean, scala.Function2, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
boolean allowLocal,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.reflect.ClassTag<U> evidence$15)</code>
<div class="block">Run a function on a given set of partitions in an RDD and pass the results to the given
handler function.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T,U> java.lang.Object</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.collection.Seq, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
scala.reflect.ClassTag<U> evidence$13)</code>
<div class="block">Run a function on a given set of partitions in an RDD and return the results as an array.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T,U> void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.collection.Seq, scala.Function2, scala.reflect.ClassTag)">runJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.reflect.ClassTag<U> evidence$12)</code>
<div class="block">Run a function on a given set of partitions in an RDD and pass the results to the given
handler function.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#sequenceFile(java.lang.String, java.lang.Class, java.lang.Class)">sequenceFile</a></strong>(java.lang.String path,
java.lang.Class<K> keyClass,
java.lang.Class<V> valueClass)</code>
<div class="block">Get an RDD for a Hadoop SequenceFile with given key and value types.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#sequenceFile(java.lang.String, java.lang.Class, java.lang.Class, int)">sequenceFile</a></strong>(java.lang.String path,
java.lang.Class<K> keyClass,
java.lang.Class<V> valueClass,
int minPartitions)</code>
<div class="block">Get an RDD for a Hadoop SequenceFile with given key and value types.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#sequenceFile(java.lang.String, int, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.Function0, scala.Function0)">sequenceFile</a></strong>(java.lang.String path,
int minPartitions,
scala.reflect.ClassTag<K> km,
scala.reflect.ClassTag<V> vm,
scala.Function0<org.apache.spark.WritableConverter<K>> kcf,
scala.Function0<org.apache.spark.WritableConverter<V>> vcf)</code>
<div class="block">Version of sequenceFile() for types implicitly convertible to Writables through a
WritableConverter.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#setCallSite(java.lang.String)">setCallSite</a></strong>(java.lang.String shortCallSite)</code>
<div class="block">Set the thread-local property for overriding the call sites
of actions and RDDs.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#setCheckpointDir(java.lang.String)">setCheckpointDir</a></strong>(java.lang.String directory)</code>
<div class="block">Set the directory under which RDDs are going to be checkpointed.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#setJobDescription(java.lang.String)">setJobDescription</a></strong>(java.lang.String value)</code>
<div class="block">Set a human readable description of the current job.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#setJobGroup(java.lang.String, java.lang.String, boolean)">setJobGroup</a></strong>(java.lang.String groupId,
java.lang.String description,
boolean interruptOnCancel)</code>
<div class="block">Assigns a group ID to all the jobs started by this thread until the group ID is set to a
different value or cleared.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#setLocalProperty(java.lang.String, java.lang.String)">setLocalProperty</a></strong>(java.lang.String key,
java.lang.String value)</code>
<div class="block">Set a local property that affects jobs submitted from this thread, such as the
Spark fair scheduler pool.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#setLogLevel(java.lang.String)">setLogLevel</a></strong>(java.lang.String logLevel)</code>
<div class="block">Control our logLevel.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#SPARK_JOB_DESCRIPTION()">SPARK_JOB_DESCRIPTION</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#SPARK_JOB_GROUP_ID()">SPARK_JOB_GROUP_ID</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#SPARK_JOB_INTERRUPT_ON_CANCEL()">SPARK_JOB_INTERRUPT_ON_CANCEL</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#sparkUser()">sparkUser</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>long</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#startTime()">startTime</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/SparkStatusTracker.html" title="class in org.apache.spark">SparkStatusTracker</a></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#statusTracker()">statusTracker</a></strong>()</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#stop()">stop</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static org.apache.hadoop.io.Text</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#stringToText(java.lang.String)">stringToText</a></strong>(java.lang.String s)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static org.apache.spark.WritableConverter<java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#stringWritableConverter()">stringWritableConverter</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T,U,R> <a href="../../../org/apache/spark/SimpleFutureAction.html" title="class in org.apache.spark">SimpleFutureAction</a><R></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#submitJob(org.apache.spark.rdd.RDD, scala.Function1, scala.collection.Seq, scala.Function2, scala.Function0)">submitJob</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> processPartition,
scala.collection.Seq<java.lang.Object> partitions,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.Function0<R> resultFunc)</code>
<div class="block">:: Experimental ::
Submit a job for execution and return a FutureJob holding the result.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#tachyonFolderName()">tachyonFolderName</a></strong>()</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><java.lang.String></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#textFile(java.lang.String, int)">textFile</a></strong>(java.lang.String path,
int minPartitions)</code>
<div class="block">Read a text file from HDFS, a local file system (available on all nodes), or any
Hadoop-supported file system URI, and return it as an RDD of Strings.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#union(org.apache.spark.rdd.RDD, scala.collection.Seq, scala.reflect.ClassTag)">union</a></strong>(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> first,
scala.collection.Seq<<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T>> rest,
scala.reflect.ClassTag<T> evidence$7)</code>
<div class="block">Build the union of a list of RDDs passed as variable-length arguments.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#union(scala.collection.Seq, scala.reflect.ClassTag)">union</a></strong>(scala.collection.Seq<<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T>> rdds,
scala.reflect.ClassTag<T> evidence$6)</code>
<div class="block">Build the union of a list of RDDs.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>java.lang.String</code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#version()">version</a></strong>()</code>
<div class="block">The version of Spark on which this application is running.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code><a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<java.lang.String,java.lang.String>></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#wholeTextFiles(java.lang.String, int)">wholeTextFiles</a></strong>(java.lang.String path,
int minPartitions)</code>
<div class="block">Read a directory of text files from HDFS, a local file system (available on all nodes), or any
Hadoop-supported file system URI.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static <T extends org.apache.hadoop.io.Writable> <br>org.apache.spark.WritableConverter<T></code></td>
<td class="colLast"><code><strong><a href="../../../org/apache/spark/SparkContext.html#writableWritableConverter()">writableWritableConverter</a></strong>()</code> </td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a name="methods_inherited_from_class_java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
<ul class="blockList">
<li class="blockList"><a name="methods_inherited_from_class_org.apache.spark.Logging">
<!-- -->
</a>
<h3>Methods inherited from interface org.apache.spark.<a href="../../../org/apache/spark/Logging.html" title="interface in org.apache.spark">Logging</a></h3>
<code><a href="../../../org/apache/spark/Logging.html#initializeIfNecessary()">initializeIfNecessary</a>, <a href="../../../org/apache/spark/Logging.html#initializeLogging()">initializeLogging</a>, <a href="../../../org/apache/spark/Logging.html#isTraceEnabled()">isTraceEnabled</a>, <a href="../../../org/apache/spark/Logging.html#log_()">log_</a>, <a href="../../../org/apache/spark/Logging.html#log()">log</a>, <a href="../../../org/apache/spark/Logging.html#logDebug(scala.Function0)">logDebug</a>, <a href="../../../org/apache/spark/Logging.html#logDebug(scala.Function0, java.lang.Throwable)">logDebug</a>, <a href="../../../org/apache/spark/Logging.html#logError(scala.Function0)">logError</a>, <a href="../../../org/apache/spark/Logging.html#logError(scala.Function0, java.lang.Throwable)">logError</a>, <a href="../../../org/apache/spark/Logging.html#logInfo(scala.Function0)">logInfo</a>, <a href="../../../org/apache/spark/Logging.html#logInfo(scala.Function0, java.lang.Throwable)">logInfo</a>, <a href="../../../org/apache/spark/Logging.html#logName()">logName</a>, <a href="../../../org/apache/spark/Logging.html#logTrace(scala.Function0)">logTrace</a>, <a href="../../../org/apache/spark/Logging.html#logTrace(scala.Function0, java.lang.Throwable)">logTrace</a>, <a href="../../../org/apache/spark/Logging.html#logWarning(scala.Function0)">logWarning</a>, <a href="../../../org/apache/spark/Logging.html#logWarning(scala.Function0, java.lang.Throwable)">logWarning</a></code></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor_detail">
<!-- -->
</a>
<h3>Constructor Detail</h3>
<a name="SparkContext(org.apache.spark.SparkConf)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>SparkContext</h4>
<pre>public SparkContext(<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> config)</pre>
</li>
</ul>
<a name="SparkContext()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>SparkContext</h4>
<pre>public SparkContext()</pre>
<div class="block">Create a SparkContext that loads settings from system properties (for instance, when
launching with ./bin/spark-submit).</div>
</li>
</ul>
<a name="SparkContext(org.apache.spark.SparkConf, scala.collection.Map)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>SparkContext</h4>
<pre>public SparkContext(<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> config,
scala.collection.Map<java.lang.String,scala.collection.Set<<a href="../../../org/apache/spark/scheduler/SplitInfo.html" title="class in org.apache.spark.scheduler">SplitInfo</a>>> preferredNodeLocationData)</pre>
<div class="block">:: DeveloperApi ::
Alternative constructor for setting preferred locations where Spark will create executors.
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>config</code> - a <a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark"><code>SparkConf</code></a> object specifying other Spark parameters</dd><dd><code>preferredNodeLocationData</code> - used in YARN mode to select nodes to launch containers on.
Can be generated using <code>org.apache.spark.scheduler.InputFormatInfo.computePreferredLocations</code>
from a list of input files or InputFormats for the application.</dd></dl>
</li>
</ul>
<a name="SparkContext(java.lang.String, java.lang.String, org.apache.spark.SparkConf)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>SparkContext</h4>
<pre>public SparkContext(java.lang.String master,
java.lang.String appName,
<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> conf)</pre>
<div class="block">Alternative constructor that allows setting common Spark properties directly
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>master</code> - Cluster URL to connect to (e.g. mesos://host:port, spark://host:port, local[4]).</dd><dd><code>appName</code> - A name for your application, to display on the cluster web UI</dd><dd><code>conf</code> - a <a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark"><code>SparkConf</code></a> object specifying other Spark parameters</dd></dl>
</li>
</ul>
<a name="SparkContext(java.lang.String, java.lang.String, java.lang.String, scala.collection.Seq, scala.collection.Map, scala.collection.Map)">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>SparkContext</h4>
<pre>public SparkContext(java.lang.String master,
java.lang.String appName,
java.lang.String sparkHome,
scala.collection.Seq<java.lang.String> jars,
scala.collection.Map<java.lang.String,java.lang.String> environment,
scala.collection.Map<java.lang.String,scala.collection.Set<<a href="../../../org/apache/spark/scheduler/SplitInfo.html" title="class in org.apache.spark.scheduler">SplitInfo</a>>> preferredNodeLocationData)</pre>
<div class="block">Alternative constructor that allows setting common Spark properties directly
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>master</code> - Cluster URL to connect to (e.g. mesos://host:port, spark://host:port, local[4]).</dd><dd><code>appName</code> - A name for your application, to display on the cluster web UI.</dd><dd><code>sparkHome</code> - Location where Spark is installed on cluster nodes.</dd><dd><code>jars</code> - Collection of JARs to send to the cluster. These can be paths on the local file
system or HDFS, HTTP, HTTPS, or FTP URLs.</dd><dd><code>environment</code> - Environment variables to set on worker nodes.</dd><dd><code>preferredNodeLocationData</code> - used in YARN mode to select nodes to launch containers on.
Can be generated using <code>org.apache.spark.scheduler.InputFormatInfo.computePreferredLocations</code>
from a list of input files or InputFormats for the application.</dd></dl>
</li>
</ul>
</li>
</ul>
<!-- ============ METHOD DETAIL ========== -->
<ul class="blockList">
<li class="blockList"><a name="method_detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a name="getOrCreate(org.apache.spark.SparkConf)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getOrCreate</h4>
<pre>public static <a href="../../../org/apache/spark/SparkContext.html" title="class in org.apache.spark">SparkContext</a> getOrCreate(<a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> config)</pre>
<div class="block">This function may be used to get or instantiate a SparkContext and register it as a
singleton object. Because we can only have one active SparkContext per JVM,
this is useful when applications may wish to share a SparkContext.
<p>
Note: This function cannot be used to create multiple SparkContext instances
even if multiple contexts are allowed.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>config</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="getOrCreate()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getOrCreate</h4>
<pre>public static <a href="../../../org/apache/spark/SparkContext.html" title="class in org.apache.spark">SparkContext</a> getOrCreate()</pre>
<div class="block">This function may be used to get or instantiate a SparkContext and register it as a
singleton object. Because we can only have one active SparkContext per JVM,
this is useful when applications may wish to share a SparkContext.
<p>
This method allows not passing a SparkConf (useful if just retrieving).
<p>
Note: This function cannot be used to create multiple SparkContext instances
even if multiple contexts are allowed.</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="SPARK_JOB_DESCRIPTION()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>SPARK_JOB_DESCRIPTION</h4>
<pre>public static java.lang.String SPARK_JOB_DESCRIPTION()</pre>
</li>
</ul>
<a name="SPARK_JOB_GROUP_ID()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>SPARK_JOB_GROUP_ID</h4>
<pre>public static java.lang.String SPARK_JOB_GROUP_ID()</pre>
</li>
</ul>
<a name="SPARK_JOB_INTERRUPT_ON_CANCEL()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>SPARK_JOB_INTERRUPT_ON_CANCEL</h4>
<pre>public static java.lang.String SPARK_JOB_INTERRUPT_ON_CANCEL()</pre>
</li>
</ul>
<a name="RDD_SCOPE_KEY()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>RDD_SCOPE_KEY</h4>
<pre>public static java.lang.String RDD_SCOPE_KEY()</pre>
</li>
</ul>
<a name="RDD_SCOPE_NO_OVERRIDE_KEY()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>RDD_SCOPE_NO_OVERRIDE_KEY</h4>
<pre>public static java.lang.String RDD_SCOPE_NO_OVERRIDE_KEY()</pre>
</li>
</ul>
<a name="DRIVER_IDENTIFIER()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>DRIVER_IDENTIFIER</h4>
<pre>public static java.lang.String DRIVER_IDENTIFIER()</pre>
<div class="block">Executor id for the driver. In earlier versions of Spark, this was <code><driver></code>, but this was
changed to <code>driver</code> because the angle brackets caused escaping issues in URLs and XML (see
SPARK-6716 for more details).</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="LEGACY_DRIVER_IDENTIFIER()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>LEGACY_DRIVER_IDENTIFIER</h4>
<pre>public static java.lang.String LEGACY_DRIVER_IDENTIFIER()</pre>
<div class="block">Legacy version of DRIVER_IDENTIFIER, retained for backwards-compatibility.</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="rddToPairRDDFunctions(org.apache.spark.rdd.RDD, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.math.Ordering)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>rddToPairRDDFunctions</h4>
<pre>public static <K,V> <a href="../../../org/apache/spark/rdd/PairRDDFunctions.html" title="class in org.apache.spark.rdd">PairRDDFunctions</a><K,V> rddToPairRDDFunctions(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> rdd,
scala.reflect.ClassTag<K> kt,
scala.reflect.ClassTag<V> vt,
scala.math.Ordering<K> ord)</pre>
</li>
</ul>
<a name="rddToAsyncRDDActions(org.apache.spark.rdd.RDD, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>rddToAsyncRDDActions</h4>
<pre>public static <T> <a href="../../../org/apache/spark/rdd/AsyncRDDActions.html" title="class in org.apache.spark.rdd">AsyncRDDActions</a><T> rddToAsyncRDDActions(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.reflect.ClassTag<T> evidence$22)</pre>
</li>
</ul>
<a name="rddToSequenceFileRDDFunctions(org.apache.spark.rdd.RDD, scala.Function1, scala.reflect.ClassTag, scala.Function1, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>rddToSequenceFileRDDFunctions</h4>
<pre>public static <K,V> <a href="../../../org/apache/spark/rdd/SequenceFileRDDFunctions.html" title="class in org.apache.spark.rdd">SequenceFileRDDFunctions</a><K,V> rddToSequenceFileRDDFunctions(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> rdd,
scala.Function1<K,org.apache.hadoop.io.Writable> evidence$23,
scala.reflect.ClassTag<K> evidence$24,
scala.Function1<V,org.apache.hadoop.io.Writable> evidence$25,
scala.reflect.ClassTag<V> evidence$26)</pre>
</li>
</ul>
<a name="rddToOrderedRDDFunctions(org.apache.spark.rdd.RDD, scala.math.Ordering, scala.reflect.ClassTag, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>rddToOrderedRDDFunctions</h4>
<pre>public static <K,V> <a href="../../../org/apache/spark/rdd/OrderedRDDFunctions.html" title="class in org.apache.spark.rdd">OrderedRDDFunctions</a><K,V,scala.Tuple2<K,V>> rddToOrderedRDDFunctions(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> rdd,
scala.math.Ordering<K> evidence$27,
scala.reflect.ClassTag<K> evidence$28,
scala.reflect.ClassTag<V> evidence$29)</pre>
</li>
</ul>
<a name="doubleRDDToDoubleRDDFunctions(org.apache.spark.rdd.RDD)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>doubleRDDToDoubleRDDFunctions</h4>
<pre>public static <a href="../../../org/apache/spark/rdd/DoubleRDDFunctions.html" title="class in org.apache.spark.rdd">DoubleRDDFunctions</a> doubleRDDToDoubleRDDFunctions(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><java.lang.Object> rdd)</pre>
</li>
</ul>
<a name="numericRDDToDoubleRDDFunctions(org.apache.spark.rdd.RDD, scala.math.Numeric)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>numericRDDToDoubleRDDFunctions</h4>
<pre>public static <T> <a href="../../../org/apache/spark/rdd/DoubleRDDFunctions.html" title="class in org.apache.spark.rdd">DoubleRDDFunctions</a> numericRDDToDoubleRDDFunctions(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.math.Numeric<T> num)</pre>
</li>
</ul>
<a name="intToIntWritable(int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>intToIntWritable</h4>
<pre>public static org.apache.hadoop.io.IntWritable intToIntWritable(int i)</pre>
</li>
</ul>
<a name="longToLongWritable(long)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>longToLongWritable</h4>
<pre>public static org.apache.hadoop.io.LongWritable longToLongWritable(long l)</pre>
</li>
</ul>
<a name="floatToFloatWritable(float)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>floatToFloatWritable</h4>
<pre>public static org.apache.hadoop.io.FloatWritable floatToFloatWritable(float f)</pre>
</li>
</ul>
<a name="doubleToDoubleWritable(double)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>doubleToDoubleWritable</h4>
<pre>public static org.apache.hadoop.io.DoubleWritable doubleToDoubleWritable(double d)</pre>
</li>
</ul>
<a name="boolToBoolWritable(boolean)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>boolToBoolWritable</h4>
<pre>public static org.apache.hadoop.io.BooleanWritable boolToBoolWritable(boolean b)</pre>
</li>
</ul>
<a name="bytesToBytesWritable(byte[])">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>bytesToBytesWritable</h4>
<pre>public static org.apache.hadoop.io.BytesWritable bytesToBytesWritable(byte[] aob)</pre>
</li>
</ul>
<a name="stringToText(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>stringToText</h4>
<pre>public static org.apache.hadoop.io.Text stringToText(java.lang.String s)</pre>
</li>
</ul>
<a name="intWritableConverter()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>intWritableConverter</h4>
<pre>public static org.apache.spark.WritableConverter<java.lang.Object> intWritableConverter()</pre>
</li>
</ul>
<a name="longWritableConverter()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>longWritableConverter</h4>
<pre>public static org.apache.spark.WritableConverter<java.lang.Object> longWritableConverter()</pre>
</li>
</ul>
<a name="doubleWritableConverter()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>doubleWritableConverter</h4>
<pre>public static org.apache.spark.WritableConverter<java.lang.Object> doubleWritableConverter()</pre>
</li>
</ul>
<a name="floatWritableConverter()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>floatWritableConverter</h4>
<pre>public static org.apache.spark.WritableConverter<java.lang.Object> floatWritableConverter()</pre>
</li>
</ul>
<a name="booleanWritableConverter()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>booleanWritableConverter</h4>
<pre>public static org.apache.spark.WritableConverter<java.lang.Object> booleanWritableConverter()</pre>
</li>
</ul>
<a name="bytesWritableConverter()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>bytesWritableConverter</h4>
<pre>public static org.apache.spark.WritableConverter<byte[]> bytesWritableConverter()</pre>
</li>
</ul>
<a name="stringWritableConverter()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>stringWritableConverter</h4>
<pre>public static org.apache.spark.WritableConverter<java.lang.String> stringWritableConverter()</pre>
</li>
</ul>
<a name="writableWritableConverter()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>writableWritableConverter</h4>
<pre>public static <T extends org.apache.hadoop.io.Writable> org.apache.spark.WritableConverter<T> writableWritableConverter()</pre>
</li>
</ul>
<a name="jarOfClass(java.lang.Class)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>jarOfClass</h4>
<pre>public static scala.Option<java.lang.String> jarOfClass(java.lang.Class<?> cls)</pre>
<div class="block">Find the JAR from which a given class was loaded, to make it easy for users to pass
their JARs to SparkContext.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>cls</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="jarOfObject(java.lang.Object)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>jarOfObject</h4>
<pre>public static scala.Option<java.lang.String> jarOfObject(java.lang.Object obj)</pre>
<div class="block">Find the JAR that contains the class of a particular object, to make it easy for users
to pass their JARs to SparkContext. In most cases you can call jarOfObject(this) in
your driver program.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>obj</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="preferredNodeLocationData()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>preferredNodeLocationData</h4>
<pre>public scala.collection.Map<java.lang.String,scala.collection.Set<<a href="../../../org/apache/spark/scheduler/SplitInfo.html" title="class in org.apache.spark.scheduler">SplitInfo</a>>> preferredNodeLocationData()</pre>
</li>
</ul>
<a name="startTime()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>startTime</h4>
<pre>public long startTime()</pre>
</li>
</ul>
<a name="getConf()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getConf</h4>
<pre>public <a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark">SparkConf</a> getConf()</pre>
<div class="block">Return a copy of this SparkContext's configuration. The configuration ''cannot'' be
changed at runtime.</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="jars()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>jars</h4>
<pre>public scala.collection.Seq<java.lang.String> jars()</pre>
</li>
</ul>
<a name="files()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>files</h4>
<pre>public scala.collection.Seq<java.lang.String> files()</pre>
</li>
</ul>
<a name="master()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>master</h4>
<pre>public java.lang.String master()</pre>
</li>
</ul>
<a name="appName()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>appName</h4>
<pre>public java.lang.String appName()</pre>
</li>
</ul>
<a name="externalBlockStoreFolderName()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>externalBlockStoreFolderName</h4>
<pre>public java.lang.String externalBlockStoreFolderName()</pre>
</li>
</ul>
<a name="tachyonFolderName()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>tachyonFolderName</h4>
<pre>public java.lang.String tachyonFolderName()</pre>
</li>
</ul>
<a name="isLocal()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>isLocal</h4>
<pre>public boolean isLocal()</pre>
</li>
</ul>
<a name="listenerBus()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>listenerBus</h4>
<pre>public org.apache.spark.scheduler.LiveListenerBus listenerBus()</pre>
</li>
</ul>
<a name="addedFiles()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>addedFiles</h4>
<pre>public scala.collection.mutable.HashMap<java.lang.String,java.lang.Object> addedFiles()</pre>
</li>
</ul>
<a name="addedJars()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>addedJars</h4>
<pre>public scala.collection.mutable.HashMap<java.lang.String,java.lang.Object> addedJars()</pre>
</li>
</ul>
<a name="persistentRdds()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>persistentRdds</h4>
<pre>public <any> persistentRdds()</pre>
</li>
</ul>
<a name="statusTracker()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>statusTracker</h4>
<pre>public <a href="../../../org/apache/spark/SparkStatusTracker.html" title="class in org.apache.spark">SparkStatusTracker</a> statusTracker()</pre>
</li>
</ul>
<a name="hadoopConfiguration()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>hadoopConfiguration</h4>
<pre>public org.apache.hadoop.conf.Configuration hadoopConfiguration()</pre>
<div class="block">A default Hadoop Configuration for the Hadoop code (e.g. file systems) that we reuse.
<p>
'''Note:''' As it will be reused in all Hadoop RDDs, it's better not to modify it unless you
plan to set some global configurations for all Hadoop RDDs.</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="executorEnvs()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>executorEnvs</h4>
<pre>public scala.collection.mutable.HashMap<java.lang.String,java.lang.String> executorEnvs()</pre>
</li>
</ul>
<a name="sparkUser()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>sparkUser</h4>
<pre>public java.lang.String sparkUser()</pre>
</li>
</ul>
<a name="applicationId()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>applicationId</h4>
<pre>public java.lang.String applicationId()</pre>
<div class="block">A unique identifier for the Spark application.
Its format depends on the scheduler implementation.
(i.e.
in case of local spark app something like 'local-1433865536131'
in case of YARN something like 'application_1433865536131_34483'
)</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="applicationAttemptId()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>applicationAttemptId</h4>
<pre>public scala.Option<java.lang.String> applicationAttemptId()</pre>
</li>
</ul>
<a name="metricsSystem()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>metricsSystem</h4>
<pre>public org.apache.spark.metrics.MetricsSystem metricsSystem()</pre>
</li>
</ul>
<a name="checkpointDir()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>checkpointDir</h4>
<pre>public scala.Option<java.lang.String> checkpointDir()</pre>
</li>
</ul>
<a name="localProperties()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>localProperties</h4>
<pre>protected java.lang.InheritableThreadLocal<java.util.Properties> localProperties()</pre>
</li>
</ul>
<a name="setLogLevel(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>setLogLevel</h4>
<pre>public void setLogLevel(java.lang.String logLevel)</pre>
<div class="block">Control our logLevel. This overrides any user-defined log settings.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>logLevel</code> - The desired log level as a string.
Valid log levels include: ALL, DEBUG, ERROR, FATAL, INFO, OFF, TRACE, WARN</dd></dl>
</li>
</ul>
<a name="initLocalProperties()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>initLocalProperties</h4>
<pre>public void initLocalProperties()</pre>
</li>
</ul>
<a name="setLocalProperty(java.lang.String, java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>setLocalProperty</h4>
<pre>public void setLocalProperty(java.lang.String key,
java.lang.String value)</pre>
<div class="block">Set a local property that affects jobs submitted from this thread, such as the
Spark fair scheduler pool.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>key</code> - (undocumented)</dd><dd><code>value</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="getLocalProperty(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getLocalProperty</h4>
<pre>public java.lang.String getLocalProperty(java.lang.String key)</pre>
<div class="block">Get a local property set in this thread, or null if it is missing. See
<code>org.apache.spark.SparkContext.setLocalProperty</code>.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>key</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="setJobDescription(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>setJobDescription</h4>
<pre>public void setJobDescription(java.lang.String value)</pre>
<div class="block">Set a human readable description of the current job.</div>
</li>
</ul>
<a name="setJobGroup(java.lang.String, java.lang.String, boolean)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>setJobGroup</h4>
<pre>public void setJobGroup(java.lang.String groupId,
java.lang.String description,
boolean interruptOnCancel)</pre>
<div class="block">Assigns a group ID to all the jobs started by this thread until the group ID is set to a
different value or cleared.
<p>
Often, a unit of execution in an application consists of multiple Spark actions or jobs.
Application programmers can use this method to group all those jobs together and give a
group description. Once set, the Spark web UI will associate such jobs with this group.
<p>
The application can also use <code>org.apache.spark.SparkContext.cancelJobGroup</code> to cancel all
running jobs in this group. For example,
<pre><code>
// In the main thread:
sc.setJobGroup("some_job_to_cancel", "some job description")
sc.parallelize(1 to 10000, 2).map { i => Thread.sleep(10); i }.count()
// In a separate thread:
sc.cancelJobGroup("some_job_to_cancel")
</code></pre>
<p>
If interruptOnCancel is set to true for the job group, then job cancellation will result
in Thread.interrupt() being called on the job's executor threads. This is useful to help ensure
that the tasks are actually stopped in a timely manner, but is off by default due to HDFS-1208,
where HDFS may respond to Thread.interrupt() by marking nodes as dead.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>groupId</code> - (undocumented)</dd><dd><code>description</code> - (undocumented)</dd><dd><code>interruptOnCancel</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="clearJobGroup()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>clearJobGroup</h4>
<pre>public void clearJobGroup()</pre>
<div class="block">Clear the current thread's job group ID and its description.</div>
</li>
</ul>
<a name="parallelize(scala.collection.Seq, int, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>parallelize</h4>
<pre>public <T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> parallelize(scala.collection.Seq<T> seq,
int numSlices,
scala.reflect.ClassTag<T> evidence$1)</pre>
<div class="block">Distribute a local Scala collection to form an RDD.
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>seq</code> - (undocumented)</dd><dd><code>numSlices</code> - (undocumented)</dd><dd><code>evidence$1</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="range(long, long, long, int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>range</h4>
<pre>public <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><java.lang.Object> range(long start,
long end,
long step,
int numSlices)</pre>
<div class="block">Creates a new RDD[Long] containing elements from <code>start</code> to <code>end</code>(exclusive), increased by
<code>step</code> every element.
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>start</code> - the start value.</dd><dd><code>end</code> - the end value.</dd><dd><code>step</code> - the incremental step</dd><dd><code>numSlices</code> - the partition number of the new RDD.</dd>
<dt><span class="strong">Returns:</span></dt><dd></dd></dl>
</li>
</ul>
<a name="makeRDD(scala.collection.Seq, int, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>makeRDD</h4>
<pre>public <T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> makeRDD(scala.collection.Seq<T> seq,
int numSlices,
scala.reflect.ClassTag<T> evidence$2)</pre>
<div class="block">Distribute a local Scala collection to form an RDD.
<p>
This method is identical to <code>parallelize</code>.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>seq</code> - (undocumented)</dd><dd><code>numSlices</code> - (undocumented)</dd><dd><code>evidence$2</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="makeRDD(scala.collection.Seq, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>makeRDD</h4>
<pre>public <T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> makeRDD(scala.collection.Seq<scala.Tuple2<T,scala.collection.Seq<java.lang.String>>> seq,
scala.reflect.ClassTag<T> evidence$3)</pre>
<div class="block">Distribute a local Scala collection to form an RDD, with one or more
location preferences (hostnames of Spark nodes) for each object.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>seq</code> - (undocumented)</dd><dd><code>evidence$3</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)
Create a new partition for each collection item.</dd></dl>
</li>
</ul>
<a name="textFile(java.lang.String, int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>textFile</h4>
<pre>public <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><java.lang.String> textFile(java.lang.String path,
int minPartitions)</pre>
<div class="block">Read a text file from HDFS, a local file system (available on all nodes), or any
Hadoop-supported file system URI, and return it as an RDD of Strings.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>minPartitions</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="wholeTextFiles(java.lang.String, int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>wholeTextFiles</h4>
<pre>public <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<java.lang.String,java.lang.String>> wholeTextFiles(java.lang.String path,
int minPartitions)</pre>
<div class="block">Read a directory of text files from HDFS, a local file system (available on all nodes), or any
Hadoop-supported file system URI. Each file is read as a single record and returned in a
key-value pair, where the key is the path of each file, the value is the content of each file.
<p>
<p> For example, if you have the following files:
<pre><code>
hdfs://a-hdfs-path/part-00000
hdfs://a-hdfs-path/part-00001
...
hdfs://a-hdfs-path/part-nnnnn
</code></pre>
<p>
Do <code>val rdd = sparkContext.wholeTextFile("hdfs://a-hdfs-path")</code>,
<p>
<p> then <code>rdd</code> contains
<pre><code>
(a-hdfs-path/part-00000, its content)
(a-hdfs-path/part-00001, its content)
...
(a-hdfs-path/part-nnnnn, its content)
</code></pre>
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - Directory to the input data files, the path can be comma separated paths as the
list of inputs.</dd><dd><code>minPartitions</code> - A suggestion value of the minimal splitting number for input data.</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="binaryFiles(java.lang.String, int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>binaryFiles</h4>
<pre>public <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<java.lang.String,<a href="../../../org/apache/spark/input/PortableDataStream.html" title="class in org.apache.spark.input">PortableDataStream</a>>> binaryFiles(java.lang.String path,
int minPartitions)</pre>
<div class="block">:: Experimental ::
<p>
Get an RDD for a Hadoop-readable dataset as PortableDataStream for each file
(useful for binary data)
<p>
For example, if you have the following files:
<pre><code>
hdfs://a-hdfs-path/part-00000
hdfs://a-hdfs-path/part-00001
...
hdfs://a-hdfs-path/part-nnnnn
</code></pre>
<p>
Do
<code>val rdd = sparkContext.binaryFiles("hdfs://a-hdfs-path")</code>,
<p>
then <code>rdd</code> contains
<pre><code>
(a-hdfs-path/part-00000, its content)
(a-hdfs-path/part-00001, its content)
...
(a-hdfs-path/part-nnnnn, its content)
</code></pre>
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - Directory to the input data files, the path can be comma separated paths as the
list of inputs.</dd><dd><code>minPartitions</code> - A suggestion value of the minimal splitting number for input data.</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="binaryRecords(java.lang.String, int, org.apache.hadoop.conf.Configuration)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>binaryRecords</h4>
<pre>public <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><byte[]> binaryRecords(java.lang.String path,
int recordLength,
org.apache.hadoop.conf.Configuration conf)</pre>
<div class="block">:: Experimental ::
<p>
Load data from a flat binary file, assuming the length of each record is constant.
<p>
'''Note:''' We ensure that the byte array for each record in the resulting RDD
has the provided record length.
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - Directory to the input data files, the path can be comma separated paths as the
list of inputs.</dd><dd><code>recordLength</code> - The length at which to split the records</dd><dd><code>conf</code> - Configuration for setting up the dataset.
<p></dd>
<dt><span class="strong">Returns:</span></dt><dd>An RDD of data with values, represented as byte arrays</dd></dl>
</li>
</ul>
<a name="hadoopRDD(org.apache.hadoop.mapred.JobConf, java.lang.Class, java.lang.Class, java.lang.Class, int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>hadoopRDD</h4>
<pre>public <K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> hadoopRDD(org.apache.hadoop.mapred.JobConf conf,
java.lang.Class<? extends org.apache.hadoop.mapred.InputFormat<K,V>> inputFormatClass,
java.lang.Class<K> keyClass,
java.lang.Class<V> valueClass,
int minPartitions)</pre>
<div class="block">Get an RDD for a Hadoop-readable dataset from a Hadoop JobConf given its InputFormat and other
necessary info (e.g. file name for a filesystem-based dataset, table name for HyperTable),
using the older MapReduce API (<code>org.apache.hadoop.mapred</code>).
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>conf</code> - JobConf for setting up the dataset. Note: This will be put into a Broadcast.
Therefore if you plan to reuse this conf to create multiple RDDs, you need to make
sure you won't modify the conf. A safe approach is always creating a new conf for
a new RDD.</dd><dd><code>inputFormatClass</code> - Class of the InputFormat</dd><dd><code>keyClass</code> - Class of the keys</dd><dd><code>valueClass</code> - Class of the values</dd><dd><code>minPartitions</code> - Minimum number of Hadoop Splits to generate.
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="hadoopFile(java.lang.String, java.lang.Class, java.lang.Class, java.lang.Class, int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>hadoopFile</h4>
<pre>public <K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> hadoopFile(java.lang.String path,
java.lang.Class<? extends org.apache.hadoop.mapred.InputFormat<K,V>> inputFormatClass,
java.lang.Class<K> keyClass,
java.lang.Class<V> valueClass,
int minPartitions)</pre>
<div class="block">Get an RDD for a Hadoop file with an arbitrary InputFormat
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>inputFormatClass</code> - (undocumented)</dd><dd><code>keyClass</code> - (undocumented)</dd><dd><code>valueClass</code> - (undocumented)</dd><dd><code>minPartitions</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="hadoopFile(java.lang.String, int, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>hadoopFile</h4>
<pre>public <K,V,F extends org.apache.hadoop.mapred.InputFormat<K,V>> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> hadoopFile(java.lang.String path,
int minPartitions,
scala.reflect.ClassTag<K> km,
scala.reflect.ClassTag<V> vm,
scala.reflect.ClassTag<F> fm)</pre>
<div class="block">Smarter version of hadoopFile() that uses class tags to figure out the classes of keys,
values and the InputFormat so that users don't need to pass them directly. Instead, callers
can just write, for example,
<pre><code>
val file = sparkContext.hadoopFile[LongWritable, Text, TextInputFormat](path, minPartitions)
</code></pre>
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>minPartitions</code> - (undocumented)</dd><dd><code>km</code> - (undocumented)</dd><dd><code>vm</code> - (undocumented)</dd><dd><code>fm</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="hadoopFile(java.lang.String, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>hadoopFile</h4>
<pre>public <K,V,F extends org.apache.hadoop.mapred.InputFormat<K,V>> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> hadoopFile(java.lang.String path,
scala.reflect.ClassTag<K> km,
scala.reflect.ClassTag<V> vm,
scala.reflect.ClassTag<F> fm)</pre>
<div class="block">Smarter version of hadoopFile() that uses class tags to figure out the classes of keys,
values and the InputFormat so that users don't need to pass them directly. Instead, callers
can just write, for example,
<pre><code>
val file = sparkContext.hadoopFile[LongWritable, Text, TextInputFormat](path)
</code></pre>
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>km</code> - (undocumented)</dd><dd><code>vm</code> - (undocumented)</dd><dd><code>fm</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="newAPIHadoopFile(java.lang.String, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>newAPIHadoopFile</h4>
<pre>public <K,V,F extends org.apache.hadoop.mapreduce.InputFormat<K,V>> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> newAPIHadoopFile(java.lang.String path,
scala.reflect.ClassTag<K> km,
scala.reflect.ClassTag<V> vm,
scala.reflect.ClassTag<F> fm)</pre>
<div class="block">Get an RDD for a Hadoop file with an arbitrary new API InputFormat.</div>
</li>
</ul>
<a name="newAPIHadoopFile(java.lang.String, java.lang.Class, java.lang.Class, java.lang.Class, org.apache.hadoop.conf.Configuration)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>newAPIHadoopFile</h4>
<pre>public <K,V,F extends org.apache.hadoop.mapreduce.InputFormat<K,V>> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> newAPIHadoopFile(java.lang.String path,
java.lang.Class<F> fClass,
java.lang.Class<K> kClass,
java.lang.Class<V> vClass,
org.apache.hadoop.conf.Configuration conf)</pre>
<div class="block">Get an RDD for a given Hadoop file with an arbitrary new API InputFormat
and extra configuration options to pass to the input format.
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>fClass</code> - (undocumented)</dd><dd><code>kClass</code> - (undocumented)</dd><dd><code>vClass</code> - (undocumented)</dd><dd><code>conf</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="newAPIHadoopRDD(org.apache.hadoop.conf.Configuration, java.lang.Class, java.lang.Class, java.lang.Class)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>newAPIHadoopRDD</h4>
<pre>public <K,V,F extends org.apache.hadoop.mapreduce.InputFormat<K,V>> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> newAPIHadoopRDD(org.apache.hadoop.conf.Configuration conf,
java.lang.Class<F> fClass,
java.lang.Class<K> kClass,
java.lang.Class<V> vClass)</pre>
<div class="block">Get an RDD for a given Hadoop file with an arbitrary new API InputFormat
and extra configuration options to pass to the input format.
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>conf</code> - Configuration for setting up the dataset. Note: This will be put into a Broadcast.
Therefore if you plan to reuse this conf to create multiple RDDs, you need to make
sure you won't modify the conf. A safe approach is always creating a new conf for
a new RDD.</dd><dd><code>fClass</code> - Class of the InputFormat</dd><dd><code>kClass</code> - Class of the keys</dd><dd><code>vClass</code> - Class of the values
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="sequenceFile(java.lang.String, java.lang.Class, java.lang.Class, int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>sequenceFile</h4>
<pre>public <K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> sequenceFile(java.lang.String path,
java.lang.Class<K> keyClass,
java.lang.Class<V> valueClass,
int minPartitions)</pre>
<div class="block">Get an RDD for a Hadoop SequenceFile with given key and value types.
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>keyClass</code> - (undocumented)</dd><dd><code>valueClass</code> - (undocumented)</dd><dd><code>minPartitions</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="sequenceFile(java.lang.String, java.lang.Class, java.lang.Class)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>sequenceFile</h4>
<pre>public <K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> sequenceFile(java.lang.String path,
java.lang.Class<K> keyClass,
java.lang.Class<V> valueClass)</pre>
<div class="block">Get an RDD for a Hadoop SequenceFile with given key and value types.
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>keyClass</code> - (undocumented)</dd><dd><code>valueClass</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="sequenceFile(java.lang.String, int, scala.reflect.ClassTag, scala.reflect.ClassTag, scala.Function0, scala.Function0)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>sequenceFile</h4>
<pre>public <K,V> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><scala.Tuple2<K,V>> sequenceFile(java.lang.String path,
int minPartitions,
scala.reflect.ClassTag<K> km,
scala.reflect.ClassTag<V> vm,
scala.Function0<org.apache.spark.WritableConverter<K>> kcf,
scala.Function0<org.apache.spark.WritableConverter<V>> vcf)</pre>
<div class="block">Version of sequenceFile() for types implicitly convertible to Writables through a
WritableConverter. For example, to access a SequenceFile where the keys are Text and the
values are IntWritable, you could simply write
<pre><code>
sparkContext.sequenceFile[String, Int](path, ...)
</code></pre>
<p>
WritableConverters are provided in a somewhat strange way (by an implicit function) to support
both subclasses of Writable and types for which we define a converter (e.g. Int to
IntWritable). The most natural thing would've been to have implicit objects for the
converters, but then we couldn't have an object for every subclass of Writable (you can't
have a parameterized singleton object). We use functions instead to create a new converter
for the appropriate type. In addition, we pass the converter a ClassTag of its type to
allow it to figure out the Writable class to use in the subclass case.
<p>
'''Note:''' Because Hadoop's RecordReader class re-uses the same Writable object for each
record, directly caching the returned RDD or directly passing it to an aggregation or shuffle
operation will create many references to the same object.
If you plan to directly cache, sort, or aggregate Hadoop writable objects, you should first
copy them using a <code>map</code> function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>minPartitions</code> - (undocumented)</dd><dd><code>km</code> - (undocumented)</dd><dd><code>vm</code> - (undocumented)</dd><dd><code>kcf</code> - (undocumented)</dd><dd><code>vcf</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="objectFile(java.lang.String, int, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>objectFile</h4>
<pre>public <T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> objectFile(java.lang.String path,
int minPartitions,
scala.reflect.ClassTag<T> evidence$4)</pre>
<div class="block">Load an RDD saved as a SequenceFile containing serialized objects, with NullWritable keys and
BytesWritable values that contain a serialized partition. This is still an experimental
storage format and may not be supported exactly as is in future Spark releases. It will also
be pretty slow if you use the default serializer (Java serialization),
though the nice thing about it is that there's very little effort required to save arbitrary
objects.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>minPartitions</code> - (undocumented)</dd><dd><code>evidence$4</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="checkpointFile(java.lang.String, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>checkpointFile</h4>
<pre>protected <T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> checkpointFile(java.lang.String path,
scala.reflect.ClassTag<T> evidence$5)</pre>
</li>
</ul>
<a name="union(scala.collection.Seq, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>union</h4>
<pre>public <T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> union(scala.collection.Seq<<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T>> rdds,
scala.reflect.ClassTag<T> evidence$6)</pre>
<div class="block">Build the union of a list of RDDs.</div>
</li>
</ul>
<a name="union(org.apache.spark.rdd.RDD, scala.collection.Seq, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>union</h4>
<pre>public <T> <a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> union(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> first,
scala.collection.Seq<<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T>> rest,
scala.reflect.ClassTag<T> evidence$7)</pre>
<div class="block">Build the union of a list of RDDs passed as variable-length arguments.</div>
</li>
</ul>
<a name="emptyRDD(scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>emptyRDD</h4>
<pre>public <T> <any> emptyRDD(scala.reflect.ClassTag<T> evidence$8)</pre>
<div class="block">Get an RDD that has no partitions or elements.</div>
</li>
</ul>
<a name="accumulator(java.lang.Object,org.apache.spark.AccumulatorParam)">
<!-- -->
</a><a name="accumulator(T, org.apache.spark.AccumulatorParam)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>accumulator</h4>
<pre>public <T> <a href="../../../org/apache/spark/Accumulator.html" title="class in org.apache.spark">Accumulator</a><T> accumulator(T initialValue,
<a href="../../../org/apache/spark/AccumulatorParam.html" title="interface in org.apache.spark">AccumulatorParam</a><T> param)</pre>
<div class="block">Create an <a href="../../../org/apache/spark/Accumulator.html" title="class in org.apache.spark"><code>Accumulator</code></a> variable of a given type, which tasks can "add"
values to using the <code>+=</code> method. Only the driver can access the accumulator's <code>value</code>.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>initialValue</code> - (undocumented)</dd><dd><code>param</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="accumulator(java.lang.Object,java.lang.String,org.apache.spark.AccumulatorParam)">
<!-- -->
</a><a name="accumulator(T, java.lang.String, org.apache.spark.AccumulatorParam)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>accumulator</h4>
<pre>public <T> <a href="../../../org/apache/spark/Accumulator.html" title="class in org.apache.spark">Accumulator</a><T> accumulator(T initialValue,
java.lang.String name,
<a href="../../../org/apache/spark/AccumulatorParam.html" title="interface in org.apache.spark">AccumulatorParam</a><T> param)</pre>
<div class="block">Create an <a href="../../../org/apache/spark/Accumulator.html" title="class in org.apache.spark"><code>Accumulator</code></a> variable of a given type, with a name for display
in the Spark UI. Tasks can "add" values to the accumulator using the <code>+=</code> method. Only the
driver can access the accumulator's <code>value</code>.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>initialValue</code> - (undocumented)</dd><dd><code>name</code> - (undocumented)</dd><dd><code>param</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="accumulable(java.lang.Object,org.apache.spark.AccumulableParam)">
<!-- -->
</a><a name="accumulable(R, org.apache.spark.AccumulableParam)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>accumulable</h4>
<pre>public <R,T> <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark">Accumulable</a><R,T> accumulable(R initialValue,
<a href="../../../org/apache/spark/AccumulableParam.html" title="interface in org.apache.spark">AccumulableParam</a><R,T> param)</pre>
<div class="block">Create an <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark"><code>Accumulable</code></a> shared variable, to which tasks can add values
with <code>+=</code>. Only the driver can access the accumuable's <code>value</code>.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>initialValue</code> - (undocumented)</dd><dd><code>param</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="accumulable(java.lang.Object,java.lang.String,org.apache.spark.AccumulableParam)">
<!-- -->
</a><a name="accumulable(R, java.lang.String, org.apache.spark.AccumulableParam)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>accumulable</h4>
<pre>public <R,T> <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark">Accumulable</a><R,T> accumulable(R initialValue,
java.lang.String name,
<a href="../../../org/apache/spark/AccumulableParam.html" title="interface in org.apache.spark">AccumulableParam</a><R,T> param)</pre>
<div class="block">Create an <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark"><code>Accumulable</code></a> shared variable, with a name for display in the
Spark UI. Tasks can add values to the accumuable using the <code>+=</code> operator. Only the driver can
access the accumuable's <code>value</code>.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>initialValue</code> - (undocumented)</dd><dd><code>name</code> - (undocumented)</dd><dd><code>param</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="accumulableCollection(java.lang.Object,scala.Function1,scala.reflect.ClassTag)">
<!-- -->
</a><a name="accumulableCollection(R, scala.Function1, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>accumulableCollection</h4>
<pre>public <R,T> <a href="../../../org/apache/spark/Accumulable.html" title="class in org.apache.spark">Accumulable</a><R,T> accumulableCollection(R initialValue,
scala.Function1<R,scala.collection.generic.Growable<T>> evidence$9,
scala.reflect.ClassTag<R> evidence$10)</pre>
<div class="block">Create an accumulator from a "mutable collection" type.
<p>
Growable and TraversableOnce are the standard APIs that guarantee += and ++=, implemented by
standard mutable collections. So you can use this with mutable Map, Set, etc.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>initialValue</code> - (undocumented)</dd><dd><code>evidence$9</code> - (undocumented)</dd><dd><code>evidence$10</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="broadcast(java.lang.Object,scala.reflect.ClassTag)">
<!-- -->
</a><a name="broadcast(T, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>broadcast</h4>
<pre>public <T> <a href="../../../org/apache/spark/broadcast/Broadcast.html" title="class in org.apache.spark.broadcast">Broadcast</a><T> broadcast(T value,
scala.reflect.ClassTag<T> evidence$11)</pre>
<div class="block">Broadcast a read-only variable to the cluster, returning a
<a href="../../../org/apache/spark/broadcast/Broadcast.html" title="class in org.apache.spark.broadcast"><code>Broadcast</code></a> object for reading it in distributed functions.
The variable will be sent to each cluster only once.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>value</code> - (undocumented)</dd><dd><code>evidence$11</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="addFile(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>addFile</h4>
<pre>public void addFile(java.lang.String path)</pre>
<div class="block">Add a file to be downloaded with this Spark job on every node.
The <code>path</code> passed can be either a local file, a file in HDFS (or other Hadoop-supported
filesystems), or an HTTP, HTTPS or FTP URI. To access the file in Spark jobs,
use <code>SparkFiles.get(fileName)</code> to find its download location.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="addFile(java.lang.String, boolean)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>addFile</h4>
<pre>public void addFile(java.lang.String path,
boolean recursive)</pre>
<div class="block">Add a file to be downloaded with this Spark job on every node.
The <code>path</code> passed can be either a local file, a file in HDFS (or other Hadoop-supported
filesystems), or an HTTP, HTTPS or FTP URI. To access the file in Spark jobs,
use <code>SparkFiles.get(fileName)</code> to find its download location.
<p>
A directory can be given if the recursive option is set to true. Currently directories are only
supported for Hadoop-supported filesystems.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd><dd><code>recursive</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="addSparkListener(org.apache.spark.scheduler.SparkListener)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>addSparkListener</h4>
<pre>public void addSparkListener(<a href="../../../org/apache/spark/scheduler/SparkListener.html" title="interface in org.apache.spark.scheduler">SparkListener</a> listener)</pre>
<div class="block">:: DeveloperApi ::
Register a listener to receive up-calls from events that happen during execution.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>listener</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="requestExecutors(int)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>requestExecutors</h4>
<pre>public boolean requestExecutors(int numAdditionalExecutors)</pre>
<div class="block">:: DeveloperApi ::
Request an additional number of executors from the cluster manager.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>numAdditionalExecutors</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>whether the request is received.</dd></dl>
</li>
</ul>
<a name="killExecutors(scala.collection.Seq)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>killExecutors</h4>
<pre>public boolean killExecutors(scala.collection.Seq<java.lang.String> executorIds)</pre>
<div class="block">:: DeveloperApi ::
Request that the cluster manager kill the specified executors.
<p>
Note: This is an indication to the cluster manager that the application wishes to adjust
its resource usage downwards. If the application wishes to replace the executors it kills
through this method with new ones, it should follow up explicitly with a call to
{{SparkContext#requestExecutors}}.
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>executorIds</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>whether the request is received.</dd></dl>
</li>
</ul>
<a name="killExecutor(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>killExecutor</h4>
<pre>public boolean killExecutor(java.lang.String executorId)</pre>
<div class="block">:: DeveloperApi ::
Request that the cluster manager kill the specified executor.
<p>
Note: This is an indication to the cluster manager that the application wishes to adjust
its resource usage downwards. If the application wishes to replace the executor it kills
through this method with a new one, it should follow up explicitly with a call to
{{SparkContext#requestExecutors}}.
<p></div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>executorId</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>whether the request is received.</dd></dl>
</li>
</ul>
<a name="version()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>version</h4>
<pre>public java.lang.String version()</pre>
<div class="block">The version of Spark on which this application is running.</div>
</li>
</ul>
<a name="getExecutorMemoryStatus()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getExecutorMemoryStatus</h4>
<pre>public scala.collection.Map<java.lang.String,scala.Tuple2<java.lang.Object,java.lang.Object>> getExecutorMemoryStatus()</pre>
<div class="block">Return a map from the slave to the max memory available for caching and the remaining
memory available for caching.</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="getRDDStorageInfo()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getRDDStorageInfo</h4>
<pre>public <a href="../../../org/apache/spark/storage/RDDInfo.html" title="class in org.apache.spark.storage">RDDInfo</a>[] getRDDStorageInfo()</pre>
<div class="block">:: DeveloperApi ::
Return information about what RDDs are cached, if they are in mem or on disk, how much space
they take, etc.</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="getPersistentRDDs()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getPersistentRDDs</h4>
<pre>public scala.collection.Map<java.lang.Object,<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><?>> getPersistentRDDs()</pre>
<div class="block">Returns an immutable map of RDDs that have marked themselves as persistent via cache() call.
Note that this does not necessarily mean the caching or computation was successful.</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="getExecutorStorageStatus()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getExecutorStorageStatus</h4>
<pre>public <a href="../../../org/apache/spark/storage/StorageStatus.html" title="class in org.apache.spark.storage">StorageStatus</a>[] getExecutorStorageStatus()</pre>
<div class="block">:: DeveloperApi ::
Return information about blocks stored in all of the slaves</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="getAllPools()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getAllPools</h4>
<pre>public scala.collection.Seq<org.apache.spark.scheduler.Schedulable> getAllPools()</pre>
<div class="block">:: DeveloperApi ::
Return pools for fair scheduler</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="getPoolForName(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getPoolForName</h4>
<pre>public scala.Option<org.apache.spark.scheduler.Schedulable> getPoolForName(java.lang.String pool)</pre>
<div class="block">:: DeveloperApi ::
Return the pool associated with the given name, if one exists</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>pool</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="getSchedulingMode()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getSchedulingMode</h4>
<pre>public scala.Enumeration.Value getSchedulingMode()</pre>
<div class="block">Return current scheduling mode</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="clearFiles()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>clearFiles</h4>
<pre>public void clearFiles()</pre>
<div class="block">Clear the job's list of files added by <code>addFile</code> so that they do not get downloaded to
any new nodes.</div>
</li>
</ul>
<a name="addJar(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>addJar</h4>
<pre>public void addJar(java.lang.String path)</pre>
<div class="block">Adds a JAR dependency for all tasks to be executed on this SparkContext in the future.
The <code>path</code> passed can be either a local file, a file in HDFS (or other Hadoop-supported
filesystems), an HTTP, HTTPS or FTP URI, or local:/path for a file on every worker node.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>path</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="clearJars()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>clearJars</h4>
<pre>public void clearJars()</pre>
<div class="block">Clear the job's list of JARs added by <code>addJar</code> so that they do not get downloaded to
any new nodes.</div>
</li>
</ul>
<a name="stop()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>stop</h4>
<pre>public void stop()</pre>
</li>
</ul>
<a name="setCallSite(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>setCallSite</h4>
<pre>public void setCallSite(java.lang.String shortCallSite)</pre>
<div class="block">Set the thread-local property for overriding the call sites
of actions and RDDs.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>shortCallSite</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="clearCallSite()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>clearCallSite</h4>
<pre>public void clearCallSite()</pre>
<div class="block">Clear the thread-local property for overriding the call sites
of actions and RDDs.</div>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.collection.Seq, scala.Function2, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> void runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.reflect.ClassTag<U> evidence$12)</pre>
<div class="block">Run a function on a given set of partitions in an RDD and pass the results to the given
handler function. This is the main entry point for all actions in Spark.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>partitions</code> - (undocumented)</dd><dd><code>resultHandler</code> - (undocumented)</dd><dd><code>evidence$12</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.collection.Seq, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> java.lang.Object runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
scala.reflect.ClassTag<U> evidence$13)</pre>
<div class="block">Run a function on a given set of partitions in an RDD and return the results as an array.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>partitions</code> - (undocumented)</dd><dd><code>evidence$13</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function1, scala.collection.Seq, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> java.lang.Object runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
scala.reflect.ClassTag<U> evidence$14)</pre>
<div class="block">Run a job on a given set of partitions of an RDD, but take a function of type
<code>Iterator[T] => U</code> instead of <code>(TaskContext, Iterator[T]) => U</code>.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>partitions</code> - (undocumented)</dd><dd><code>evidence$14</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.collection.Seq, boolean, scala.Function2, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> void runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
boolean allowLocal,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.reflect.ClassTag<U> evidence$15)</pre>
<div class="block">Run a function on a given set of partitions in an RDD and pass the results to the given
handler function. This is the main entry point for all actions in Spark.
<p>
The allowLocal flag is deprecated as of Spark 1.5.0+.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>partitions</code> - (undocumented)</dd><dd><code>allowLocal</code> - (undocumented)</dd><dd><code>resultHandler</code> - (undocumented)</dd><dd><code>evidence$15</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.collection.Seq, boolean, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> java.lang.Object runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
boolean allowLocal,
scala.reflect.ClassTag<U> evidence$16)</pre>
<div class="block">Run a function on a given set of partitions in an RDD and return the results as an array.
<p>
The allowLocal flag is deprecated as of Spark 1.5.0+.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>partitions</code> - (undocumented)</dd><dd><code>allowLocal</code> - (undocumented)</dd><dd><code>evidence$16</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function1, scala.collection.Seq, boolean, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> java.lang.Object runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> func,
scala.collection.Seq<java.lang.Object> partitions,
boolean allowLocal,
scala.reflect.ClassTag<U> evidence$17)</pre>
<div class="block">Run a job on a given set of partitions of an RDD, but take a function of type
<code>Iterator[T] => U</code> instead of <code>(TaskContext, Iterator[T]) => U</code>.
<p>
The allowLocal argument is deprecated as of Spark 1.5.0+.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>partitions</code> - (undocumented)</dd><dd><code>allowLocal</code> - (undocumented)</dd><dd><code>evidence$17</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> java.lang.Object runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
scala.reflect.ClassTag<U> evidence$18)</pre>
<div class="block">Run a job on all partitions in an RDD and return the results in an array.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>evidence$18</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function1, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> java.lang.Object runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> func,
scala.reflect.ClassTag<U> evidence$19)</pre>
<div class="block">Run a job on all partitions in an RDD and return the results in an array.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>evidence$19</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function2, scala.Function2, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> void runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> processPartition,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.reflect.ClassTag<U> evidence$20)</pre>
<div class="block">Run a job on all partitions in an RDD and pass the results to a handler function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>processPartition</code> - (undocumented)</dd><dd><code>resultHandler</code> - (undocumented)</dd><dd><code>evidence$20</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="runJob(org.apache.spark.rdd.RDD, scala.Function1, scala.Function2, scala.reflect.ClassTag)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runJob</h4>
<pre>public <T,U> void runJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> processPartition,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.reflect.ClassTag<U> evidence$21)</pre>
<div class="block">Run a job on all partitions in an RDD and pass the results to a handler function.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>processPartition</code> - (undocumented)</dd><dd><code>resultHandler</code> - (undocumented)</dd><dd><code>evidence$21</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="runApproximateJob(org.apache.spark.rdd.RDD, scala.Function2, , long)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>runApproximateJob</h4>
<pre>public <T,U,R> <a href="../../../org/apache/spark/partial/PartialResult.html" title="class in org.apache.spark.partial">PartialResult</a><R> runApproximateJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function2<<a href="../../../org/apache/spark/TaskContext.html" title="class in org.apache.spark">TaskContext</a>,scala.collection.Iterator<T>,U> func,
<any> evaluator,
long timeout)</pre>
<div class="block">:: DeveloperApi ::
Run a job that can return approximate results.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>func</code> - (undocumented)</dd><dd><code>evaluator</code> - (undocumented)</dd><dd><code>timeout</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="submitJob(org.apache.spark.rdd.RDD, scala.Function1, scala.collection.Seq, scala.Function2, scala.Function0)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>submitJob</h4>
<pre>public <T,U,R> <a href="../../../org/apache/spark/SimpleFutureAction.html" title="class in org.apache.spark">SimpleFutureAction</a><R> submitJob(<a href="../../../org/apache/spark/rdd/RDD.html" title="class in org.apache.spark.rdd">RDD</a><T> rdd,
scala.Function1<scala.collection.Iterator<T>,U> processPartition,
scala.collection.Seq<java.lang.Object> partitions,
scala.Function2<java.lang.Object,U,scala.runtime.BoxedUnit> resultHandler,
scala.Function0<R> resultFunc)</pre>
<div class="block">:: Experimental ::
Submit a job for execution and return a FutureJob holding the result.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>rdd</code> - (undocumented)</dd><dd><code>processPartition</code> - (undocumented)</dd><dd><code>partitions</code> - (undocumented)</dd><dd><code>resultHandler</code> - (undocumented)</dd><dd><code>resultFunc</code> - (undocumented)</dd>
<dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
<a name="cancelJobGroup(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>cancelJobGroup</h4>
<pre>public void cancelJobGroup(java.lang.String groupId)</pre>
<div class="block">Cancel active jobs for the specified group. See <code>org.apache.spark.SparkContext.setJobGroup</code>
for more information.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>groupId</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="cancelAllJobs()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>cancelAllJobs</h4>
<pre>public void cancelAllJobs()</pre>
<div class="block">Cancel all jobs that have been scheduled or are running.</div>
</li>
</ul>
<a name="setCheckpointDir(java.lang.String)">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>setCheckpointDir</h4>
<pre>public void setCheckpointDir(java.lang.String directory)</pre>
<div class="block">Set the directory under which RDDs are going to be checkpointed. The directory must
be a HDFS path if running on a cluster.</div>
<dl><dt><span class="strong">Parameters:</span></dt><dd><code>directory</code> - (undocumented)</dd></dl>
</li>
</ul>
<a name="getCheckpointDir()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getCheckpointDir</h4>
<pre>public scala.Option<java.lang.String> getCheckpointDir()</pre>
</li>
</ul>
<a name="defaultParallelism()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>defaultParallelism</h4>
<pre>public int defaultParallelism()</pre>
<div class="block">Default level of parallelism to use when not given by user (e.g. parallelize and makeRDD).</div>
</li>
</ul>
<a name="defaultMinSplits()">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>defaultMinSplits</h4>
<pre>public int defaultMinSplits()</pre>
<div class="block">Default min number of partitions for Hadoop RDDs when not given by user</div>
</li>
</ul>
<a name="defaultMinPartitions()">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>defaultMinPartitions</h4>
<pre>public int defaultMinPartitions()</pre>
<div class="block">Default min number of partitions for Hadoop RDDs when not given by user
Notice that we use math.min so the "defaultMinPartitions" cannot be higher than 2.
The reasons for this are discussed in https://github.com/mesos/spark/pull/718</div>
<dl><dt><span class="strong">Returns:</span></dt><dd>(undocumented)</dd></dl>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<!-- ========= END OF CLASS DATA ========= -->
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar_bottom">
<!-- -->
</a><a href="#skip-navbar_bottom" title="Skip navigation links"></a><a name="navbar_bottom_firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../index-all.html">Index</a></li>
<li><a href="../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../org/apache/spark/SparkConf.html" title="class in org.apache.spark"><span class="strong">Prev Class</span></a></li>
<li><a href="../../../org/apache/spark/SparkContext.DoubleAccumulatorParam$.html" title="class in org.apache.spark"><span class="strong">Next Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../index.html?org/apache/spark/SparkContext.html" target="_top">Frames</a></li>
<li><a href="SparkContext.html" target="_top">No Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../allclasses-noframe.html">All Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary: </li>
<li><a href="#nested_class_summary">Nested</a> | </li>
<li>Field | </li>
<li><a href="#constructor_summary">Constr</a> | </li>
<li><a href="#method_summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail: </li>
<li>Field | </li>
<li><a href="#constructor_detail">Constr</a> | </li>
<li><a href="#method_detail">Method</a></li>
</ul>
</div>
<a name="skip-navbar_bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
<script defer="defer" type="text/javascript" src="../../../lib/jquery.js"></script><script defer="defer" type="text/javascript" src="../../../lib/api-javadocs.js"></script></body>
</html>