diff options
author | zsxwing <zsxwing@gmail.com> | 2015-08-07 00:09:58 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2015-08-07 00:09:58 -0700 |
commit | ebfd91c542aaead343cb154277fcf9114382fee7 (patch) | |
tree | 89a15f29d335398b02f37749cfaa7d8c5a28abb9 /core | |
parent | e57d6b56137bf3557efe5acea3ad390c1987b257 (diff) | |
download | spark-ebfd91c542aaead343cb154277fcf9114382fee7.tar.gz spark-ebfd91c542aaead343cb154277fcf9114382fee7.tar.bz2 spark-ebfd91c542aaead343cb154277fcf9114382fee7.zip |
[SPARK-9467][SQL]Add SQLMetric to specialize accumulators to avoid boxing
This PR adds SQLMetric/SQLMetricParam/SQLMetricValue to specialize accumulators to avoid boxing. All SQL metrics should use these classes rather than `Accumulator`.
Author: zsxwing <zsxwing@gmail.com>
Closes #7996 from zsxwing/sql-accu and squashes the following commits:
14a5f0a [zsxwing] Address comments
367ca23 [zsxwing] Use localValue directly to avoid changing Accumulable
42f50c3 [zsxwing] Add SQLMetric to specialize accumulators to avoid boxing
Diffstat (limited to 'core')
-rw-r--r-- | core/src/main/scala/org/apache/spark/Accumulators.scala | 2 | ||||
-rw-r--r-- | core/src/main/scala/org/apache/spark/SparkContext.scala | 15 |
2 files changed, 1 insertions, 16 deletions
diff --git a/core/src/main/scala/org/apache/spark/Accumulators.scala b/core/src/main/scala/org/apache/spark/Accumulators.scala index 462d5c96d4..064246dfa7 100644 --- a/core/src/main/scala/org/apache/spark/Accumulators.scala +++ b/core/src/main/scala/org/apache/spark/Accumulators.scala @@ -257,7 +257,7 @@ GrowableAccumulableParam[R <% Growable[T] with TraversableOnce[T] with Serializa */ class Accumulator[T] private[spark] ( @transient private[spark] val initialValue: T, - private[spark] val param: AccumulatorParam[T], + param: AccumulatorParam[T], name: Option[String], internal: Boolean) extends Accumulable[T, T](initialValue, param, name, internal) { diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 5662686436..9ced44131b 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -1239,21 +1239,6 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli } /** - * Create an [[org.apache.spark.Accumulator]] variable of a given type, with a name for display - * in the Spark UI. Tasks can "add" values to the accumulator using the `+=` method. Only the - * driver can access the accumulator's `value`. The latest local value of such accumulator will be - * sent back to the driver via heartbeats. - * - * @tparam T type that can be added to the accumulator, must be thread safe - */ - private[spark] def internalAccumulator[T](initialValue: T, name: String)( - implicit param: AccumulatorParam[T]): Accumulator[T] = { - val acc = new Accumulator(initialValue, param, Some(name), internal = true) - cleaner.foreach(_.registerAccumulatorForCleanup(acc)) - acc - } - - /** * Create an [[org.apache.spark.Accumulable]] shared variable, to which tasks can add values * with `+=`. Only the driver can access the accumuable's `value`. * @tparam R accumulator result type |