diff options
| author | Tathagata Das <tathagata.das1565@gmail.com> | 2013-12-18 15:35:24 -0800 |
|---|---|---|
| committer | Tathagata Das <tathagata.das1565@gmail.com> | 2013-12-18 15:35:24 -0800 |
| commit | b80ec05635132f96772545803a10a1bbfa1250e7 (patch) | |
| tree | 999562e0dc469fa2f767c43b884af5f78a8d6b34 /core/src/main/scala/org/apache | |
| parent | 097e120c0c4132f007bfd0b0254b362ee9a02d8f (diff) | |
| download | spark-b80ec05635132f96772545803a10a1bbfa1250e7.tar.gz spark-b80ec05635132f96772545803a10a1bbfa1250e7.tar.bz2 spark-b80ec05635132f96772545803a10a1bbfa1250e7.zip | |
Added StatsReportListener to generate processing time statistics across multiple batches.
Diffstat (limited to 'core/src/main/scala/org/apache')
| -rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala | 5 |
1 files changed, 1 insertions, 4 deletions
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala index 2c5d87419d..ee63b3c4a1 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala @@ -131,8 +131,8 @@ object StatsReportListener extends Logging { def showDistribution(heading: String, d: Distribution, formatNumber: Double => String) { val stats = d.statCounter - logInfo(heading + stats) val quantiles = d.getQuantiles(probabilities).map{formatNumber} + logInfo(heading + stats) logInfo(percentilesHeader) logInfo("\t" + quantiles.mkString("\t")) } @@ -173,8 +173,6 @@ object StatsReportListener extends Logging { showMillisDistribution(heading, extractLongDistribution(stage, getMetric)) } - - val seconds = 1000L val minutes = seconds * 60 val hours = minutes * 60 @@ -198,7 +196,6 @@ object StatsReportListener extends Logging { } - case class RuntimePercentage(executorPct: Double, fetchPct: Option[Double], other: Double) object RuntimePercentage { def apply(totalTime: Long, metrics: TaskMetrics): RuntimePercentage = { |