diff options
| author | Shivaram Venkataraman <shivaram@cs.berkeley.edu> | 2014-08-10 12:44:17 -0700 |
|---|---|---|
| committer | Shivaram Venkataraman <shivaram@cs.berkeley.edu> | 2014-08-10 12:44:17 -0700 |
| commit | 1d03a26a4895c24ebfab1a3cf6656af75cb53003 (patch) | |
| tree | 0b791ef42229acb03a85df1fd3770dd66e6ebdc7 | |
| parent | 3570119c34ab8d61507e7703a171b742fb0957d4 (diff) | |
| download | spark-1d03a26a4895c24ebfab1a3cf6656af75cb53003.tar.gz spark-1d03a26a4895c24ebfab1a3cf6656af75cb53003.tar.bz2 spark-1d03a26a4895c24ebfab1a3cf6656af75cb53003.zip | |
[SPARK-2950] Add gc time and shuffle write time to JobLogger
The JobLogger is very useful for performing offline performance profiling of Spark jobs. GC Time and Shuffle Write time are available in TaskMetrics but are currently missed from the JobLogger output. This patch adds these two fields.
~~Since this is a small change, I didn't create a JIRA. Let me know if I should do that.~~
cc kayousterhout
Author: Shivaram Venkataraman <shivaram@cs.berkeley.edu>
Closes #1869 from shivaram/job-logger and squashes the following commits:
1b709fc [Shivaram Venkataraman] Add a space before GC_TIME
c418105 [Shivaram Venkataraman] Add gc time and shuffle write time to JobLogger
| -rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala | 9 |
1 files changed, 6 insertions, 3 deletions
diff --git a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala b/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala index 47dd112f68..4d6b5c8188 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/JobLogger.scala @@ -162,6 +162,7 @@ class JobLogger(val user: String, val logDirName: String) extends SparkListener " START_TIME=" + taskInfo.launchTime + " FINISH_TIME=" + taskInfo.finishTime + " EXECUTOR_ID=" + taskInfo.executorId + " HOST=" + taskMetrics.hostname val executorRunTime = " EXECUTOR_RUN_TIME=" + taskMetrics.executorRunTime + val gcTime = " GC_TIME=" + taskMetrics.jvmGCTime val inputMetrics = taskMetrics.inputMetrics match { case Some(metrics) => " READ_METHOD=" + metrics.readMethod.toString + @@ -179,11 +180,13 @@ class JobLogger(val user: String, val logDirName: String) extends SparkListener case None => "" } val writeMetrics = taskMetrics.shuffleWriteMetrics match { - case Some(metrics) => " SHUFFLE_BYTES_WRITTEN=" + metrics.shuffleBytesWritten + case Some(metrics) => + " SHUFFLE_BYTES_WRITTEN=" + metrics.shuffleBytesWritten + + " SHUFFLE_WRITE_TIME=" + metrics.shuffleWriteTime case None => "" } - stageLogInfo(stageId, status + info + executorRunTime + inputMetrics + shuffleReadMetrics + - writeMetrics) + stageLogInfo(stageId, status + info + executorRunTime + gcTime + inputMetrics + + shuffleReadMetrics + writeMetrics) } /** |