diff options
| -rw-r--r-- | core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala | 9 | ||||
| -rw-r--r-- | spark-class2.cmd | 2 |
2 files changed, 4 insertions, 7 deletions
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala index 07a42f0503..2e51dd5a99 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala @@ -146,12 +146,9 @@ private[spark] class JobProgressListener(val sc: SparkContext) extends SparkList // update duration y.taskTime += taskEnd.taskInfo.duration - taskEnd.taskMetrics.shuffleReadMetrics.foreach { shuffleRead => - y.shuffleRead += shuffleRead.remoteBytesRead - } - - taskEnd.taskMetrics.shuffleWriteMetrics.foreach { shuffleWrite => - y.shuffleWrite += shuffleWrite.shuffleBytesWritten + Option(taskEnd.taskMetrics).foreach { taskMetrics => + taskMetrics.shuffleReadMetrics.foreach { y.shuffleRead += _.remoteBytesRead } + taskMetrics.shuffleWriteMetrics.foreach { y.shuffleWrite += _.shuffleBytesWritten } } } case _ => {} diff --git a/spark-class2.cmd b/spark-class2.cmd index a60c17d050..dc9dadf356 100644 --- a/spark-class2.cmd +++ b/spark-class2.cmd @@ -17,7 +17,7 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem -set SCALA_VERSION=2.9.3 +set SCALA_VERSION=2.10 rem Figure out where the Spark framework is installed set FWDIR=%~dp0 |