From 35b6358a7c4e9558789577e07c1953c9008d3e9c Mon Sep 17 00:00:00 2001 From: Ankur Dave Date: Wed, 26 Oct 2011 21:07:17 +0000 Subject: Report errors in tasks to the driver via a Mesos status update When a task throws an exception, the Spark executor previously just logged it to a local file on the slave and exited. This commit causes Spark to also report the exception back to the driver using a Mesos status update, so the user doesn't have to look through a log file on the slave. Here's what the reporting currently looks like: # ./run spark.examples.ExceptionHandlingTest master@203.0.113.1:5050 [...] 11/10/26 21:04:13 INFO spark.SimpleJob: Lost TID 1 (task 0:1) 11/10/26 21:04:13 INFO spark.SimpleJob: Loss was due to java.lang.Exception: Testing exception handling [...] 11/10/26 21:04:16 INFO spark.SparkContext: Job finished in 5.988547328 s --- .../scala/spark/examples/ExceptionHandlingTest.scala | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 examples/src/main/scala/spark/examples/ExceptionHandlingTest.scala (limited to 'examples') diff --git a/examples/src/main/scala/spark/examples/ExceptionHandlingTest.scala b/examples/src/main/scala/spark/examples/ExceptionHandlingTest.scala new file mode 100644 index 0000000000..46f658eab2 --- /dev/null +++ b/examples/src/main/scala/spark/examples/ExceptionHandlingTest.scala @@ -0,0 +1,18 @@ +package spark.examples + +import spark.SparkContext + +object ExceptionHandlingTest { + def main(args: Array[String]) { + if (args.length == 0) { + System.err.println("Usage: ExceptionHandlingTest ") + System.exit(1) + } + + val sc = new SparkContext(args(0), "ExceptionHandlingTest") + sc.parallelize(0 until sc.defaultParallelism).foreach { i => + if (Math.random > 0.75) + throw new Exception("Testing exception handling") + } + } +} -- cgit v1.2.3