aboutsummaryrefslogtreecommitdiff
path: root/examples
diff options
context:
space:
mode:
authorAnkur Dave <ankurdave@gmail.com>2011-10-26 21:07:17 +0000
committerAnkur Dave <ankurdave@gmail.com>2011-11-14 01:54:53 +0000
commit35b6358a7c4e9558789577e07c1953c9008d3e9c (patch)
tree093262d22a10a948754193831ba69ca3e57404f2 /examples
parent07532021fee9e2d27ee954b21c30830687478d8b (diff)
downloadspark-35b6358a7c4e9558789577e07c1953c9008d3e9c.tar.gz
spark-35b6358a7c4e9558789577e07c1953c9008d3e9c.tar.bz2
spark-35b6358a7c4e9558789577e07c1953c9008d3e9c.zip
Report errors in tasks to the driver via a Mesos status update
When a task throws an exception, the Spark executor previously just logged it to a local file on the slave and exited. This commit causes Spark to also report the exception back to the driver using a Mesos status update, so the user doesn't have to look through a log file on the slave. Here's what the reporting currently looks like: # ./run spark.examples.ExceptionHandlingTest master@203.0.113.1:5050 [...] 11/10/26 21:04:13 INFO spark.SimpleJob: Lost TID 1 (task 0:1) 11/10/26 21:04:13 INFO spark.SimpleJob: Loss was due to java.lang.Exception: Testing exception handling [...] 11/10/26 21:04:16 INFO spark.SparkContext: Job finished in 5.988547328 s
Diffstat (limited to 'examples')
-rw-r--r--examples/src/main/scala/spark/examples/ExceptionHandlingTest.scala18
1 files changed, 18 insertions, 0 deletions
diff --git a/examples/src/main/scala/spark/examples/ExceptionHandlingTest.scala b/examples/src/main/scala/spark/examples/ExceptionHandlingTest.scala
new file mode 100644
index 0000000000..46f658eab2
--- /dev/null
+++ b/examples/src/main/scala/spark/examples/ExceptionHandlingTest.scala
@@ -0,0 +1,18 @@
+package spark.examples
+
+import spark.SparkContext
+
+object ExceptionHandlingTest {
+ def main(args: Array[String]) {
+ if (args.length == 0) {
+ System.err.println("Usage: ExceptionHandlingTest <host>")
+ System.exit(1)
+ }
+
+ val sc = new SparkContext(args(0), "ExceptionHandlingTest")
+ sc.parallelize(0 until sc.defaultParallelism).foreach { i =>
+ if (Math.random > 0.75)
+ throw new Exception("Testing exception handling")
+ }
+ }
+}