diff options
| author | Tathagata Das <tathagata.das1565@gmail.com> | 2013-12-22 15:23:35 -0800 |
|---|---|---|
| committer | Tathagata Das <tathagata.das1565@gmail.com> | 2013-12-22 15:23:35 -0800 |
| commit | d91ec6f8ea3ab6683af92eddfedd9ea6c0710f00 (patch) | |
| tree | 1254875e3b8fc7c37ab3279efa8b8e6a250e512a /streaming | |
| parent | 984c5824876e0daceb8a74af57593926faf727ce (diff) | |
| parent | 3ddbdbfbc71486cd5076d875f82796a880d2dccb (diff) | |
| download | spark-d91ec6f8ea3ab6683af92eddfedd9ea6c0710f00.tar.gz spark-d91ec6f8ea3ab6683af92eddfedd9ea6c0710f00.tar.bz2 spark-d91ec6f8ea3ab6683af92eddfedd9ea6c0710f00.zip | |
Merge branch 'scheduler-update' into filestream-fix
Diffstat (limited to 'streaming')
| -rw-r--r-- | streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala | 4 | ||||
| -rw-r--r-- | streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobSet.scala | 3 |
2 files changed, 6 insertions, 1 deletions
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala index 33c5322358..9511ccfbed 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala @@ -24,7 +24,8 @@ import scala.collection.mutable.HashSet import org.apache.spark.streaming._ /** - * This class drives the generation of Spark jobs from the DStreams. + * This class schedules jobs to be run on Spark. It uses the JobGenerator to generate + * the jobs and runs them using a thread pool. Number of threads */ private[streaming] class JobScheduler(val ssc: StreamingContext) extends Logging { @@ -91,6 +92,7 @@ class JobScheduler(val ssc: StreamingContext) extends Logging { } } + private[streaming] class JobHandler(job: Job) extends Runnable { def run() { beforeJobStart(job) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobSet.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobSet.scala index 05233d095b..cf7431a8a3 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobSet.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobSet.scala @@ -20,6 +20,9 @@ package org.apache.spark.streaming.scheduler import scala.collection.mutable.HashSet import org.apache.spark.streaming.Time +/** Class representing a set of Jobs + * belong to the same batch. + */ private[streaming] case class JobSet(time: Time, jobs: Seq[Job]) { |