aboutsummaryrefslogtreecommitdiff
path: root/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
diff options
context:
space:
mode:
Diffstat (limited to 'extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala')
-rw-r--r--extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala13
1 files changed, 5 insertions, 8 deletions
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index fffd90de08..32da0858d1 100644
--- a/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++ b/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -65,6 +65,10 @@ import org.apache.log4j.Level
* org.apache.spark.examples.streaming.KinesisWordCountASL mySparkStream \
* https://kinesis.us-east-1.amazonaws.com
*
+ *
+ * Note that number of workers/threads should be 1 more than the number of receivers.
+ * This leaves one thread available for actually processing the data.
+ *
* There is a companion helper class below called KinesisWordCountProducerASL which puts
* dummy data onto the Kinesis stream.
* Usage instructions for KinesisWordCountProducerASL are provided in that class definition.
@@ -97,17 +101,10 @@ private object KinesisWordCountASL extends Logging {
/* In this example, we're going to create 1 Kinesis Worker/Receiver/DStream for each shard. */
val numStreams = numShards
- /*
- * numSparkThreads should be 1 more thread than the number of receivers.
- * This leaves one thread available for actually processing the data.
- */
- val numSparkThreads = numStreams + 1
-
/* Setup the and SparkConfig and StreamingContext */
/* Spark Streaming batch interval */
- val batchInterval = Milliseconds(2000)
+ val batchInterval = Milliseconds(2000)
val sparkConfig = new SparkConf().setAppName("KinesisWordCount")
- .setMaster(s"local[$numSparkThreads]")
val ssc = new StreamingContext(sparkConfig, batchInterval)
/* Kinesis checkpoint interval. Same as batchInterval for this example. */