aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--examples/src/main/python/streaming/direct_kafka_wordcount.py6
-rw-r--r--examples/src/main/python/streaming/flume_wordcount.py5
-rw-r--r--examples/src/main/python/streaming/kafka_wordcount.py5
-rw-r--r--examples/src/main/python/streaming/mqtt_wordcount.py5
-rw-r--r--examples/src/main/python/streaming/queue_stream.py4
-rw-r--r--python/pyspark/streaming/flume.py4
-rw-r--r--python/pyspark/streaming/kafka.py4
-rw-r--r--python/pyspark/streaming/kinesis.py4
8 files changed, 23 insertions, 14 deletions
diff --git a/examples/src/main/python/streaming/direct_kafka_wordcount.py b/examples/src/main/python/streaming/direct_kafka_wordcount.py
index 6ef188a220..ea20678b9a 100644
--- a/examples/src/main/python/streaming/direct_kafka_wordcount.py
+++ b/examples/src/main/python/streaming/direct_kafka_wordcount.py
@@ -23,8 +23,8 @@
http://kafka.apache.org/documentation.html#quickstart
and then run the example
- `$ bin/spark-submit --jars external/kafka-assembly/target/scala-*/\
- spark-streaming-kafka-assembly-*.jar \
+ `$ bin/spark-submit --jars \
+ external/kafka-assembly/target/scala-*/spark-streaming-kafka-assembly-*.jar \
examples/src/main/python/streaming/direct_kafka_wordcount.py \
localhost:9092 test`
"""
@@ -37,7 +37,7 @@ from pyspark.streaming.kafka import KafkaUtils
if __name__ == "__main__":
if len(sys.argv) != 3:
- print >> sys.stderr, "Usage: direct_kafka_wordcount.py <broker_list> <topic>"
+ print("Usage: direct_kafka_wordcount.py <broker_list> <topic>", file=sys.stderr)
exit(-1)
sc = SparkContext(appName="PythonStreamingDirectKafkaWordCount")
diff --git a/examples/src/main/python/streaming/flume_wordcount.py b/examples/src/main/python/streaming/flume_wordcount.py
index 091b64d8c4..d75bc6daac 100644
--- a/examples/src/main/python/streaming/flume_wordcount.py
+++ b/examples/src/main/python/streaming/flume_wordcount.py
@@ -23,8 +23,9 @@
https://flume.apache.org/documentation.html
and then run the example
- `$ bin/spark-submit --jars external/flume-assembly/target/scala-*/\
- spark-streaming-flume-assembly-*.jar examples/src/main/python/streaming/flume_wordcount.py \
+ `$ bin/spark-submit --jars \
+ external/flume-assembly/target/scala-*/spark-streaming-flume-assembly-*.jar \
+ examples/src/main/python/streaming/flume_wordcount.py \
localhost 12345
"""
from __future__ import print_function
diff --git a/examples/src/main/python/streaming/kafka_wordcount.py b/examples/src/main/python/streaming/kafka_wordcount.py
index b178e7899b..8d697f620f 100644
--- a/examples/src/main/python/streaming/kafka_wordcount.py
+++ b/examples/src/main/python/streaming/kafka_wordcount.py
@@ -23,8 +23,9 @@
http://kafka.apache.org/documentation.html#quickstart
and then run the example
- `$ bin/spark-submit --jars external/kafka-assembly/target/scala-*/\
- spark-streaming-kafka-assembly-*.jar examples/src/main/python/streaming/kafka_wordcount.py \
+ `$ bin/spark-submit --jars \
+ external/kafka-assembly/target/scala-*/spark-streaming-kafka-assembly-*.jar \
+ examples/src/main/python/streaming/kafka_wordcount.py \
localhost:2181 test`
"""
from __future__ import print_function
diff --git a/examples/src/main/python/streaming/mqtt_wordcount.py b/examples/src/main/python/streaming/mqtt_wordcount.py
index 617ce5ea67..abf9c0e21d 100644
--- a/examples/src/main/python/streaming/mqtt_wordcount.py
+++ b/examples/src/main/python/streaming/mqtt_wordcount.py
@@ -26,8 +26,9 @@
http://www.eclipse.org/paho/#getting-started
and then run the example
- `$ bin/spark-submit --jars external/mqtt-assembly/target/scala-*/\
- spark-streaming-mqtt-assembly-*.jar examples/src/main/python/streaming/mqtt_wordcount.py \
+ `$ bin/spark-submit --jars \
+ external/mqtt-assembly/target/scala-*/spark-streaming-mqtt-assembly-*.jar \
+ examples/src/main/python/streaming/mqtt_wordcount.py \
tcp://localhost:1883 foo`
"""
diff --git a/examples/src/main/python/streaming/queue_stream.py b/examples/src/main/python/streaming/queue_stream.py
index dcd6a0fc6f..b3808907f7 100644
--- a/examples/src/main/python/streaming/queue_stream.py
+++ b/examples/src/main/python/streaming/queue_stream.py
@@ -36,8 +36,8 @@ if __name__ == "__main__":
# Create the queue through which RDDs can be pushed to
# a QueueInputDStream
rddQueue = []
- for i in xrange(5):
- rddQueue += [ssc.sparkContext.parallelize([j for j in xrange(1, 1001)], 10)]
+ for i in range(5):
+ rddQueue += [ssc.sparkContext.parallelize([j for j in range(1, 1001)], 10)]
# Create the QueueInputDStream and use it do some processing
inputStream = ssc.queueStream(rddQueue)
diff --git a/python/pyspark/streaming/flume.py b/python/pyspark/streaming/flume.py
index cbb573f226..c0cdc50d8d 100644
--- a/python/pyspark/streaming/flume.py
+++ b/python/pyspark/streaming/flume.py
@@ -31,7 +31,9 @@ __all__ = ['FlumeUtils', 'utf8_decoder']
def utf8_decoder(s):
""" Decode the unicode as UTF-8 """
- return s and s.decode('utf-8')
+ if s is None:
+ return None
+ return s.decode('utf-8')
class FlumeUtils(object):
diff --git a/python/pyspark/streaming/kafka.py b/python/pyspark/streaming/kafka.py
index dc5b7fd878..8a814c64c0 100644
--- a/python/pyspark/streaming/kafka.py
+++ b/python/pyspark/streaming/kafka.py
@@ -29,7 +29,9 @@ __all__ = ['Broker', 'KafkaUtils', 'OffsetRange', 'TopicAndPartition', 'utf8_dec
def utf8_decoder(s):
""" Decode the unicode as UTF-8 """
- return s and s.decode('utf-8')
+ if s is None:
+ return None
+ return s.decode('utf-8')
class KafkaUtils(object):
diff --git a/python/pyspark/streaming/kinesis.py b/python/pyspark/streaming/kinesis.py
index bcfe2703fe..34be5880e1 100644
--- a/python/pyspark/streaming/kinesis.py
+++ b/python/pyspark/streaming/kinesis.py
@@ -26,7 +26,9 @@ __all__ = ['KinesisUtils', 'InitialPositionInStream', 'utf8_decoder']
def utf8_decoder(s):
""" Decode the unicode as UTF-8 """
- return s and s.decode('utf-8')
+ if s is None:
+ return None
+ return s.decode('utf-8')
class KinesisUtils(object):