diff options
author | cody koeninger <cody@koeninger.org> | 2016-05-11 12:15:41 -0700 |
---|---|---|
committer | Reynold Xin <rxin@databricks.com> | 2016-05-11 12:15:41 -0700 |
commit | 89e67d6667d5f8be9c6fb6c120fbcd350ae2950d (patch) | |
tree | 670699f20dcc785e1889c8c1afc4db1a0b2b11ee /python/pyspark/streaming | |
parent | 6d0368ab8d1043735e5fe89f801aae1c6826876c (diff) | |
download | spark-89e67d6667d5f8be9c6fb6c120fbcd350ae2950d.tar.gz spark-89e67d6667d5f8be9c6fb6c120fbcd350ae2950d.tar.bz2 spark-89e67d6667d5f8be9c6fb6c120fbcd350ae2950d.zip |
[SPARK-15085][STREAMING][KAFKA] Rename streaming-kafka artifact
## What changes were proposed in this pull request?
Renaming the streaming-kafka artifact to include kafka version, in anticipation of needing a different artifact for later kafka versions
## How was this patch tested?
Unit tests
Author: cody koeninger <cody@koeninger.org>
Closes #12946 from koeninger/SPARK-15085.
Diffstat (limited to 'python/pyspark/streaming')
-rw-r--r-- | python/pyspark/streaming/kafka.py | 6 | ||||
-rw-r--r-- | python/pyspark/streaming/tests.py | 6 |
2 files changed, 6 insertions, 6 deletions
diff --git a/python/pyspark/streaming/kafka.py b/python/pyspark/streaming/kafka.py index 02a88699a2..015ca77dbf 100644 --- a/python/pyspark/streaming/kafka.py +++ b/python/pyspark/streaming/kafka.py @@ -208,13 +208,13 @@ ________________________________________________________________________________ 1. Include the Kafka library and its dependencies with in the spark-submit command as - $ bin/spark-submit --packages org.apache.spark:spark-streaming-kafka:%s ... + $ bin/spark-submit --packages org.apache.spark:spark-streaming-kafka-0-8:%s ... 2. Download the JAR of the artifact from Maven Central http://search.maven.org/, - Group Id = org.apache.spark, Artifact Id = spark-streaming-kafka-assembly, Version = %s. + Group Id = org.apache.spark, Artifact Id = spark-streaming-kafka-0-8-assembly, Version = %s. Then, include the jar in the spark-submit command as - $ bin/spark-submit --jars <spark-streaming-kafka-assembly.jar> ... + $ bin/spark-submit --jars <spark-streaming-kafka-0-8-assembly.jar> ... ________________________________________________________________________________________________ diff --git a/python/pyspark/streaming/tests.py b/python/pyspark/streaming/tests.py index f27628c895..360ba1e716 100644 --- a/python/pyspark/streaming/tests.py +++ b/python/pyspark/streaming/tests.py @@ -1476,13 +1476,13 @@ def search_jar(dir, name_prefix): def search_kafka_assembly_jar(): SPARK_HOME = os.environ["SPARK_HOME"] - kafka_assembly_dir = os.path.join(SPARK_HOME, "external/kafka-assembly") - jars = search_jar(kafka_assembly_dir, "spark-streaming-kafka-assembly") + kafka_assembly_dir = os.path.join(SPARK_HOME, "external/kafka-0-8-assembly") + jars = search_jar(kafka_assembly_dir, "spark-streaming-kafka-0-8-assembly") if not jars: raise Exception( ("Failed to find Spark Streaming kafka assembly jar in %s. " % kafka_assembly_dir) + "You need to build Spark with " - "'build/sbt assembly/package streaming-kafka-assembly/assembly' or " + "'build/sbt assembly/package streaming-kafka-0-8-assembly/assembly' or " "'build/mvn package' before running this test.") elif len(jars) > 1: raise Exception(("Found multiple Spark Streaming Kafka assembly JARs: %s; please " |