diff options
author | Sean Owen <sowen@cloudera.com> | 2016-03-09 18:27:44 +0000 |
---|---|---|
committer | Sean Owen <sowen@cloudera.com> | 2016-03-09 18:27:44 +0000 |
commit | 256704c771d301700af9ebf0d180c1ba7c4116c0 (patch) | |
tree | f9be79919b5c6ec4847c24a086fa844555e2cd12 | |
parent | 7791d0c3a9bdfe73e071266846f9ab1491fce50c (diff) | |
download | spark-256704c771d301700af9ebf0d180c1ba7c4116c0.tar.gz spark-256704c771d301700af9ebf0d180c1ba7c4116c0.tar.bz2 spark-256704c771d301700af9ebf0d180c1ba7c4116c0.zip |
[SPARK-13595][BUILD] Move docker, extras modules into external
## What changes were proposed in this pull request?
Move `docker` dirs out of top level into `external/`; move `extras/*` into `external/`
## How was this patch tested?
This is tested with Jenkins tests.
Author: Sean Owen <sowen@cloudera.com>
Closes #11523 from srowen/SPARK-13595.
-rw-r--r-- | dev/sparktestsupport/modules.py | 4 | ||||
-rw-r--r-- | docs/streaming-kinesis-integration.md | 10 | ||||
-rw-r--r-- | external/docker-integration-tests/pom.xml (renamed from docker-integration-tests/pom.xml) | 2 | ||||
-rw-r--r-- | external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala (renamed from docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala) | 0 | ||||
-rw-r--r-- | external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala (renamed from docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala) | 0 | ||||
-rw-r--r-- | external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala (renamed from docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala) | 0 | ||||
-rw-r--r-- | external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala (renamed from docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala) | 0 | ||||
-rw-r--r-- | external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala (renamed from docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala) | 0 | ||||
-rw-r--r-- | external/docker/README.md (renamed from docker/README.md) | 0 | ||||
-rwxr-xr-x | external/docker/build (renamed from docker/build) | 0 | ||||
-rw-r--r-- | external/docker/spark-mesos/Dockerfile (renamed from docker/spark-mesos/Dockerfile) | 0 | ||||
-rw-r--r-- | external/docker/spark-test/README.md (renamed from docker/spark-test/README.md) | 0 | ||||
-rw-r--r-- | external/docker/spark-test/base/Dockerfile (renamed from docker/spark-test/base/Dockerfile) | 0 | ||||
-rwxr-xr-x | external/docker/spark-test/build (renamed from docker/spark-test/build) | 0 | ||||
-rw-r--r-- | external/docker/spark-test/master/Dockerfile (renamed from docker/spark-test/master/Dockerfile) | 0 | ||||
-rwxr-xr-x | external/docker/spark-test/master/default_cmd (renamed from docker/spark-test/master/default_cmd) | 0 | ||||
-rw-r--r-- | external/docker/spark-test/worker/Dockerfile (renamed from docker/spark-test/worker/Dockerfile) | 0 | ||||
-rwxr-xr-x | external/docker/spark-test/worker/default_cmd (renamed from docker/spark-test/worker/default_cmd) | 0 | ||||
-rw-r--r-- | external/java8-tests/README.md (renamed from extras/java8-tests/README.md) | 0 | ||||
-rw-r--r-- | external/java8-tests/pom.xml (renamed from extras/java8-tests/pom.xml) | 0 | ||||
-rw-r--r-- | external/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java (renamed from extras/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java) | 0 | ||||
-rw-r--r-- | external/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java (renamed from extras/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java) | 0 | ||||
-rw-r--r-- | external/java8-tests/src/test/resources/log4j.properties (renamed from extras/java8-tests/src/test/resources/log4j.properties) | 0 | ||||
-rw-r--r-- | external/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala (renamed from extras/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl-assembly/pom.xml (renamed from extras/kinesis-asl-assembly/pom.xml) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/pom.xml (renamed from extras/kinesis-asl/pom.xml) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java (renamed from extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py (renamed from extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py) | 4 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/resources/log4j.properties (renamed from extras/kinesis-asl/src/main/resources/log4j.properties) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala (renamed from extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala (renamed from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala (renamed from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala (renamed from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala (renamed from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala (renamed from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala (renamed from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala (renamed from extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java (renamed from extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/test/resources/log4j.properties (renamed from extras/kinesis-asl/src/test/resources/log4j.properties) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala (renamed from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala (renamed from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala (renamed from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala (renamed from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala (renamed from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala) | 0 | ||||
-rw-r--r-- | external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala (renamed from extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala) | 0 | ||||
-rw-r--r-- | external/spark-ganglia-lgpl/pom.xml (renamed from extras/spark-ganglia-lgpl/pom.xml) | 0 | ||||
-rw-r--r-- | external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala (renamed from extras/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala) | 0 | ||||
-rw-r--r-- | extras/README.md | 1 | ||||
-rw-r--r-- | pom.xml | 10 | ||||
-rw-r--r-- | python/pyspark/streaming/tests.py | 2 |
50 files changed, 16 insertions, 17 deletions
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index e4f2edaf95..1781de4c65 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -195,8 +195,8 @@ streaming_kinesis_asl = Module( name="streaming-kinesis-asl", dependencies=[], source_file_regexes=[ - "extras/kinesis-asl/", - "extras/kinesis-asl-assembly/", + "external/kinesis-asl/", + "external/kinesis-asl-assembly/", ], build_profile_flags=[ "-Pkinesis-asl", diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md index 2a868e8bca..5b9a7554d2 100644 --- a/docs/streaming-kinesis-integration.md +++ b/docs/streaming-kinesis-integration.md @@ -37,7 +37,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m [region name], [initial position], [checkpoint interval], StorageLevel.MEMORY_AND_DISK_2) See the [API docs](api/scala/index.html#org.apache.spark.streaming.kinesis.KinesisUtils$) - and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala). Refer to the [Running the Example](#running-the-example) subsection for instructions on how to run the example. + and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala). Refer to the [Running the Example](#running-the-example) subsection for instructions on how to run the example. </div> <div data-lang="java" markdown="1"> @@ -50,7 +50,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m [region name], [initial position], [checkpoint interval], StorageLevel.MEMORY_AND_DISK_2); See the [API docs](api/java/index.html?org/apache/spark/streaming/kinesis/KinesisUtils.html) - and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java). Refer to the [Running the Example](#running-the-example) subsection for instructions to run the example. + and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java). Refer to the [Running the Example](#running-the-example) subsection for instructions to run the example. </div> <div data-lang="python" markdown="1"> @@ -61,7 +61,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m [region name], [initial position], [checkpoint interval], StorageLevel.MEMORY_AND_DISK_2) See the [API docs](api/python/pyspark.streaming.html#pyspark.streaming.kinesis.KinesisUtils) - and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py). Refer to the [Running the Example](#running-the-example) subsection for instructions to run the example. + and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py). Refer to the [Running the Example](#running-the-example) subsection for instructions to run the example. </div> </div> @@ -190,9 +190,9 @@ To run the example, </div> <div data-lang="python" markdown="1"> - bin/spark-submit --jars extras/kinesis-asl/target/scala-*/\ + bin/spark-submit --jars external/kinesis-asl/target/scala-*/\ spark-streaming-kinesis-asl-assembly_*.jar \ - extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \ + external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \ [Kinesis app name] [Kinesis stream name] [endpoint URL] [region name] </div> diff --git a/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 048e58d888..1764aa9465 100644 --- a/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -23,7 +23,7 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> + <relativePath>../../pom.xml</relativePath> </parent> <artifactId>spark-docker-integration-tests_2.11</artifactId> diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala index f73231fc80..f73231fc80 100644 --- a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala index c68e4dc493..c68e4dc493 100644 --- a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MySQLIntegrationSuite.scala diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala index 8a0f938f7e..8a0f938f7e 100644 --- a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala index d55cdcf28b..d55cdcf28b 100644 --- a/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala diff --git a/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala index fda377e032..fda377e032 100644 --- a/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala diff --git a/docker/README.md b/external/docker/README.md index 40ba9c3065..40ba9c3065 100644 --- a/docker/README.md +++ b/external/docker/README.md diff --git a/docker/build b/external/docker/build index 253a2fc8dd..253a2fc8dd 100755 --- a/docker/build +++ b/external/docker/build diff --git a/docker/spark-mesos/Dockerfile b/external/docker/spark-mesos/Dockerfile index fb3f267fe5..fb3f267fe5 100644 --- a/docker/spark-mesos/Dockerfile +++ b/external/docker/spark-mesos/Dockerfile diff --git a/docker/spark-test/README.md b/external/docker/spark-test/README.md index ec0baf6e6d..ec0baf6e6d 100644 --- a/docker/spark-test/README.md +++ b/external/docker/spark-test/README.md diff --git a/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile index 76f550f886..76f550f886 100644 --- a/docker/spark-test/base/Dockerfile +++ b/external/docker/spark-test/base/Dockerfile diff --git a/docker/spark-test/build b/external/docker/spark-test/build index 6f9e197433..6f9e197433 100755 --- a/docker/spark-test/build +++ b/external/docker/spark-test/build diff --git a/docker/spark-test/master/Dockerfile b/external/docker/spark-test/master/Dockerfile index f729534ab6..f729534ab6 100644 --- a/docker/spark-test/master/Dockerfile +++ b/external/docker/spark-test/master/Dockerfile diff --git a/docker/spark-test/master/default_cmd b/external/docker/spark-test/master/default_cmd index 5a7da3446f..5a7da3446f 100755 --- a/docker/spark-test/master/default_cmd +++ b/external/docker/spark-test/master/default_cmd diff --git a/docker/spark-test/worker/Dockerfile b/external/docker/spark-test/worker/Dockerfile index 890febe7b6..890febe7b6 100644 --- a/docker/spark-test/worker/Dockerfile +++ b/external/docker/spark-test/worker/Dockerfile diff --git a/docker/spark-test/worker/default_cmd b/external/docker/spark-test/worker/default_cmd index 31b06cb0eb..31b06cb0eb 100755 --- a/docker/spark-test/worker/default_cmd +++ b/external/docker/spark-test/worker/default_cmd diff --git a/extras/java8-tests/README.md b/external/java8-tests/README.md index dc9e87f2ee..dc9e87f2ee 100644 --- a/extras/java8-tests/README.md +++ b/external/java8-tests/README.md diff --git a/extras/java8-tests/pom.xml b/external/java8-tests/pom.xml index 0ad9c5303a..0ad9c5303a 100644 --- a/extras/java8-tests/pom.xml +++ b/external/java8-tests/pom.xml diff --git a/extras/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java b/external/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java index c0b58e713f..c0b58e713f 100644 --- a/extras/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java +++ b/external/java8-tests/src/test/java/org/apache/spark/Java8APISuite.java diff --git a/extras/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java b/external/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java index 604d818ef1..604d818ef1 100644 --- a/extras/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java +++ b/external/java8-tests/src/test/java/org/apache/spark/streaming/Java8APISuite.java diff --git a/extras/java8-tests/src/test/resources/log4j.properties b/external/java8-tests/src/test/resources/log4j.properties index eb3b1999eb..eb3b1999eb 100644 --- a/extras/java8-tests/src/test/resources/log4j.properties +++ b/external/java8-tests/src/test/resources/log4j.properties diff --git a/extras/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala b/external/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala index fa0681db41..fa0681db41 100644 --- a/extras/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala +++ b/external/java8-tests/src/test/scala/org/apache/spark/JDK8ScalaSuite.scala diff --git a/extras/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index d1c38c7ca5..d1c38c7ca5 100644 --- a/extras/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml diff --git a/extras/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 935155eb5d..935155eb5d 100644 --- a/extras/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml diff --git a/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java index 5dc825dfdc..5dc825dfdc 100644 --- a/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java +++ b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java diff --git a/extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py index 51f8c5ca66..4d7fc9a549 100644 --- a/extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py +++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py @@ -34,9 +34,9 @@ $ export AWS_SECRET_KEY=<your-secret-key> # run the example - $ bin/spark-submit -jar extras/kinesis-asl/target/scala-*/\ + $ bin/spark-submit -jar external/kinesis-asl/target/scala-*/\ spark-streaming-kinesis-asl-assembly_*.jar \ - extras/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \ + external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \ myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com There is a companion helper class called KinesisWordProducerASL which puts dummy data diff --git a/extras/kinesis-asl/src/main/resources/log4j.properties b/external/kinesis-asl/src/main/resources/log4j.properties index 6cdc9286c5..6cdc9286c5 100644 --- a/extras/kinesis-asl/src/main/resources/log4j.properties +++ b/external/kinesis-asl/src/main/resources/log4j.properties diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala index 6a73bc0e30..6a73bc0e30 100644 --- a/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala index 3996f168e6..3996f168e6 100644 --- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDD.scala diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala index 1ca6d4302c..1ca6d4302c 100644 --- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointer.scala diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala index 5223c81a8e..5223c81a8e 100644 --- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.scala diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala index 48ee2a9597..48ee2a9597 100644 --- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisReceiver.scala diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala index b5b76cb92d..b5b76cb92d 100644 --- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisRecordProcessor.scala diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala index 0ace453ee9..0ace453ee9 100644 --- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisTestUtils.scala diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala index 15ac588b82..15ac588b82 100644 --- a/extras/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala +++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtils.scala diff --git a/extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java b/external/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java index 5c2371c543..5c2371c543 100644 --- a/extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java +++ b/external/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java diff --git a/extras/kinesis-asl/src/test/resources/log4j.properties b/external/kinesis-asl/src/test/resources/log4j.properties index edbecdae92..edbecdae92 100644 --- a/extras/kinesis-asl/src/test/resources/log4j.properties +++ b/external/kinesis-asl/src/test/resources/log4j.properties diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala index fdb270eaad..fdb270eaad 100644 --- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KPLBasedKinesisTestUtils.scala diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala index 2555332d22..2555332d22 100644 --- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala index e1499a8220..e1499a8220 100644 --- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisCheckpointerSuite.scala diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala index ee428f31d6..ee428f31d6 100644 --- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisFunSuite.scala diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala index fd15b6ccdc..fd15b6ccdc 100644 --- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala index ca5d13da46..ca5d13da46 100644 --- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala +++ b/external/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisStreamSuite.scala diff --git a/extras/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index bfb92791de..bfb92791de 100644 --- a/extras/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml diff --git a/extras/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala b/external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala index 3b1880e143..3b1880e143 100644 --- a/extras/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala +++ b/external/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala diff --git a/extras/README.md b/extras/README.md deleted file mode 100644 index 1b4174b7d5..0000000000 --- a/extras/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory contains build components not included by default in Spark's build. @@ -99,7 +99,7 @@ <module>sql/catalyst</module> <module>sql/core</module> <module>sql/hive</module> - <module>docker-integration-tests</module> + <module>external/docker-integration-tests</module> <module>assembly</module> <module>external/twitter</module> <module>external/flume</module> @@ -2354,7 +2354,7 @@ <profile> <id>spark-ganglia-lgpl</id> <modules> - <module>extras/spark-ganglia-lgpl</module> + <module>external/spark-ganglia-lgpl</module> </modules> </profile> @@ -2362,8 +2362,8 @@ <profile> <id>kinesis-asl</id> <modules> - <module>extras/kinesis-asl</module> - <module>extras/kinesis-asl-assembly</module> + <module>external/kinesis-asl</module> + <module>external/kinesis-asl-assembly</module> </modules> </profile> @@ -2387,7 +2387,7 @@ </build> <modules> - <module>extras/java8-tests</module> + <module>external/java8-tests</module> </modules> </profile> diff --git a/python/pyspark/streaming/tests.py b/python/pyspark/streaming/tests.py index b33e8252a7..469c068134 100644 --- a/python/pyspark/streaming/tests.py +++ b/python/pyspark/streaming/tests.py @@ -1619,7 +1619,7 @@ def search_mqtt_test_jar(): def search_kinesis_asl_assembly_jar(): SPARK_HOME = os.environ["SPARK_HOME"] - kinesis_asl_assembly_dir = os.path.join(SPARK_HOME, "extras/kinesis-asl-assembly") + kinesis_asl_assembly_dir = os.path.join(SPARK_HOME, "external/kinesis-asl-assembly") jars = search_jar(kinesis_asl_assembly_dir, "spark-streaming-kinesis-asl-assembly") if not jars: return None |