diff options
| author | Ryan Williams <ryan.blake.williams@gmail.com> | 2016-12-21 16:37:20 -0800 |
|---|---|---|
| committer | Marcelo Vanzin <vanzin@cloudera.com> | 2016-12-21 16:37:20 -0800 |
| commit | afd9bc1d8a85adf88c412d8bc75e46e7ecb4bcdd (patch) | |
| tree | e7d9c3e6b0a055c692de05c75f7532cdeca4269b | |
| parent | 95efc895e929701a605313b87ad0cd91edee2f81 (diff) | |
| download | spark-afd9bc1d8a85adf88c412d8bc75e46e7ecb4bcdd.tar.gz spark-afd9bc1d8a85adf88c412d8bc75e46e7ecb4bcdd.tar.bz2 spark-afd9bc1d8a85adf88c412d8bc75e46e7ecb4bcdd.zip | |
[SPARK-17807][CORE] split test-tags into test-JAR
Remove spark-tag's compile-scope dependency (and, indirectly, spark-core's compile-scope transitive-dependency) on scalatest by splitting test-oriented tags into spark-tags' test JAR.
Alternative to #16303.
Author: Ryan Williams <ryan.blake.williams@gmail.com>
Closes #16311 from ryan-williams/tt.
30 files changed, 272 insertions, 9 deletions
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index b63c0ca173..8657af744c 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -91,6 +91,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 5fc92afccb..24c10fb1dd 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -70,6 +70,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 9fcd636c88..5e5a80bd44 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -50,6 +50,17 @@ <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <!-- Provided dependencies --> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 7f57108584..bcd26d4352 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -39,6 +39,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 81191cb8f4..09f6fa12b9 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -34,14 +34,6 @@ <sbt.project.name>tags</sbt.project.name> </properties> - <dependencies> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.binary.version}</artifactId> - <scope>compile</scope> - </dependency> - </dependencies> - <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> diff --git a/common/tags/src/main/java/org/apache/spark/tags/DockerTest.java b/common/tags/src/test/java/org/apache/spark/tags/DockerTest.java index 0fecf3b8f9..0fecf3b8f9 100644 --- a/common/tags/src/main/java/org/apache/spark/tags/DockerTest.java +++ b/common/tags/src/test/java/org/apache/spark/tags/DockerTest.java diff --git a/common/tags/src/main/java/org/apache/spark/tags/ExtendedHiveTest.java b/common/tags/src/test/java/org/apache/spark/tags/ExtendedHiveTest.java index 83279e5e93..83279e5e93 100644 --- a/common/tags/src/main/java/org/apache/spark/tags/ExtendedHiveTest.java +++ b/common/tags/src/test/java/org/apache/spark/tags/ExtendedHiveTest.java diff --git a/common/tags/src/main/java/org/apache/spark/tags/ExtendedYarnTest.java b/common/tags/src/test/java/org/apache/spark/tags/ExtendedYarnTest.java index 108300168e..108300168e 100644 --- a/common/tags/src/main/java/org/apache/spark/tags/ExtendedYarnTest.java +++ b/common/tags/src/test/java/org/apache/spark/tags/ExtendedYarnTest.java diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index d00bddf551..dc19f4ad5f 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -39,6 +39,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.twitter</groupId> <artifactId>chill_${scala.binary.version}</artifactId> diff --git a/core/pom.xml b/core/pom.xml index 8ee88cc730..97a463abbe 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -337,6 +337,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-crypto</artifactId> diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 6741932f3d..8948df2da8 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -96,7 +96,7 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> - <version>${project.version}</version> + <type>test-jar</type> <scope>test</scope> </dependency> <dependency> diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 83619ccf25..6d547c46d6 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -93,6 +93,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 4a40f8ba28..46901d64ed 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -69,6 +69,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> diff --git a/external/java8-tests/pom.xml b/external/java8-tests/pom.xml index 98467f2cc5..8fc46d7af2 100644 --- a/external/java8-tests/pom.xml +++ b/external/java8-tests/pom.xml @@ -73,6 +73,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 29abd88fcf..6cf448e65e 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -88,6 +88,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index b9ab57624c..88499240cd 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -89,6 +89,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml index bdb3b4d2f5..8368a1f122 100644 --- a/external/kafka-0-8/pom.xml +++ b/external/kafka-0-8/pom.xml @@ -89,6 +89,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 32ea114856..b2bac7c938 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -78,6 +78,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> diff --git a/graphx/pom.xml b/graphx/pom.xml index a9f7839917..8df33660ea 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -78,6 +78,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> diff --git a/launcher/pom.xml b/launcher/pom.xml index 899957f627..025cd84f20 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -67,6 +67,17 @@ <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <!-- Not needed by the test code, but referenced by SparkSubmit which is used by the tests. --> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 201446efb9..663f7fb0b0 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -56,6 +56,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <profiles> <profile> diff --git a/mllib/pom.xml b/mllib/pom.xml index aad6974277..82f840b0fc 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -113,6 +113,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> <profiles> <profile> @@ -294,6 +294,12 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + </dependency> + <dependency> <groupId>com.twitter</groupId> <artifactId>chill_${scala.binary.version}</artifactId> <version>${chill.version}</version> diff --git a/repl/pom.xml b/repl/pom.xml index 1baf1340cf..a256ae3b84 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -92,6 +92,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.xbean</groupId> <artifactId>xbean-asm5-shaded</artifactId> diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 04b51dc92a..f090d2427d 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -54,6 +54,8 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index ba5b5c965f..765c92b8d3 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -56,6 +56,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-unsafe_${scala.binary.version}</artifactId> diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 4178b0a1fc..b8aa698090 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -74,6 +74,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-column</artifactId> diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 7a6e966f96..9c879218dd 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -85,6 +85,18 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>net.sf.jpam</groupId> <artifactId>jpam</artifactId> diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index af02d05550..9aedaf234e 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -60,6 +60,8 @@ <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> </dependency> <!-- <dependency> diff --git a/streaming/pom.xml b/streaming/pom.xml index 7168999898..de1be9c13e 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -51,6 +51,17 @@ <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> + <!-- + This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude + them will yield errors. + --> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. --> <dependency> <groupId>com.google.guava</groupId> |