diff options
35 files changed, 59 insertions, 41 deletions
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index bd507c2cb6..5444ae6d70 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -66,7 +66,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>org.mockito</groupId> diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 810ec10ca0..e736436aec 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -80,7 +80,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>log4j</groupId> diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index bc83ef24c3..1fd3af2e6e 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -48,7 +48,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <!-- Provided dependencies --> diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 8bc1f52798..bbbb0bd5aa 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -38,7 +38,7 @@ <dependencies> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 8e702b4fef..14e94eca93 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -27,12 +27,12 @@ </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_2.11</artifactId> + <artifactId>spark-tags_2.11</artifactId> <packaging>jar</packaging> - <name>Spark Project Test Tags</name> + <name>Spark Project Tags</name> <url>http://spark.apache.org/</url> <properties> - <sbt.project.name>test-tags</sbt.project.name> + <sbt.project.name>tags</sbt.project.name> </properties> <dependencies> diff --git a/core/src/main/java/org/apache/spark/annotation/AlphaComponent.java b/common/tags/src/main/java/org/apache/spark/annotation/AlphaComponent.java index db7b25c727..db7b25c727 100644 --- a/core/src/main/java/org/apache/spark/annotation/AlphaComponent.java +++ b/common/tags/src/main/java/org/apache/spark/annotation/AlphaComponent.java diff --git a/core/src/main/java/org/apache/spark/annotation/DeveloperApi.java b/common/tags/src/main/java/org/apache/spark/annotation/DeveloperApi.java index 0ecef6db0e..0ecef6db0e 100644 --- a/core/src/main/java/org/apache/spark/annotation/DeveloperApi.java +++ b/common/tags/src/main/java/org/apache/spark/annotation/DeveloperApi.java diff --git a/core/src/main/java/org/apache/spark/annotation/Experimental.java b/common/tags/src/main/java/org/apache/spark/annotation/Experimental.java index ff81202914..ff81202914 100644 --- a/core/src/main/java/org/apache/spark/annotation/Experimental.java +++ b/common/tags/src/main/java/org/apache/spark/annotation/Experimental.java diff --git a/core/src/main/java/org/apache/spark/annotation/Private.java b/common/tags/src/main/java/org/apache/spark/annotation/Private.java index 9082fcf0c8..9082fcf0c8 100644 --- a/core/src/main/java/org/apache/spark/annotation/Private.java +++ b/common/tags/src/main/java/org/apache/spark/annotation/Private.java diff --git a/core/src/main/scala/org/apache/spark/annotation/Since.scala b/common/tags/src/main/scala/org/apache/spark/annotation/Since.scala index af483e361e..af483e361e 100644 --- a/core/src/main/scala/org/apache/spark/annotation/Since.scala +++ b/common/tags/src/main/scala/org/apache/spark/annotation/Since.scala diff --git a/core/src/main/scala/org/apache/spark/annotation/package-info.java b/common/tags/src/main/scala/org/apache/spark/annotation/package-info.java index 9efdccf6b0..9efdccf6b0 100644 --- a/core/src/main/scala/org/apache/spark/annotation/package-info.java +++ b/common/tags/src/main/scala/org/apache/spark/annotation/package-info.java diff --git a/core/src/main/scala/org/apache/spark/annotation/package.scala b/common/tags/src/main/scala/org/apache/spark/annotation/package.scala index c3f4026a29..c3f4026a29 100644 --- a/core/src/main/scala/org/apache/spark/annotation/package.scala +++ b/common/tags/src/main/scala/org/apache/spark/annotation/package.scala diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 93b9580f26..d0d1da69ea 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -37,6 +37,10 @@ <dependencies> <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + </dependency> + <dependency> <groupId>com.twitter</groupId> <artifactId>chill_${scala.binary.version}</artifactId> </dependency> @@ -60,10 +64,6 @@ <!-- Test dependencies --> <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/core/pom.xml b/core/pom.xml index 4b8fb4e5fb..45f8bfcc05 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -331,7 +331,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index f81db8e44a..d7eb44d7a5 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -92,10 +92,17 @@ class Module(object): def __hash__(self): return hash(self.name) +tags = Module( + name="tags", + dependencies=[], + source_file_regexes=[ + "common/tags/", + ] +) catalyst = Module( name="catalyst", - dependencies=[], + dependencies=[tags], source_file_regexes=[ "sql/catalyst/", ], @@ -165,7 +172,7 @@ hivecontext_compatibility = Module( sketch = Module( name="sketch", - dependencies=[], + dependencies=[tags], source_file_regexes=[ "common/sketch/", ], @@ -177,7 +184,7 @@ sketch = Module( graphx = Module( name="graphx", - dependencies=[], + dependencies=[tags], source_file_regexes=[ "graphx/", ], @@ -189,7 +196,7 @@ graphx = Module( streaming = Module( name="streaming", - dependencies=[], + dependencies=[tags], source_file_regexes=[ "streaming", ], @@ -205,7 +212,7 @@ streaming = Module( # fail other PRs. streaming_kinesis_asl = Module( name="streaming-kinesis-asl", - dependencies=[], + dependencies=[tags], source_file_regexes=[ "external/kinesis-asl/", "external/kinesis-asl-assembly/", @@ -270,7 +277,7 @@ streaming_flume_assembly = Module( mllib_local = Module( name="mllib-local", - dependencies=[], + dependencies=[tags], source_file_regexes=[ "mllib-local", ], diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 3169a0ebae..21d40863b7 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -126,7 +126,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index e4effe158c..016af934bb 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -92,7 +92,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> diff --git a/external/flume/pom.xml b/external/flume/pom.xml index d650dd034d..f51d334de0 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -68,7 +68,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> diff --git a/external/java8-tests/pom.xml b/external/java8-tests/pom.xml index 1ea9196e9d..60e3ff60df 100644 --- a/external/java8-tests/pom.xml +++ b/external/java8-tests/pom.xml @@ -72,7 +72,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml index cccfda3c61..4a20b78917 100644 --- a/external/kafka-0-8/pom.xml +++ b/external/kafka-0-8/pom.xml @@ -88,7 +88,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 935155eb5d..b5f5ff2854 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -77,7 +77,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> diff --git a/graphx/pom.xml b/graphx/pom.xml index 1813f383cd..fc6c700dd1 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -72,7 +72,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <build> diff --git a/launcher/pom.xml b/launcher/pom.xml index ef73194882..e7303853e6 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -65,7 +65,7 @@ <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <!-- Not needed by the test code, but referenced by SparkSubmit which is used by the tests. --> diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 68f15dd905..1c6ab2b62d 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -53,6 +53,10 @@ <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> + </dependency> </dependencies> <profiles> <profile> diff --git a/mllib/pom.xml b/mllib/pom.xml index 24d8274e22..c324afb2a2 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -116,7 +116,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> </dependencies> <profiles> @@ -286,9 +286,8 @@ <dependencies> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> <version>${project.version}</version> - <scope>test</scope> </dependency> <dependency> <groupId>com.twitter</groupId> diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 69161e0d61..1a02f660fd 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -716,6 +716,14 @@ object MimaExcludes { ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.executor.ShuffleReadMetrics.localBlocksFetched"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.remoteBlocksFetched"), ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.localBlocksFetched") + ) ++ Seq( + // [SPARK-15290] Move annotations, like @Since / @DeveloperApi, into spark-tags + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.package$"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.package"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.Private"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.AlphaComponent"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.Experimental"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.annotation.DeveloperApi") ) case v if v.startsWith("1.6") => Seq( diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index d83afa03f5..3ad9873f43 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -50,10 +50,10 @@ object BuildCommons { ).map(ProjectRef(buildLocation, _)) val allProjects@Seq( - core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, testTags, sketch, _* + core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, _* ) = Seq( "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe", - "test-tags", "sketch" + "tags", "sketch" ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects val optionallyEnabledProjects@Seq(yarn, java8Tests, sparkGangliaLgpl, @@ -340,7 +340,7 @@ object SparkBuild extends PomBuild { val mimaProjects = allProjects.filterNot { x => Seq( spark, hive, hiveThriftServer, hiveCompatibility, catalyst, repl, networkCommon, networkShuffle, networkYarn, - unsafe, testTags, sketch, mllibLocal + unsafe, tags, sketch, mllibLocal ).contains(x) } @@ -685,9 +685,9 @@ object Unidoc { publish := {}, unidocProjectFilter in(ScalaUnidoc, unidoc) := - inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, streamingFlumeSink, yarn, testTags), + inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, streamingFlumeSink, yarn, tags), unidocProjectFilter in(JavaUnidoc, unidoc) := - inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, streamingFlumeSink, yarn, testTags), + inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, streamingFlumeSink, yarn, tags), // Skip actual catalyst, but include the subproject. // Catalyst is not public API and contains quasiquotes which break scaladoc. diff --git a/repl/pom.xml b/repl/pom.xml index 0f396c9b80..c12d121c61 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -87,7 +87,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>org.apache.xbean</groupId> diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 1748fa2778..1923199f4b 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -55,7 +55,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>org.apache.spark</groupId> diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 184fa2f6bd..2ea980bf20 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -73,7 +73,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>org.apache.parquet</groupId> diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 5cf16e42b9..809d36dc69 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -84,7 +84,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>net.sf.jpam</groupId> diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 177b6884fa..c8b20f0afc 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -60,7 +60,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <!-- <dependency> diff --git a/streaming/pom.xml b/streaming/pom.xml index 2055cffe75..3f67745936 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -49,7 +49,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. --> diff --git a/yarn/pom.xml b/yarn/pom.xml index 11df2b3f4f..e07b93ab95 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -54,7 +54,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_${scala.binary.version}</artifactId> + <artifactId>spark-tags_${scala.binary.version}</artifactId> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> |