diff options
| -rwxr-xr-x | bin/compute-classpath.sh | 4 | ||||
| -rw-r--r-- | core/pom.xml | 22 | ||||
| -rw-r--r-- | network/common/pom.xml | 12 | ||||
| -rw-r--r-- | pom.xml | 32 | ||||
| -rw-r--r-- | repl/pom.xml | 24 | ||||
| -rw-r--r-- | streaming/pom.xml | 16 |
6 files changed, 95 insertions, 15 deletions
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index 9e8d0b7851..a8c344b1ca 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -50,8 +50,8 @@ fi if [ -n "$SPARK_PREPEND_CLASSES" ]; then echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\ "classes ahead of assembly." >&2 + # Spark classes CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes" - CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*" CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes" @@ -63,6 +63,8 @@ if [ -n "$SPARK_PREPEND_CLASSES" ]; then CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes" + # Jars for shaded deps in their original form (copied here during build) + CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*" fi # Use spark-assembly jar from either RELEASE or assembly directory diff --git a/core/pom.xml b/core/pom.xml index 6fce10a0ae..4d24ae93c4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -94,23 +94,36 @@ <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> </dependency> + + <!-- Jetty dependencies promoted to compile here so they are shaded + and inlined into spark-core jar --> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-plus</artifactId> + <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-security</artifactId> + <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> + <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> + <scope>compile</scope> </dependency> <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + <scope>compile</scope> + </dependency> + + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> @@ -348,19 +361,24 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <executions> + <!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use + shaded deps. So here we store jars in their original form which are added + when the classpath is computed. --> <execution> <id>copy-dependencies</id> <phase>package</phase> <goals> <goal>copy-dependencies</goal> </goals> - <configuration> + <configuration> <outputDirectory>${project.build.directory}</outputDirectory> <overWriteReleases>false</overWriteReleases> <overWriteSnapshots>false</overWriteSnapshots> <overWriteIfNewer>true</overWriteIfNewer> <useSubDirectoryPerType>true</useSubDirectoryPerType> - <includeArtifactIds>guava</includeArtifactIds> + <includeArtifactIds> + guava,jetty-io,jetty-http,jetty-plus,jetty-util,jetty-server + </includeArtifactIds> <silent>true</silent> </configuration> </execution> diff --git a/network/common/pom.xml b/network/common/pom.xml index 5a9bbe105d..8f7c924d6b 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -101,18 +101,6 @@ </execution> </executions> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <shadedArtifactAttached>false</shadedArtifactAttached> - <artifactSet> - <includes> - <include>com.google.guava:guava</include> - </includes> - </artifactSet> - </configuration> - </plugin> </plugins> </build> </project> @@ -337,25 +337,39 @@ </exclusion> </exclusions> </dependency> + + <!-- Shaded deps marked as provided. These are promoted to compile scope + in the modules where we want the shaded classes to appear in the + associated jar. --> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + <version>${jetty.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> <version>${jetty.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-security</artifactId> <version>${jetty.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-plus</artifactId> <version>${jetty.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> <version>${jetty.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -363,6 +377,8 @@ <version>14.0.1</version> <scope>provided</scope> </dependency> + <!-- End of shaded deps --> + <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> @@ -1276,11 +1292,27 @@ <shadedArtifactAttached>false</shadedArtifactAttached> <artifactSet> <includes> + <!-- At a minimum we must include this to force effective pom generation --> <include>org.spark-project.spark:unused</include> + + <include>org.eclipse.jetty:jetty-io</include> + <include>org.eclipse.jetty:jetty-http</include> + <include>org.eclipse.jetty:jetty-plus</include> + <include>org.eclipse.jetty:jetty-security</include> + <include>org.eclipse.jetty:jetty-util</include> + <include>org.eclipse.jetty:jetty-server</include> + <include>com.google.guava:guava</include> </includes> </artifactSet> <relocations> <relocation> + <pattern>org.eclipse.jetty</pattern> + <shadedPattern>org.spark-project.jetty</shadedPattern> + <includes> + <include>org.eclipse.jetty.**</include> + </includes> + </relocation> + <relocation> <pattern>com.google.common</pattern> <shadedPattern>org.spark-project.guava</shadedPattern> <excludes> diff --git a/repl/pom.xml b/repl/pom.xml index ae7c31aef4..bd39b90fd8 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -87,6 +87,30 @@ <artifactId>scalacheck_${scala.binary.version}</artifactId> <scope>test</scope> </dependency> + + <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. --> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-plus</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + </dependency> + <!-- End of shaded deps. --> + + <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + </dependency> </dependencies> <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> diff --git a/streaming/pom.xml b/streaming/pom.xml index d032491e2f..5efed16039 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -40,6 +40,8 @@ <artifactId>spark-core_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> + + <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> @@ -49,6 +51,20 @@ <artifactId>jetty-server</artifactId> </dependency> <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-plus</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-http</artifactId> + </dependency> + <!-- End of shaded deps. --> + + <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> </dependency> |