diff options
| -rwxr-xr-x | make-distribution.sh | 1 | ||||
| -rw-r--r-- | network/common/pom.xml | 5 | ||||
| -rw-r--r-- | network/shuffle/pom.xml | 5 | ||||
| -rw-r--r-- | network/yarn/pom.xml | 33 | ||||
| -rw-r--r-- | project/SparkBuild.scala | 19 |
5 files changed, 53 insertions, 10 deletions
diff --git a/make-distribution.sh b/make-distribution.sh index d46edbc50d..55cbdc14ac 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -181,6 +181,7 @@ echo "Spark $VERSION$GITREVSTRING built for Hadoop $SPARK_HADOOP_VERSION" > "$DI # Copy jars cp "$FWDIR"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/" cp "$FWDIR"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/" +cp "$FWDIR"/network/yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/lib/" # Copy example sources (needed for python and SQL) mkdir -p "$DISTDIR/examples/src/main" diff --git a/network/common/pom.xml b/network/common/pom.xml index 8b24ebf1ba..a6bee7ed09 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -41,12 +41,13 @@ <groupId>io.netty</groupId> <artifactId>netty-all</artifactId> </dependency> + + <!-- Provided dependencies --> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> + <scope>provided</scope> </dependency> - - <!-- Provided dependencies --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index a180a5e5f9..be78331ea9 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -42,12 +42,13 @@ <artifactId>spark-network-common_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> + + <!-- Provided dependencies --> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> + <scope>provided</scope> </dependency> - - <!-- Provided dependencies --> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 85960eb85b..7845011ec3 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -54,5 +54,38 @@ <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <shadedArtifactAttached>false</shadedArtifactAttached> + <outputFile>${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar</outputFile> + <artifactSet> + <includes> + <include>*:*</include> + </includes> + </artifactSet> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> </build> </project> diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 5eb3ed439c..d95d50a1d8 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -38,12 +38,12 @@ object BuildCommons { "streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter", "streaming-zeromq").map(ProjectRef(buildLocation, _)) - val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, networkYarn, java8Tests, - sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha", "network-yarn", + val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests, + sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha", "java8-tests", "ganglia-lgpl", "kinesis-asl").map(ProjectRef(buildLocation, _)) - val assemblyProjects@Seq(assembly, examples) = Seq("assembly", "examples") - .map(ProjectRef(buildLocation, _)) + val assemblyProjects@Seq(assembly, examples, networkYarn) = + Seq("assembly", "examples", "network-yarn").map(ProjectRef(buildLocation, _)) val tools = ProjectRef(buildLocation, "tools") // Root project. @@ -289,8 +289,15 @@ object Assembly { lazy val settings = assemblySettings ++ Seq( test in assembly := {}, - jarName in assembly <<= (version, moduleName) map { (v, mName) => mName + "-"+v + "-hadoop" + - Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" }, + jarName in assembly <<= (version, moduleName) map { (v, mName) => + if (mName.contains("network-yarn")) { + // This must match the same name used in maven (see network/yarn/pom.xml) + "spark-" + v + "-yarn-shuffle.jar" + } else { + mName + "-" + v + "-hadoop" + + Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" + } + }, mergeStrategy in assembly := { case PathList("org", "datanucleus", xs @ _*) => MergeStrategy.discard case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard |