diff options
| author | Thomas Graves <tgraves@apache.org> | 2014-05-06 12:00:09 -0700 |
|---|---|---|
| committer | Patrick Wendell <pwendell@gmail.com> | 2014-05-06 12:00:09 -0700 |
| commit | 1e829905c791fbf1dfd8e0c1caa62ead7354605e (patch) | |
| tree | e0cf8d483c3b4d4c1672685f690fc297aef1a762 | |
| parent | 0a5a46811475a7accac1ed8dfefeeba9554456db (diff) | |
| download | spark-1e829905c791fbf1dfd8e0c1caa62ead7354605e.tar.gz spark-1e829905c791fbf1dfd8e0c1caa62ead7354605e.tar.bz2 spark-1e829905c791fbf1dfd8e0c1caa62ead7354605e.zip | |
SPARK-1474: Spark on yarn assembly doesn't include AmIpFilter
We use org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter in spark on yarn but are not included it in the assembly jar.
I tested this on yarn cluster by removing the yarn jars from the classpath and spark runs fine now.
Author: Thomas Graves <tgraves@apache.org>
Closes #406 from tgravescs/SPARK-1474 and squashes the following commits:
1548bf9 [Thomas Graves] SPARK-1474: Spark on yarn assembly doesn't include org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter
| -rw-r--r-- | pom.xml | 25 | ||||
| -rw-r--r-- | project/SparkBuild.scala | 3 | ||||
| -rw-r--r-- | yarn/pom.xml | 4 |
3 files changed, 30 insertions, 2 deletions
@@ -616,7 +616,25 @@ </exclusion> </exclusions> </dependency> - + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + <version>${yarn.version}</version> + <exclusions> + <exclusion> + <groupId>asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + <exclusion> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + <exclusion> + <groupId>org.jboss.netty</groupId> + <artifactId>netty</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> @@ -960,6 +978,11 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> <scope>provided</scope> </dependency> diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index a2597e3e6d..1ad05d9e46 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -563,7 +563,8 @@ object SparkBuild extends Build { "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm), "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm), "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm), - "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm) + "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm), + "org.apache.hadoop" % "hadoop-yarn-server-web-proxy" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm) ) ) diff --git a/yarn/pom.xml b/yarn/pom.xml index e009473826..6993c89525 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -45,6 +45,10 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> </dependency> <dependency> |