diff options
| author | Jey Kottalam <jey@cs.berkeley.edu> | 2013-08-15 15:31:31 -0700 |
|---|---|---|
| committer | Jey Kottalam <jey@cs.berkeley.edu> | 2013-08-16 13:50:12 -0700 |
| commit | 11b42a84db255eb659412e9d0bf4622cb2e8b20a (patch) | |
| tree | 1cdcacf8ad3efbb9c942e5567d67a9967c6a6047 | |
| parent | 353fab2440dbf1369df20393e0377de2b327de72 (diff) | |
| download | spark-11b42a84db255eb659412e9d0bf4622cb2e8b20a.tar.gz spark-11b42a84db255eb659412e9d0bf4622cb2e8b20a.tar.bz2 spark-11b42a84db255eb659412e9d0bf4622cb2e8b20a.zip | |
Maven build now works with CDH hadoop-2.0.0-mr1
| -rw-r--r-- | assembly/pom.xml | 14 | ||||
| -rw-r--r-- | bagel/pom.xml | 27 | ||||
| -rw-r--r-- | core/pom.xml | 52 | ||||
| -rw-r--r-- | examples/pom.xml | 44 | ||||
| -rw-r--r-- | mllib/pom.xml | 27 | ||||
| -rw-r--r-- | pom.xml | 55 | ||||
| -rw-r--r-- | repl-bin/pom.xml | 40 | ||||
| -rw-r--r-- | repl/pom.xml | 64 | ||||
| -rw-r--r-- | streaming/pom.xml | 27 | ||||
| -rw-r--r-- | tools/pom.xml | 33 |
10 files changed, 30 insertions, 353 deletions
diff --git a/assembly/pom.xml b/assembly/pom.xml index 76ac9f5478..3d645e0379 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -37,20 +37,6 @@ </plugins> </build> - <profiles> - <profile> - <id>hadoop2</id> - <properties> - <classifier.name>hadoop2</classifier.name> - </properties> - </profile> - <profile> - <id>hadoop2-yarn</id> - <properties> - <classifier.name>hadoop2-yarn</classifier.name> - </properties> - </profile> - </profiles> <dependencies> <dependency> <groupId>org.spark-project</groupId> diff --git a/bagel/pom.xml b/bagel/pom.xml index 3c82af3b33..1b555bead7 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -70,33 +70,6 @@ <profiles> <profile> - <id>hadoop2</id> - <dependencies> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-core</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <classifier>hadoop2</classifier> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>hadoop2-yarn</id> <dependencies> <dependency> diff --git a/core/pom.xml b/core/pom.xml index 680ae94a11..9310d000fd 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -215,58 +215,6 @@ <profiles> <profile> - <id>hadoop2</id> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>add-source</id> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>src/main/scala</source> - <source>src/hadoop2/scala</source> - </sources> - </configuration> - </execution> - <execution> - <id>add-scala-test-sources</id> - <phase>generate-test-sources</phase> - <goals> - <goal>add-test-source</goal> - </goals> - <configuration> - <sources> - <source>src/test/scala</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <classifier>hadoop2</classifier> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>hadoop2-yarn</id> <dependencies> <dependency> diff --git a/examples/pom.xml b/examples/pom.xml index 6a9c19ed6f..6e54e94cf5 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -129,50 +129,6 @@ <profiles> <profile> - <id>hadoop2</id> - <dependencies> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-core</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-streaming</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-mllib</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase</artifactId> - <version>0.94.6</version> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <classifier>hadoop2</classifier> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>hadoop2-yarn</id> <dependencies> <dependency> diff --git a/mllib/pom.xml b/mllib/pom.xml index 36f410d3b0..863aef9392 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -80,33 +80,6 @@ <profiles> <profile> - <id>hadoop2</id> - <dependencies> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-core</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <classifier>hadoop2</classifier> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>hadoop2-yarn</id> <dependencies> <dependency> @@ -73,8 +73,9 @@ <mesos.version>0.12.1</mesos.version> <akka.version>2.0.3</akka.version> <slf4j.version>1.7.2</slf4j.version> - <cdh.version>4.1.2</cdh.version> <log4j.version>1.2.17</log4j.version> + <hadoop.version>1.2.1</hadoop.version> + <!-- <hadoop.version>2.0.0-mr1-cdh4.1.2</hadoop.version> --> <PermGen>64m</PermGen> <MaxPermGen>512m</MaxPermGen> @@ -328,7 +329,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> - <version>1.2.1</version> + <version>${hadoop.version}</version> <exclusions> <exclusion> <groupId>asm</groupId> @@ -336,6 +337,23 @@ </exclusion> </exclusions> </dependency> + <!-- Specify Avro version because Kafka also has it as a dependency --> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <version>1.7.4</version> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro-ipc</artifactId> + <version>1.7.4</version> + <exclusions> + <exclusion> + <groupId>org.jboss.netty</groupId> + <artifactId>netty</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> </dependencyManagement> @@ -542,39 +560,6 @@ <profiles> <profile> - <id>hadoop2</id> - <properties> - <hadoop.major.version>2</hadoop.major.version> - </properties> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <version>2.0.0-mr1-cdh${cdh.version}</version> - </dependency> - <!-- Specify Avro version because Kafka also has it as a dependency --> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - <version>1.7.4</version> - </dependency> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro-ipc</artifactId> - <version>1.7.4</version> - <exclusions> - <exclusion> - <groupId>org.jboss.netty</groupId> - <artifactId>netty</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - </dependencyManagement> - </profile> - - <profile> <id>hadoop2-yarn</id> <properties> <hadoop.major.version>2</hadoop.major.version> diff --git a/repl-bin/pom.xml b/repl-bin/pom.xml index 81aba06e14..eaee8ea016 100644 --- a/repl-bin/pom.xml +++ b/repl-bin/pom.xml @@ -117,46 +117,6 @@ <profiles> <profile> - <id>hadoop2</id> - <properties> - <classifier>hadoop2</classifier> - </properties> - <dependencies> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-core</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-bagel</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-examples</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-repl</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>runtime</scope> - </dependency> - </dependencies> - </profile> - <profile> <id>hadoop2-yarn</id> <properties> <classifier>hadoop2-yarn</classifier> diff --git a/repl/pom.xml b/repl/pom.xml index 81cebe178a..032c20e118 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -60,6 +60,16 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro-ipc</artifactId> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> </dependency> @@ -139,60 +149,6 @@ <profiles> <profile> - <id>hadoop2</id> - <properties> - <classifier>hadoop2</classifier> - </properties> - <dependencies> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-core</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-bagel</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-examples</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro-ipc</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <classifier>hadoop2</classifier> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>hadoop2-yarn</id> <properties> <classifier>hadoop2-yarn</classifier> diff --git a/streaming/pom.xml b/streaming/pom.xml index 9b478f7a05..612ff0a024 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -128,33 +128,6 @@ <profiles> <profile> - <id>hadoop2</id> - <dependencies> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-core</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <classifier>hadoop2</classifier> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>hadoop2-yarn</id> <dependencies> <dependency> diff --git a/tools/pom.xml b/tools/pom.xml index c123c2ab23..5864c9f217 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -74,39 +74,6 @@ <profiles> <profile> - <id>hadoop2</id> - <dependencies> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-core</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.spark-project</groupId> - <artifactId>spark-streaming</artifactId> - <version>${project.version}</version> - <classifier>hadoop2</classifier> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <classifier>hadoop2</classifier> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> <id>hadoop2-yarn</id> <dependencies> <dependency> |