diff options
author | Prashant Sharma <prashant.s@imaginea.com> | 2014-11-11 21:36:48 -0800 |
---|---|---|
committer | Patrick Wendell <pwendell@gmail.com> | 2014-11-11 21:36:48 -0800 |
commit | daaca14c16dc2c1abc98f15ab8c6f7c14761b627 (patch) | |
tree | de60da38655f8a7d4b0712872b08a7aedf73460f /pom.xml | |
parent | 2ddb1415e2bea94004947506ded090c2e8ff8dad (diff) | |
download | spark-daaca14c16dc2c1abc98f15ab8c6f7c14761b627.tar.gz spark-daaca14c16dc2c1abc98f15ab8c6f7c14761b627.tar.bz2 spark-daaca14c16dc2c1abc98f15ab8c6f7c14761b627.zip |
Support cross building for Scala 2.11
Let's give this another go using a version of Hive that shades its JLine dependency.
Author: Prashant Sharma <prashant.s@imaginea.com>
Author: Patrick Wendell <pwendell@gmail.com>
Closes #3159 from pwendell/scala-2.11-prashant and squashes the following commits:
e93aa3e [Patrick Wendell] Restoring -Phive-thriftserver profile and cleaning up build script.
f65d17d [Patrick Wendell] Fixing build issue due to merge conflict
a8c41eb [Patrick Wendell] Reverting dev/run-tests back to master state.
7a6eb18 [Patrick Wendell] Merge remote-tracking branch 'apache/master' into scala-2.11-prashant
583aa07 [Prashant Sharma] REVERT ME: removed hive thirftserver
3680e58 [Prashant Sharma] Revert "REVERT ME: Temporarily removing some Cli tests."
935fb47 [Prashant Sharma] Revert "Fixed by disabling a few tests temporarily."
925e90f [Prashant Sharma] Fixed by disabling a few tests temporarily.
2fffed3 [Prashant Sharma] Exclude groovy from sbt build, and also provide a way for such instances in future.
8bd4e40 [Prashant Sharma] Switched to gmaven plus, it fixes random failures observer with its predecessor gmaven.
5272ce5 [Prashant Sharma] SPARK_SCALA_VERSION related bugs.
2121071 [Patrick Wendell] Migrating version detection to PySpark
b1ed44d [Patrick Wendell] REVERT ME: Temporarily removing some Cli tests.
1743a73 [Patrick Wendell] Removing decimal test that doesn't work with Scala 2.11
f5cad4e [Patrick Wendell] Add Scala 2.11 docs
210d7e1 [Patrick Wendell] Revert "Testing new Hive version with shaded jline"
48518ce [Patrick Wendell] Remove association of Hive and Thriftserver profiles.
e9d0a06 [Patrick Wendell] Revert "Enable thritfserver for Scala 2.10 only"
67ec364 [Patrick Wendell] Guard building of thriftserver around Scala 2.10 check
8502c23 [Patrick Wendell] Enable thritfserver for Scala 2.10 only
e22b104 [Patrick Wendell] Small fix in pom file
ec402ab [Patrick Wendell] Various fixes
0be5a9d [Patrick Wendell] Testing new Hive version with shaded jline
4eaec65 [Prashant Sharma] Changed scripts to ignore target.
5167bea [Prashant Sharma] small correction
a4fcac6 [Prashant Sharma] Run against scala 2.11 on jenkins.
80285f4 [Prashant Sharma] MAven equivalent of setting spark.executor.extraClasspath during tests.
034b369 [Prashant Sharma] Setting test jars on executor classpath during tests from sbt.
d4874cb [Prashant Sharma] Fixed Python Runner suite. null check should be first case in scala 2.11.
6f50f13 [Prashant Sharma] Fixed build after rebasing with master. We should use ${scala.binary.version} instead of just 2.10
e56ca9d [Prashant Sharma] Print an error if build for 2.10 and 2.11 is spotted.
937c0b8 [Prashant Sharma] SCALA_VERSION -> SPARK_SCALA_VERSION
cb059b0 [Prashant Sharma] Code review
0476e5e [Prashant Sharma] Scala 2.11 support with repl and all build changes.
Diffstat (limited to 'pom.xml')
-rw-r--r-- | pom.xml | 178 |
1 files changed, 131 insertions, 47 deletions
@@ -97,30 +97,26 @@ <module>sql/catalyst</module> <module>sql/core</module> <module>sql/hive</module> - <module>repl</module> <module>assembly</module> <module>external/twitter</module> - <module>external/kafka</module> <module>external/flume</module> <module>external/flume-sink</module> - <module>external/zeromq</module> <module>external/mqtt</module> + <module>external/zeromq</module> <module>examples</module> + <module>repl</module> </modules> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - + <akka.group>org.spark-project.akka</akka.group> + <akka.version>2.3.4-spark</akka.version> <java.version>1.6</java.version> <sbt.project.name>spark</sbt.project.name> - <scala.version>2.10.4</scala.version> - <scala.binary.version>2.10</scala.binary.version> <scala.macros.version>2.0.1</scala.macros.version> <mesos.version>0.18.1</mesos.version> <mesos.classifier>shaded-protobuf</mesos.classifier> - <akka.group>org.spark-project.akka</akka.group> - <akka.version>2.3.4-spark</akka.version> <slf4j.version>1.7.5</slf4j.version> <log4j.version>1.2.17</log4j.version> <hadoop.version>1.0.4</hadoop.version> @@ -137,7 +133,7 @@ <parquet.version>1.6.0rc3</parquet.version> <jblas.version>1.2.3</jblas.version> <jetty.version>8.1.14.v20131031</jetty.version> - <chill.version>0.3.6</chill.version> + <chill.version>0.5.0</chill.version> <codahale.metrics.version>3.0.0</codahale.metrics.version> <avro.version>1.7.6</avro.version> <avro.mapred.classifier></avro.mapred.classifier> @@ -146,9 +142,13 @@ <aws.kinesis.client.version>1.1.0</aws.kinesis.client.version> <commons.httpclient.version>4.2.6</commons.httpclient.version> <commons.math3.version>3.1.1</commons.math3.version> - + <test_classpath_file>${project.build.directory}/spark-test-classpath.txt</test_classpath_file> <PermGen>64m</PermGen> <MaxPermGen>512m</MaxPermGen> + <scala.version>2.10.4</scala.version> + <scala.binary.version>2.10</scala.binary.version> + <jline.version>${scala.version}</jline.version> + <jline.groupid>org.scala-lang</jline.groupid> </properties> <repositories> @@ -267,20 +267,67 @@ </snapshots> </pluginRepository> </pluginRepositories> + + <dependencies> <!-- This is a dummy dependency that is used along with the shading plug-in to create effective poms on publishing (see SPARK-3812). --> - <dependencies> <dependency> <groupId>org.spark-project.spark</groupId> <artifactId>unused</artifactId> <version>1.0.0</version> </dependency> + <!-- + This depndency has been added to provided scope as it is needed for excuting build + specific groovy scripts using gmaven+ and not required for downstream project building + with spark. + --> + <dependency> + <groupId>org.codehaus.groovy</groupId> + <artifactId>groovy-all</artifactId> + <version>2.3.7</version> + <scope>provided</scope> + </dependency> </dependencies> <dependencyManagement> <dependencies> <dependency> + <groupId>${jline.groupid}</groupId> + <artifactId>jline</artifactId> + <version>${jline.version}</version> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill_${scala.binary.version}</artifactId> + <version>${chill.version}</version> + <exclusions> + <exclusion> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + <exclusion> + <groupId>org.ow2.asm</groupId> + <artifactId>asm-commons</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.twitter</groupId> + <artifactId>chill-java</artifactId> + <version>${chill.version}</version> + <exclusions> + <exclusion> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + <exclusion> + <groupId>org.ow2.asm</groupId> + <artifactId>asm-commons</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> <version>${jetty.version}</version> @@ -396,36 +443,6 @@ <version>${protobuf.version}</version> </dependency> <dependency> - <groupId>com.twitter</groupId> - <artifactId>chill_${scala.binary.version}</artifactId> - <version>${chill.version}</version> - <exclusions> - <exclusion> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - </exclusion> - <exclusion> - <groupId>org.ow2.asm</groupId> - <artifactId>asm-commons</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>chill-java</artifactId> - <version>${chill.version}</version> - <exclusions> - <exclusion> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - </exclusion> - <exclusion> - <groupId>org.ow2.asm</groupId> - <artifactId>asm-commons</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>${akka.group}</groupId> <artifactId>akka-actor_${scala.binary.version}</artifactId> <version>${akka.version}</version> @@ -514,11 +531,6 @@ </dependency> <dependency> <groupId>org.scala-lang</groupId> - <artifactId>jline</artifactId> - <version>${scala.version}</version> - </dependency> - <dependency> - <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> <version>${scala.version}</version> </dependency> @@ -965,6 +977,7 @@ <spark.test.home>${session.executionRootDirectory}</spark.test.home> <spark.testing>1</spark.testing> <spark.ui.enabled>false</spark.ui.enabled> + <spark.executor.extraClassPath>${test_classpath}</spark.executor.extraClassPath> </systemProperties> </configuration> <executions> @@ -1026,6 +1039,47 @@ </pluginManagement> <plugins> + <!-- This plugin dumps the test classpath into a file --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.9</version> + <executions> + <execution> + <phase>test-compile</phase> + <goals> + <goal>build-classpath</goal> + </goals> + <configuration> + <includeScope>test</includeScope> + <outputFile>${test_classpath_file}</outputFile> + </configuration> + </execution> + </executions> + </plugin> + + <!-- This plugin reads a file into maven property. And it lets us write groovy !! --> + <plugin> + <groupId>org.codehaus.gmavenplus</groupId> + <artifactId>gmavenplus-plugin</artifactId> + <version>1.2</version> + <executions> + <execution> + <phase>process-test-classes</phase> + <goals> + <goal>execute</goal> + </goals> + <configuration> + <scripts> + <script><![CDATA[ + def file = new File(project.properties.test_classpath_file) + project.properties.test_classpath = file.getText().split().join(":") + ]]></script> + </scripts> + </configuration> + </execution> + </executions> + </plugin> <!-- The shade plug-in is used here to create effective pom's (see SPARK-3812). --> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -1335,7 +1389,7 @@ </dependencies> </profile> <profile> - <id>hive</id> + <id>hive-thriftserver</id> <activation> <activeByDefault>false</activeByDefault> </activation> @@ -1365,5 +1419,35 @@ <derby.version>10.10.1.1</derby.version> </properties> </profile> + + <profile> + <id>scala-2.10</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <properties> + <scala.version>2.10.4</scala.version> + <scala.binary.version>2.10</scala.binary.version> + <jline.version>${scala.version}</jline.version> + <jline.groupid>org.scala-lang</jline.groupid> + </properties> + <modules> + <module>external/kafka</module> + </modules> + </profile> + + <profile> + <id>scala-2.11</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <properties> + <scala.version>2.11.2</scala.version> + <scala.binary.version>2.11</scala.binary.version> + <jline.version>2.12</jline.version> + <jline.groupid>jline</jline.groupid> + </properties> + </profile> + </profiles> </project> |