[SPARK-6363][BUILD] Make Scala 2.11 the default Scala version

This patch changes Spark's build to make Scala 2.11 the default Scala version. To be clear, this does not mean that Spark will stop supporting Scala 2.10: users will still be able to compile Spark for Scala 2.10 by following the instructions on the "Building Spark" page; however, it does mean that Scala 2.11 will be the default Scala version used by our CI builds (including pull request builds).

The Scala 2.11 compiler is faster than 2.10, so I think we'll be able to look forward to a slight speedup in our CI builds (it looks like it's about 2X faster for the Maven compile-only builds, for instance).

After this patch is merged, I'll update Jenkins to add new compile-only jobs to ensure that Scala 2.10 compilation doesn't break.

Author: Josh Rosen <joshrosen@databricks.com>

Closes #10608 from JoshRosen/SPARK-6363.

6 files changed, 82 insertions, 87 deletions

diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 00bf81120d..2fd7fcc39e 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -134,9 +134,9 @@ if [[ "$1" == "package" ]]; then
 
     cd spark-$SPARK_VERSION-bin-$NAME
 
-    # TODO There should probably be a flag to make-distribution to allow 2.11 support
-    if [[ $FLAGS == *scala-2.11* ]]; then
-      ./dev/change-scala-version.sh 2.11
+    # TODO There should probably be a flag to make-distribution to allow 2.10 support
+    if [[ $FLAGS == *scala-2.10* ]]; then
+      ./dev/change-scala-version.sh 2.10
     fi
 
     export ZINC_PORT=$ZINC_PORT
@@ -228,8 +228,8 @@ if [[ "$1" == "publish-snapshot" ]]; then
 
   $MVN -DzincPort=$ZINC_PORT --settings $tmp_settings -DskipTests $PUBLISH_PROFILES \
     -Phive-thriftserver deploy
-  ./dev/change-scala-version.sh 2.11
-  $MVN -DzincPort=$ZINC_PORT -Dscala-2.11 --settings $tmp_settings \
+  ./dev/change-scala-version.sh 2.10
+  $MVN -DzincPort=$ZINC_PORT -Dscala-2.10 --settings $tmp_settings \
     -DskipTests $PUBLISH_PROFILES clean deploy
 
   # Clean-up Zinc nailgun process
@@ -266,9 +266,9 @@ if [[ "$1" == "publish-release" ]]; then
   $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -DskipTests $PUBLISH_PROFILES \
     -Phive-thriftserver clean install
 
-  ./dev/change-scala-version.sh 2.11
+  ./dev/change-scala-version.sh 2.10
 
-  $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.11 \
+  $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.10 \
     -DskipTests $PUBLISH_PROFILES clean install
 
   # Clean-up Zinc nailgun process
diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2
index 4d9937c5cb..3a14499d9b 100644
--- a/dev/deps/spark-deps-hadoop-2.2
+++ b/dev/deps/spark-deps-hadoop-2.2
@@ -14,13 +14,13 @@ avro-ipc-1.7.7-tests.jar
 avro-ipc-1.7.7.jar
 avro-mapred-1.7.7-hadoop2.jar
 bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
 calcite-avatica-1.2.0-incubating.jar
 calcite-core-1.2.0-incubating.jar
 calcite-linq4j-1.2.0-incubating.jar
 chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
 commons-beanutils-1.7.0.jar
 commons-beanutils-core-1.8.0.jar
 commons-cli-1.2.jar
@@ -86,10 +86,9 @@ jackson-core-asl-1.9.13.jar
 jackson-databind-2.5.3.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
 jackson-xc-1.9.13.jar
 janino-2.7.8.jar
-jansi-1.4.jar
 javax.inject-1.jar
 javax.servlet-3.0.0.v201112011016.jar
 javax.servlet-3.1.jar
@@ -111,15 +110,14 @@ jets3t-0.7.1.jar
 jettison-1.1.jar
 jetty-all-7.6.0.v20120127.jar
 jetty-util-6.1.26.jar
-jline-2.10.5.jar
 jline-2.12.jar
 joda-time-2.9.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
 json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
 jsr305-1.3.9.jar
 jta-1.1.jar
 jtransforms-2.4.0.jar
@@ -158,19 +156,20 @@ pmml-schema-1.2.7.jar
 protobuf-java-2.5.0.jar
 py4j-0.9.1.jar
 pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
 reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
 servlet-api-2.5.jar
 slf4j-api-1.7.10.jar
 slf4j-log4j12-1.7.10.jar
 snappy-0.2.jar
 snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
 stax-api-1.0-2.jar
 stax-api-1.0.1.jar
 stream-2.7.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3
index fd659ee20d..615836b3d3 100644
--- a/dev/deps/spark-deps-hadoop-2.3
+++ b/dev/deps/spark-deps-hadoop-2.3
@@ -16,13 +16,13 @@ avro-mapred-1.7.7-hadoop2.jar
 base64-2.3.8.jar
 bcprov-jdk15on-1.51.jar
 bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
 calcite-avatica-1.2.0-incubating.jar
 calcite-core-1.2.0-incubating.jar
 calcite-linq4j-1.2.0-incubating.jar
 chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
 commons-beanutils-1.7.0.jar
 commons-beanutils-core-1.8.0.jar
 commons-cli-1.2.jar
@@ -81,10 +81,9 @@ jackson-core-asl-1.9.13.jar
 jackson-databind-2.5.3.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
 jackson-xc-1.9.13.jar
 janino-2.7.8.jar
-jansi-1.4.jar
 java-xmlbuilder-1.0.jar
 javax.inject-1.jar
 javax.servlet-3.0.0.v201112011016.jar
@@ -102,15 +101,14 @@ jettison-1.1.jar
 jetty-6.1.26.jar
 jetty-all-7.6.0.v20120127.jar
 jetty-util-6.1.26.jar
-jline-2.10.5.jar
 jline-2.12.jar
 joda-time-2.9.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
 json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
 jsr305-1.3.9.jar
 jta-1.1.jar
 jtransforms-2.4.0.jar
@@ -149,19 +147,20 @@ pmml-schema-1.2.7.jar
 protobuf-java-2.5.0.jar
 py4j-0.9.1.jar
 pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
 reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
 servlet-api-2.5.jar
 slf4j-api-1.7.10.jar
 slf4j-log4j12-1.7.10.jar
 snappy-0.2.jar
 snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
 stax-api-1.0-2.jar
 stax-api-1.0.1.jar
 stream-2.7.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4
index afae3deb9a..f275226f1d 100644
--- a/dev/deps/spark-deps-hadoop-2.4
+++ b/dev/deps/spark-deps-hadoop-2.4
@@ -16,13 +16,13 @@ avro-mapred-1.7.7-hadoop2.jar
 base64-2.3.8.jar
 bcprov-jdk15on-1.51.jar
 bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
 calcite-avatica-1.2.0-incubating.jar
 calcite-core-1.2.0-incubating.jar
 calcite-linq4j-1.2.0-incubating.jar
 chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
 commons-beanutils-1.7.0.jar
 commons-beanutils-core-1.8.0.jar
 commons-cli-1.2.jar
@@ -81,10 +81,9 @@ jackson-core-asl-1.9.13.jar
 jackson-databind-2.5.3.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
 jackson-xc-1.9.13.jar
 janino-2.7.8.jar
-jansi-1.4.jar
 java-xmlbuilder-1.0.jar
 javax.inject-1.jar
 javax.servlet-3.0.0.v201112011016.jar
@@ -103,15 +102,14 @@ jettison-1.1.jar
 jetty-6.1.26.jar
 jetty-all-7.6.0.v20120127.jar
 jetty-util-6.1.26.jar
-jline-2.10.5.jar
 jline-2.12.jar
 joda-time-2.9.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
 json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
 jsr305-1.3.9.jar
 jta-1.1.jar
 jtransforms-2.4.0.jar
@@ -150,19 +148,20 @@ pmml-schema-1.2.7.jar
 protobuf-java-2.5.0.jar
 py4j-0.9.1.jar
 pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
 reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
 servlet-api-2.5.jar
 slf4j-api-1.7.10.jar
 slf4j-log4j12-1.7.10.jar
 snappy-0.2.jar
 snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
 stax-api-1.0-2.jar
 stax-api-1.0.1.jar
 stream-2.7.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index 5a6460136a..21432a16e3 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -20,13 +20,13 @@ avro-mapred-1.7.7-hadoop2.jar
 base64-2.3.8.jar
 bcprov-jdk15on-1.51.jar
 bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
 calcite-avatica-1.2.0-incubating.jar
 calcite-core-1.2.0-incubating.jar
 calcite-linq4j-1.2.0-incubating.jar
 chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
 commons-beanutils-1.7.0.jar
 commons-beanutils-core-1.8.0.jar
 commons-cli-1.2.jar
@@ -87,10 +87,9 @@ jackson-core-asl-1.9.13.jar
 jackson-databind-2.5.3.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
 jackson-xc-1.9.13.jar
 janino-2.7.8.jar
-jansi-1.4.jar
 java-xmlbuilder-1.0.jar
 javax.inject-1.jar
 javax.servlet-3.0.0.v201112011016.jar
@@ -109,15 +108,14 @@ jettison-1.1.jar
 jetty-6.1.26.jar
 jetty-all-7.6.0.v20120127.jar
 jetty-util-6.1.26.jar
-jline-2.10.5.jar
 jline-2.12.jar
 joda-time-2.9.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
 json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
 jsr305-1.3.9.jar
 jta-1.1.jar
 jtransforms-2.4.0.jar
@@ -156,19 +154,20 @@ pmml-schema-1.2.7.jar
 protobuf-java-2.5.0.jar
 py4j-0.9.1.jar
 pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
 reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
 servlet-api-2.5.jar
 slf4j-api-1.7.10.jar
 slf4j-log4j12-1.7.10.jar
 snappy-0.2.jar
 snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
 stax-api-1.0-2.jar
 stax-api-1.0.1.jar
 stream-2.7.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 70083e7f3d..20e09cd002 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -20,13 +20,13 @@ avro-mapred-1.7.7-hadoop2.jar
 base64-2.3.8.jar
 bcprov-jdk15on-1.51.jar
 bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
 calcite-avatica-1.2.0-incubating.jar
 calcite-core-1.2.0-incubating.jar
 calcite-linq4j-1.2.0-incubating.jar
 chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
 commons-beanutils-1.7.0.jar
 commons-beanutils-core-1.8.0.jar
 commons-cli-1.2.jar
@@ -87,10 +87,9 @@ jackson-core-asl-1.9.13.jar
 jackson-databind-2.5.3.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
 jackson-xc-1.9.13.jar
 janino-2.7.8.jar
-jansi-1.4.jar
 java-xmlbuilder-1.0.jar
 javax.inject-1.jar
 javax.servlet-3.0.0.v201112011016.jar
@@ -109,15 +108,14 @@ jettison-1.1.jar
 jetty-6.1.26.jar
 jetty-all-7.6.0.v20120127.jar
 jetty-util-6.1.26.jar
-jline-2.10.5.jar
 jline-2.12.jar
 joda-time-2.9.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
 json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
 jsp-api-2.1.jar
 jsr305-1.3.9.jar
 jta-1.1.jar
@@ -157,19 +155,20 @@ pmml-schema-1.2.7.jar
 protobuf-java-2.5.0.jar
 py4j-0.9.1.jar
 pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
 reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
 servlet-api-2.5.jar
 slf4j-api-1.7.10.jar
 slf4j-log4j12-1.7.10.jar
 snappy-0.2.jar
 snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
 stax-api-1.0-2.jar
 stax-api-1.0.1.jar
 stream-2.7.0.jar