diff options
| author | Josh Rosen <joshrosen@databricks.com> | 2016-01-30 00:20:28 -0800 |
|---|---|---|
| committer | Reynold Xin <rxin@databricks.com> | 2016-01-30 00:20:28 -0800 |
| commit | 289373b28cd2546165187de2e6a9185a1257b1e7 (patch) | |
| tree | b541a6e52a4ff20604689efafbfa0df7ad0901f5 | |
| parent | dab246f7e4664d36073ec49d9df8a11c5e998cdb (diff) | |
| download | spark-289373b28cd2546165187de2e6a9185a1257b1e7.tar.gz spark-289373b28cd2546165187de2e6a9185a1257b1e7.tar.bz2 spark-289373b28cd2546165187de2e6a9185a1257b1e7.zip | |
[SPARK-6363][BUILD] Make Scala 2.11 the default Scala version
This patch changes Spark's build to make Scala 2.11 the default Scala version. To be clear, this does not mean that Spark will stop supporting Scala 2.10: users will still be able to compile Spark for Scala 2.10 by following the instructions on the "Building Spark" page; however, it does mean that Scala 2.11 will be the default Scala version used by our CI builds (including pull request builds).
The Scala 2.11 compiler is faster than 2.10, so I think we'll be able to look forward to a slight speedup in our CI builds (it looks like it's about 2X faster for the Maven compile-only builds, for instance).
After this patch is merged, I'll update Jenkins to add new compile-only jobs to ensure that Scala 2.10 compilation doesn't break.
Author: Josh Rosen <joshrosen@databricks.com>
Closes #10608 from JoshRosen/SPARK-6363.
49 files changed, 186 insertions, 194 deletions
diff --git a/assembly/pom.xml b/assembly/pom.xml index 6c79f91897..477d4931c3 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-assembly_2.10</artifactId> + <artifactId>spark-assembly_2.11</artifactId> <name>Spark Project Assembly</name> <url>http://spark.apache.org/</url> <packaging>pom</packaging> diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 2cafe8c548..442043cb51 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-sketch_2.10</artifactId> + <artifactId>spark-sketch_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Sketch</name> <url>http://spark.apache.org/</url> diff --git a/core/pom.xml b/core/pom.xml index 0ab170e028..be40d9936a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-core_2.10</artifactId> + <artifactId>spark-core_2.11</artifactId> <properties> <sbt.project.name>core</sbt.project.name> </properties> diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh index 00bf81120d..2fd7fcc39e 100755 --- a/dev/create-release/release-build.sh +++ b/dev/create-release/release-build.sh @@ -134,9 +134,9 @@ if [[ "$1" == "package" ]]; then cd spark-$SPARK_VERSION-bin-$NAME - # TODO There should probably be a flag to make-distribution to allow 2.11 support - if [[ $FLAGS == *scala-2.11* ]]; then - ./dev/change-scala-version.sh 2.11 + # TODO There should probably be a flag to make-distribution to allow 2.10 support + if [[ $FLAGS == *scala-2.10* ]]; then + ./dev/change-scala-version.sh 2.10 fi export ZINC_PORT=$ZINC_PORT @@ -228,8 +228,8 @@ if [[ "$1" == "publish-snapshot" ]]; then $MVN -DzincPort=$ZINC_PORT --settings $tmp_settings -DskipTests $PUBLISH_PROFILES \ -Phive-thriftserver deploy - ./dev/change-scala-version.sh 2.11 - $MVN -DzincPort=$ZINC_PORT -Dscala-2.11 --settings $tmp_settings \ + ./dev/change-scala-version.sh 2.10 + $MVN -DzincPort=$ZINC_PORT -Dscala-2.10 --settings $tmp_settings \ -DskipTests $PUBLISH_PROFILES clean deploy # Clean-up Zinc nailgun process @@ -266,9 +266,9 @@ if [[ "$1" == "publish-release" ]]; then $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -DskipTests $PUBLISH_PROFILES \ -Phive-thriftserver clean install - ./dev/change-scala-version.sh 2.11 + ./dev/change-scala-version.sh 2.10 - $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.11 \ + $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.10 \ -DskipTests $PUBLISH_PROFILES clean install # Clean-up Zinc nailgun process diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2 index 4d9937c5cb..3a14499d9b 100644 --- a/dev/deps/spark-deps-hadoop-2.2 +++ b/dev/deps/spark-deps-hadoop-2.2 @@ -14,13 +14,13 @@ avro-ipc-1.7.7-tests.jar avro-ipc-1.7.7.jar avro-mapred-1.7.7-hadoop2.jar bonecp-0.8.0.RELEASE.jar -breeze-macros_2.10-0.11.2.jar -breeze_2.10-0.11.2.jar +breeze-macros_2.11-0.11.2.jar +breeze_2.11-0.11.2.jar calcite-avatica-1.2.0-incubating.jar calcite-core-1.2.0-incubating.jar calcite-linq4j-1.2.0-incubating.jar chill-java-0.5.0.jar -chill_2.10-0.5.0.jar +chill_2.11-0.5.0.jar commons-beanutils-1.7.0.jar commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar @@ -86,10 +86,9 @@ jackson-core-asl-1.9.13.jar jackson-databind-2.5.3.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-scala_2.10-2.5.3.jar +jackson-module-scala_2.11-2.5.3.jar jackson-xc-1.9.13.jar janino-2.7.8.jar -jansi-1.4.jar javax.inject-1.jar javax.servlet-3.0.0.v201112011016.jar javax.servlet-3.1.jar @@ -111,15 +110,14 @@ jets3t-0.7.1.jar jettison-1.1.jar jetty-all-7.6.0.v20120127.jar jetty-util-6.1.26.jar -jline-2.10.5.jar jline-2.12.jar joda-time-2.9.jar jodd-core-3.5.2.jar jpam-1.1.jar json-20090211.jar -json4s-ast_2.10-3.2.10.jar -json4s-core_2.10-3.2.10.jar -json4s-jackson_2.10-3.2.10.jar +json4s-ast_2.11-3.2.10.jar +json4s-core_2.11-3.2.10.jar +json4s-jackson_2.11-3.2.10.jar jsr305-1.3.9.jar jta-1.1.jar jtransforms-2.4.0.jar @@ -158,19 +156,20 @@ pmml-schema-1.2.7.jar protobuf-java-2.5.0.jar py4j-0.9.1.jar pyrolite-4.9.jar -quasiquotes_2.10-2.0.0-M8.jar reflectasm-1.07-shaded.jar -scala-compiler-2.10.5.jar -scala-library-2.10.5.jar -scala-reflect-2.10.5.jar -scalap-2.10.5.jar +scala-compiler-2.11.7.jar +scala-library-2.11.7.jar +scala-parser-combinators_2.11-1.0.4.jar +scala-reflect-2.11.7.jar +scala-xml_2.11-1.0.2.jar +scalap-2.11.7.jar servlet-api-2.5.jar slf4j-api-1.7.10.jar slf4j-log4j12-1.7.10.jar snappy-0.2.jar snappy-java-1.1.2.jar -spire-macros_2.10-0.7.4.jar -spire_2.10-0.7.4.jar +spire-macros_2.11-0.7.4.jar +spire_2.11-0.7.4.jar stax-api-1.0-2.jar stax-api-1.0.1.jar stream-2.7.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3 index fd659ee20d..615836b3d3 100644 --- a/dev/deps/spark-deps-hadoop-2.3 +++ b/dev/deps/spark-deps-hadoop-2.3 @@ -16,13 +16,13 @@ avro-mapred-1.7.7-hadoop2.jar base64-2.3.8.jar bcprov-jdk15on-1.51.jar bonecp-0.8.0.RELEASE.jar -breeze-macros_2.10-0.11.2.jar -breeze_2.10-0.11.2.jar +breeze-macros_2.11-0.11.2.jar +breeze_2.11-0.11.2.jar calcite-avatica-1.2.0-incubating.jar calcite-core-1.2.0-incubating.jar calcite-linq4j-1.2.0-incubating.jar chill-java-0.5.0.jar -chill_2.10-0.5.0.jar +chill_2.11-0.5.0.jar commons-beanutils-1.7.0.jar commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar @@ -81,10 +81,9 @@ jackson-core-asl-1.9.13.jar jackson-databind-2.5.3.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-scala_2.10-2.5.3.jar +jackson-module-scala_2.11-2.5.3.jar jackson-xc-1.9.13.jar janino-2.7.8.jar -jansi-1.4.jar java-xmlbuilder-1.0.jar javax.inject-1.jar javax.servlet-3.0.0.v201112011016.jar @@ -102,15 +101,14 @@ jettison-1.1.jar jetty-6.1.26.jar jetty-all-7.6.0.v20120127.jar jetty-util-6.1.26.jar -jline-2.10.5.jar jline-2.12.jar joda-time-2.9.jar jodd-core-3.5.2.jar jpam-1.1.jar json-20090211.jar -json4s-ast_2.10-3.2.10.jar -json4s-core_2.10-3.2.10.jar -json4s-jackson_2.10-3.2.10.jar +json4s-ast_2.11-3.2.10.jar +json4s-core_2.11-3.2.10.jar +json4s-jackson_2.11-3.2.10.jar jsr305-1.3.9.jar jta-1.1.jar jtransforms-2.4.0.jar @@ -149,19 +147,20 @@ pmml-schema-1.2.7.jar protobuf-java-2.5.0.jar py4j-0.9.1.jar pyrolite-4.9.jar -quasiquotes_2.10-2.0.0-M8.jar reflectasm-1.07-shaded.jar -scala-compiler-2.10.5.jar -scala-library-2.10.5.jar -scala-reflect-2.10.5.jar -scalap-2.10.5.jar +scala-compiler-2.11.7.jar +scala-library-2.11.7.jar +scala-parser-combinators_2.11-1.0.4.jar +scala-reflect-2.11.7.jar +scala-xml_2.11-1.0.2.jar +scalap-2.11.7.jar servlet-api-2.5.jar slf4j-api-1.7.10.jar slf4j-log4j12-1.7.10.jar snappy-0.2.jar snappy-java-1.1.2.jar -spire-macros_2.10-0.7.4.jar -spire_2.10-0.7.4.jar +spire-macros_2.11-0.7.4.jar +spire_2.11-0.7.4.jar stax-api-1.0-2.jar stax-api-1.0.1.jar stream-2.7.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4 index afae3deb9a..f275226f1d 100644 --- a/dev/deps/spark-deps-hadoop-2.4 +++ b/dev/deps/spark-deps-hadoop-2.4 @@ -16,13 +16,13 @@ avro-mapred-1.7.7-hadoop2.jar base64-2.3.8.jar bcprov-jdk15on-1.51.jar bonecp-0.8.0.RELEASE.jar -breeze-macros_2.10-0.11.2.jar -breeze_2.10-0.11.2.jar +breeze-macros_2.11-0.11.2.jar +breeze_2.11-0.11.2.jar calcite-avatica-1.2.0-incubating.jar calcite-core-1.2.0-incubating.jar calcite-linq4j-1.2.0-incubating.jar chill-java-0.5.0.jar -chill_2.10-0.5.0.jar +chill_2.11-0.5.0.jar commons-beanutils-1.7.0.jar commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar @@ -81,10 +81,9 @@ jackson-core-asl-1.9.13.jar jackson-databind-2.5.3.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-scala_2.10-2.5.3.jar +jackson-module-scala_2.11-2.5.3.jar jackson-xc-1.9.13.jar janino-2.7.8.jar -jansi-1.4.jar java-xmlbuilder-1.0.jar javax.inject-1.jar javax.servlet-3.0.0.v201112011016.jar @@ -103,15 +102,14 @@ jettison-1.1.jar jetty-6.1.26.jar jetty-all-7.6.0.v20120127.jar jetty-util-6.1.26.jar -jline-2.10.5.jar jline-2.12.jar joda-time-2.9.jar jodd-core-3.5.2.jar jpam-1.1.jar json-20090211.jar -json4s-ast_2.10-3.2.10.jar -json4s-core_2.10-3.2.10.jar -json4s-jackson_2.10-3.2.10.jar +json4s-ast_2.11-3.2.10.jar +json4s-core_2.11-3.2.10.jar +json4s-jackson_2.11-3.2.10.jar jsr305-1.3.9.jar jta-1.1.jar jtransforms-2.4.0.jar @@ -150,19 +148,20 @@ pmml-schema-1.2.7.jar protobuf-java-2.5.0.jar py4j-0.9.1.jar pyrolite-4.9.jar -quasiquotes_2.10-2.0.0-M8.jar reflectasm-1.07-shaded.jar -scala-compiler-2.10.5.jar -scala-library-2.10.5.jar -scala-reflect-2.10.5.jar -scalap-2.10.5.jar +scala-compiler-2.11.7.jar +scala-library-2.11.7.jar +scala-parser-combinators_2.11-1.0.4.jar +scala-reflect-2.11.7.jar +scala-xml_2.11-1.0.2.jar +scalap-2.11.7.jar servlet-api-2.5.jar slf4j-api-1.7.10.jar slf4j-log4j12-1.7.10.jar snappy-0.2.jar snappy-java-1.1.2.jar -spire-macros_2.10-0.7.4.jar -spire_2.10-0.7.4.jar +spire-macros_2.11-0.7.4.jar +spire_2.11-0.7.4.jar stax-api-1.0-2.jar stax-api-1.0.1.jar stream-2.7.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 5a6460136a..21432a16e3 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -20,13 +20,13 @@ avro-mapred-1.7.7-hadoop2.jar base64-2.3.8.jar bcprov-jdk15on-1.51.jar bonecp-0.8.0.RELEASE.jar -breeze-macros_2.10-0.11.2.jar -breeze_2.10-0.11.2.jar +breeze-macros_2.11-0.11.2.jar +breeze_2.11-0.11.2.jar calcite-avatica-1.2.0-incubating.jar calcite-core-1.2.0-incubating.jar calcite-linq4j-1.2.0-incubating.jar chill-java-0.5.0.jar -chill_2.10-0.5.0.jar +chill_2.11-0.5.0.jar commons-beanutils-1.7.0.jar commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar @@ -87,10 +87,9 @@ jackson-core-asl-1.9.13.jar jackson-databind-2.5.3.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-scala_2.10-2.5.3.jar +jackson-module-scala_2.11-2.5.3.jar jackson-xc-1.9.13.jar janino-2.7.8.jar -jansi-1.4.jar java-xmlbuilder-1.0.jar javax.inject-1.jar javax.servlet-3.0.0.v201112011016.jar @@ -109,15 +108,14 @@ jettison-1.1.jar jetty-6.1.26.jar jetty-all-7.6.0.v20120127.jar jetty-util-6.1.26.jar -jline-2.10.5.jar jline-2.12.jar joda-time-2.9.jar jodd-core-3.5.2.jar jpam-1.1.jar json-20090211.jar -json4s-ast_2.10-3.2.10.jar -json4s-core_2.10-3.2.10.jar -json4s-jackson_2.10-3.2.10.jar +json4s-ast_2.11-3.2.10.jar +json4s-core_2.11-3.2.10.jar +json4s-jackson_2.11-3.2.10.jar jsr305-1.3.9.jar jta-1.1.jar jtransforms-2.4.0.jar @@ -156,19 +154,20 @@ pmml-schema-1.2.7.jar protobuf-java-2.5.0.jar py4j-0.9.1.jar pyrolite-4.9.jar -quasiquotes_2.10-2.0.0-M8.jar reflectasm-1.07-shaded.jar -scala-compiler-2.10.5.jar -scala-library-2.10.5.jar -scala-reflect-2.10.5.jar -scalap-2.10.5.jar +scala-compiler-2.11.7.jar +scala-library-2.11.7.jar +scala-parser-combinators_2.11-1.0.4.jar +scala-reflect-2.11.7.jar +scala-xml_2.11-1.0.2.jar +scalap-2.11.7.jar servlet-api-2.5.jar slf4j-api-1.7.10.jar slf4j-log4j12-1.7.10.jar snappy-0.2.jar snappy-java-1.1.2.jar -spire-macros_2.10-0.7.4.jar -spire_2.10-0.7.4.jar +spire-macros_2.11-0.7.4.jar +spire_2.11-0.7.4.jar stax-api-1.0-2.jar stax-api-1.0.1.jar stream-2.7.0.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 70083e7f3d..20e09cd002 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -20,13 +20,13 @@ avro-mapred-1.7.7-hadoop2.jar base64-2.3.8.jar bcprov-jdk15on-1.51.jar bonecp-0.8.0.RELEASE.jar -breeze-macros_2.10-0.11.2.jar -breeze_2.10-0.11.2.jar +breeze-macros_2.11-0.11.2.jar +breeze_2.11-0.11.2.jar calcite-avatica-1.2.0-incubating.jar calcite-core-1.2.0-incubating.jar calcite-linq4j-1.2.0-incubating.jar chill-java-0.5.0.jar -chill_2.10-0.5.0.jar +chill_2.11-0.5.0.jar commons-beanutils-1.7.0.jar commons-beanutils-core-1.8.0.jar commons-cli-1.2.jar @@ -87,10 +87,9 @@ jackson-core-asl-1.9.13.jar jackson-databind-2.5.3.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-scala_2.10-2.5.3.jar +jackson-module-scala_2.11-2.5.3.jar jackson-xc-1.9.13.jar janino-2.7.8.jar -jansi-1.4.jar java-xmlbuilder-1.0.jar javax.inject-1.jar javax.servlet-3.0.0.v201112011016.jar @@ -109,15 +108,14 @@ jettison-1.1.jar jetty-6.1.26.jar jetty-all-7.6.0.v20120127.jar jetty-util-6.1.26.jar -jline-2.10.5.jar jline-2.12.jar joda-time-2.9.jar jodd-core-3.5.2.jar jpam-1.1.jar json-20090211.jar -json4s-ast_2.10-3.2.10.jar -json4s-core_2.10-3.2.10.jar -json4s-jackson_2.10-3.2.10.jar +json4s-ast_2.11-3.2.10.jar +json4s-core_2.11-3.2.10.jar +json4s-jackson_2.11-3.2.10.jar jsp-api-2.1.jar jsr305-1.3.9.jar jta-1.1.jar @@ -157,19 +155,20 @@ pmml-schema-1.2.7.jar protobuf-java-2.5.0.jar py4j-0.9.1.jar pyrolite-4.9.jar -quasiquotes_2.10-2.0.0-M8.jar reflectasm-1.07-shaded.jar -scala-compiler-2.10.5.jar -scala-library-2.10.5.jar -scala-reflect-2.10.5.jar -scalap-2.10.5.jar +scala-compiler-2.11.7.jar +scala-library-2.11.7.jar +scala-parser-combinators_2.11-1.0.4.jar +scala-reflect-2.11.7.jar +scala-xml_2.11-1.0.2.jar +scalap-2.11.7.jar servlet-api-2.5.jar slf4j-api-1.7.10.jar slf4j-log4j12-1.7.10.jar snappy-0.2.jar snappy-java-1.1.2.jar -spire-macros_2.10-0.7.4.jar -spire_2.10-0.7.4.jar +spire-macros_2.11-0.7.4.jar +spire_2.11-0.7.4.jar stax-api-1.0-2.jar stax-api-1.0.1.jar stream-2.7.0.jar diff --git a/docker-integration-tests/pom.xml b/docker-integration-tests/pom.xml index 78b638ecfa..833ca29cd8 100644 --- a/docker-integration-tests/pom.xml +++ b/docker-integration-tests/pom.xml @@ -21,12 +21,12 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> - <artifactId>spark-docker-integration-tests_2.10</artifactId> + <artifactId>spark-docker-integration-tests_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Docker Integration Tests</name> <url>http://spark.apache.org/</url> diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb index 174c202e37..f926d67e6b 100644 --- a/docs/_plugins/copy_api_dirs.rb +++ b/docs/_plugins/copy_api_dirs.rb @@ -37,7 +37,7 @@ if not (ENV['SKIP_API'] == '1') # Copy over the unified ScalaDoc for all projects to api/scala. # This directory will be copied over to _site when `jekyll` command is run. - source = "../target/scala-2.10/unidoc" + source = "../target/scala-2.11/unidoc" dest = "api/scala" puts "Making directory " + dest diff --git a/docs/building-spark.md b/docs/building-spark.md index e1abcf1be5..975e1b295c 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -114,13 +114,11 @@ By default Spark will build with Hive 0.13.1 bindings. mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -Phive -Phive-thriftserver -DskipTests clean package {% endhighlight %} -# Building for Scala 2.11 -To produce a Spark package compiled with Scala 2.11, use the `-Dscala-2.11` property: +# Building for Scala 2.10 +To produce a Spark package compiled with Scala 2.10, use the `-Dscala-2.10` property: - ./dev/change-scala-version.sh 2.11 - mvn -Pyarn -Phadoop-2.4 -Dscala-2.11 -DskipTests clean package - -Spark does not yet support its JDBC component for Scala 2.11. + ./dev/change-scala-version.sh 2.10 + mvn -Pyarn -Phadoop-2.4 -Dscala-2.10 -DskipTests clean package # Spark Tests in Maven diff --git a/examples/pom.xml b/examples/pom.xml index 9437cee2ab..82baa9085b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-examples_2.10</artifactId> + <artifactId>spark-examples_2.11</artifactId> <properties> <sbt.project.name>examples</sbt.project.name> </properties> diff --git a/external/akka/pom.xml b/external/akka/pom.xml index 06c8e8aaab..bbe644e3b3 100644 --- a/external/akka/pom.xml +++ b/external/akka/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-akka_2.10</artifactId> + <artifactId>spark-streaming-akka_2.11</artifactId> <properties> <sbt.project.name>streaming-akka</sbt.project.name> </properties> diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index b2c377fe4c..ac15b93c04 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-flume-assembly_2.10</artifactId> + <artifactId>spark-streaming-flume-assembly_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project External Flume Assembly</name> <url>http://spark.apache.org/</url> diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 4b6485ee0a..e4effe158c 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-flume-sink_2.10</artifactId> + <artifactId>spark-streaming-flume-sink_2.11</artifactId> <properties> <sbt.project.name>streaming-flume-sink</sbt.project.name> </properties> diff --git a/external/flume/pom.xml b/external/flume/pom.xml index a79656c6f7..d650dd034d 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-flume_2.10</artifactId> + <artifactId>spark-streaming-flume_2.11</artifactId> <properties> <sbt.project.name>streaming-flume</sbt.project.name> </properties> diff --git a/external/kafka-assembly/pom.xml b/external/kafka-assembly/pom.xml index 0c466b3c4a..62818f5e8f 100644 --- a/external/kafka-assembly/pom.xml +++ b/external/kafka-assembly/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-kafka-assembly_2.10</artifactId> + <artifactId>spark-streaming-kafka-assembly_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project External Kafka Assembly</name> <url>http://spark.apache.org/</url> diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index 5180ab6dba..68d52e9339 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-kafka_2.10</artifactId> + <artifactId>spark-streaming-kafka_2.11</artifactId> <properties> <sbt.project.name>streaming-kafka</sbt.project.name> </properties> diff --git a/external/mqtt-assembly/pom.xml b/external/mqtt-assembly/pom.xml index c4a1ae26ea..ac2a3f65ed 100644 --- a/external/mqtt-assembly/pom.xml +++ b/external/mqtt-assembly/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-mqtt-assembly_2.10</artifactId> + <artifactId>spark-streaming-mqtt-assembly_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project External MQTT Assembly</name> <url>http://spark.apache.org/</url> diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml index d3a2bf5825..d0d968782c 100644 --- a/external/mqtt/pom.xml +++ b/external/mqtt/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-mqtt_2.10</artifactId> + <artifactId>spark-streaming-mqtt_2.11</artifactId> <properties> <sbt.project.name>streaming-mqtt</sbt.project.name> </properties> diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml index 7b628b09ea..5d4053afcb 100644 --- a/external/twitter/pom.xml +++ b/external/twitter/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-twitter_2.10</artifactId> + <artifactId>spark-streaming-twitter_2.11</artifactId> <properties> <sbt.project.name>streaming-twitter</sbt.project.name> </properties> diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml index 7781aaeed9..f16bc0f319 100644 --- a/external/zeromq/pom.xml +++ b/external/zeromq/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-zeromq_2.10</artifactId> + <artifactId>spark-streaming-zeromq_2.11</artifactId> <properties> <sbt.project.name>streaming-zeromq</sbt.project.name> </properties> diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml index 4dfe3b654d..0ad9c5303a 100644 --- a/extras/java8-tests/pom.xml +++ b/extras/java8-tests/pom.xml @@ -19,13 +19,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>java8-tests_2.10</artifactId> + <artifactId>java8-tests_2.11</artifactId> <packaging>pom</packaging> <name>Spark Project Java8 Tests POM</name> diff --git a/extras/kinesis-asl-assembly/pom.xml b/extras/kinesis-asl-assembly/pom.xml index 601080c2e6..d1c38c7ca5 100644 --- a/extras/kinesis-asl-assembly/pom.xml +++ b/extras/kinesis-asl-assembly/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-kinesis-asl-assembly_2.10</artifactId> + <artifactId>spark-streaming-kinesis-asl-assembly_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Kinesis Assembly</name> <url>http://spark.apache.org/</url> diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml index 20e2c5e0ff..935155eb5d 100644 --- a/extras/kinesis-asl/pom.xml +++ b/extras/kinesis-asl/pom.xml @@ -19,14 +19,14 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <!-- Kinesis integration is not included by default due to ASL-licensed code. --> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming-kinesis-asl_2.10</artifactId> + <artifactId>spark-streaming-kinesis-asl_2.11</artifactId> <packaging>jar</packaging> <name>Spark Kinesis Integration</name> diff --git a/extras/spark-ganglia-lgpl/pom.xml b/extras/spark-ganglia-lgpl/pom.xml index b046a10a04..bfb92791de 100644 --- a/extras/spark-ganglia-lgpl/pom.xml +++ b/extras/spark-ganglia-lgpl/pom.xml @@ -19,14 +19,14 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <!-- Ganglia integration is not included by default due to LGPL-licensed code --> <groupId>org.apache.spark</groupId> - <artifactId>spark-ganglia-lgpl_2.10</artifactId> + <artifactId>spark-ganglia-lgpl_2.11</artifactId> <packaging>jar</packaging> <name>Spark Ganglia Integration</name> diff --git a/graphx/pom.xml b/graphx/pom.xml index 388a0ef06a..1813f383cd 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-graphx_2.10</artifactId> + <artifactId>spark-graphx_2.11</artifactId> <properties> <sbt.project.name>graphx</sbt.project.name> </properties> diff --git a/launcher/pom.xml b/launcher/pom.xml index 135866cea2..ef73194882 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-launcher_2.10</artifactId> + <artifactId>spark-launcher_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Launcher</name> <url>http://spark.apache.org/</url> diff --git a/mllib/pom.xml b/mllib/pom.xml index 42af2b8b3e..816f3f6830 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-mllib_2.10</artifactId> + <artifactId>spark-mllib_2.11</artifactId> <properties> <sbt.project.name>mllib</sbt.project.name> </properties> diff --git a/network/common/pom.xml b/network/common/pom.xml index eda2b73070..bd507c2cb6 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-network-common_2.10</artifactId> + <artifactId>spark-network-common_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Networking</name> <url>http://spark.apache.org/</url> diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index f9aa7e2dd1..810ec10ca0 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-network-shuffle_2.10</artifactId> + <artifactId>spark-network-shuffle_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Shuffle Streaming Service</name> <url>http://spark.apache.org/</url> diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index a19cbb04b1..a28785b16e 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-network-yarn_2.10</artifactId> + <artifactId>spark-network-yarn_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project YARN Shuffle Service</name> <url>http://spark.apache.org/</url> @@ -25,7 +25,7 @@ <version>14</version> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <packaging>pom</packaging> <name>Spark Project Parent POM</name> @@ -165,7 +165,7 @@ <!-- managed up from 3.2.1 for SPARK-11652 --> <commons.collections.version>3.2.2</commons.collections.version> <scala.version>2.10.5</scala.version> - <scala.binary.version>2.10</scala.binary.version> + <scala.binary.version>2.11</scala.binary.version> <jline.version>${scala.version}</jline.version> <jline.groupid>org.scala-lang</jline.groupid> <codehaus.jackson.version>1.9.13</codehaus.jackson.version> @@ -2456,7 +2456,7 @@ <profile> <id>scala-2.10</id> <activation> - <property><name>!scala-2.11</name></property> + <property><name>scala-2.10</name></property> </activation> <properties> <scala.version>2.10.5</scala.version> @@ -2488,7 +2488,7 @@ <profile> <id>scala-2.11</id> <activation> - <property><name>scala-2.11</name></property> + <property><name>!scala-2.10</name></property> </activation> <properties> <scala.version>2.11.7</scala.version> diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala index 41856443af..4adf64a5a0 100644 --- a/project/MimaBuild.scala +++ b/project/MimaBuild.scala @@ -95,7 +95,7 @@ object MimaBuild { // because spark-streaming-mqtt(1.6.0) depends on it. // Remove the setting on updating previousSparkVersion. val previousSparkVersion = "1.6.0" - val fullId = "spark-" + projectRef.project + "_2.10" + val fullId = "spark-" + projectRef.project + "_2.11" mimaDefaultSettings ++ Seq(previousArtifact := Some(organization % fullId % previousSparkVersion), binaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value), diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index a3ae4d2b73..3748e07f88 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -220,6 +220,12 @@ object MimaExcludes { // SPARK-11622 Make LibSVMRelation extends HadoopFsRelation and Add LibSVMOutputWriter ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ml.source.libsvm.DefaultSource"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.ml.source.libsvm.DefaultSource.createRelation") + ) ++ Seq( + // SPARK-6363 Make Scala 2.11 the default Scala version + ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.SparkContext.cleanup"), + ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.SparkContext.metadataCleaner"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint") ) case v if v.startsWith("1.6") => Seq( diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 4224a65a82..550b5bad8a 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -119,11 +119,11 @@ object SparkBuild extends PomBuild { v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq } - if (System.getProperty("scala-2.11") == "") { - // To activate scala-2.11 profile, replace empty property value to non-empty value + if (System.getProperty("scala-2.10") == "") { + // To activate scala-2.10 profile, replace empty property value to non-empty value // in the same way as Maven which handles -Dname as -Dname=true before executes build process. // see: https://github.com/apache/maven/blob/maven-3.0.4/maven-embedder/src/main/java/org/apache/maven/cli/MavenCli.java#L1082 - System.setProperty("scala-2.11", "true") + System.setProperty("scala-2.10", "true") } profiles } @@ -382,7 +382,7 @@ object OldDeps { lazy val project = Project("oldDeps", file("dev"), settings = oldDepsSettings) def versionArtifact(id: String): Option[sbt.ModuleID] = { - val fullId = id + "_2.10" + val fullId = id + "_2.11" Some("org.apache.spark" % fullId % "1.2.0") } @@ -390,7 +390,7 @@ object OldDeps { name := "old-deps", scalaVersion := "2.10.5", libraryDependencies := Seq("spark-streaming-mqtt", "spark-streaming-zeromq", - "spark-streaming-flume", "spark-streaming-kafka", "spark-streaming-twitter", + "spark-streaming-flume", "spark-streaming-twitter", "spark-streaming", "spark-mllib", "spark-graphx", "spark-core").map(versionArtifact(_).get intransitive()) ) @@ -704,7 +704,7 @@ object Java8TestSettings { lazy val settings = Seq( javacJVMVersion := "1.8", // Targeting Java 8 bytecode is only supported in Scala 2.11.4 and higher: - scalacJVMVersion := (if (System.getProperty("scala-2.11") == "true") "1.8" else "1.7") + scalacJVMVersion := (if (System.getProperty("scala-2.10") == "true") "1.7" else "1.8") ) } diff --git a/repl/pom.xml b/repl/pom.xml index efc3dd452e..0f396c9b80 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-repl_2.10</artifactId> + <artifactId>spark-repl_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project REPL</name> <url>http://spark.apache.org/</url> @@ -159,7 +159,7 @@ <profile> <id>scala-2.10</id> <activation> - <property><name>!scala-2.11</name></property> + <property><name>scala-2.10</name></property> </activation> <dependencies> <dependency> @@ -173,7 +173,7 @@ <profile> <id>scala-2.11</id> <activation> - <property><name>scala-2.11</name></property> + <property><name>!scala-2.10</name></property> </activation> <properties> <extra.source.dir>scala-2.11/src/main/scala</extra.source.dir> diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala index bb3081d129..07ba28bb07 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala @@ -33,7 +33,8 @@ object Main extends Logging { var sparkContext: SparkContext = _ var sqlContext: SQLContext = _ - var interp = new SparkILoop // this is a public var because tests reset it. + // this is a public var because tests reset it. + var interp: SparkILoop = _ private var hasErrors = false @@ -43,6 +44,12 @@ object Main extends Logging { } def main(args: Array[String]) { + doMain(args, new SparkILoop) + } + + // Visible for testing + private[repl] def doMain(args: Array[String], _interp: SparkILoop): Unit = { + interp = _interp val interpArguments = List( "-Yrepl-class-based", "-Yrepl-outdir", s"${outputDir.getAbsolutePath}", diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala index 63f3688c9e..b9ed79da42 100644 --- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala +++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala @@ -50,12 +50,7 @@ class ReplSuite extends SparkFunSuite { System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath) System.setProperty("spark.master", master) - val interp = { - new SparkILoop(in, new PrintWriter(out)) - } - org.apache.spark.repl.Main.interp = interp - Main.main(Array("-classpath", classpath)) // call main - org.apache.spark.repl.Main.interp = null + Main.doMain(Array("-classpath", classpath), new SparkILoop(in, new PrintWriter(out))) if (oldExecutorClasspath != null) { System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath) diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 76ca3f3bb1..c2ad9b99f3 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-catalyst_2.10</artifactId> + <artifactId>spark-catalyst_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Catalyst</name> <url>http://spark.apache.org/</url> @@ -127,13 +127,4 @@ </plugin> </plugins> </build> - <profiles> - <!-- Quasiquotes are merged into scala reflect from scala 2.11 onwards. --> - <profile> - <id>scala-2.10</id> - <activation> - <property><name>!scala-2.11</name></property> - </activation> - </profile> - </profiles> </project> diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 4bb55f6b7f..89e01fc015 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-sql_2.10</artifactId> + <artifactId>spark-sql_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project SQL</name> <url>http://spark.apache.org/</url> @@ -44,7 +44,7 @@ </dependency> <dependency> <groupId>org.apache.spark</groupId> - <artifactId>spark-sketch_2.10</artifactId> + <artifactId>spark-sketch_2.11</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 435e565f63..c8d17bd468 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-hive-thriftserver_2.10</artifactId> + <artifactId>spark-hive-thriftserver_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Hive Thrift Server</name> <url>http://spark.apache.org/</url> diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index cd0c2aeb93..14cf9acf09 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-hive_2.10</artifactId> + <artifactId>spark-hive_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Hive</name> <url>http://spark.apache.org/</url> diff --git a/streaming/pom.xml b/streaming/pom.xml index 39cbd0d00f..7d409c5d3b 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -20,13 +20,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming_2.10</artifactId> + <artifactId>spark-streaming_2.11</artifactId> <properties> <sbt.project.name>streaming</sbt.project.name> </properties> diff --git a/tags/pom.xml b/tags/pom.xml index 9e4610dae7..3e8e6f6182 100644 --- a/tags/pom.xml +++ b/tags/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-test-tags_2.10</artifactId> + <artifactId>spark-test-tags_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Test Tags</name> <url>http://spark.apache.org/</url> diff --git a/tools/pom.xml b/tools/pom.xml index 30cbb6a5a5..b3a5ae2771 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,13 +19,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-tools_2.10</artifactId> + <artifactId>spark-tools_2.11</artifactId> <properties> <sbt.project.name>tools</sbt.project.name> </properties> diff --git a/unsafe/pom.xml b/unsafe/pom.xml index 21fef3415a..75fea556ee 100644 --- a/unsafe/pom.xml +++ b/unsafe/pom.xml @@ -21,13 +21,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-unsafe_2.10</artifactId> + <artifactId>spark-unsafe_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project Unsafe</name> <url>http://spark.apache.org/</url> diff --git a/yarn/pom.xml b/yarn/pom.xml index a8c122fd40..328bb6678d 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -19,13 +19,13 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-parent_2.10</artifactId> + <artifactId>spark-parent_2.11</artifactId> <version>2.0.0-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <groupId>org.apache.spark</groupId> - <artifactId>spark-yarn_2.10</artifactId> + <artifactId>spark-yarn_2.11</artifactId> <packaging>jar</packaging> <name>Spark Project YARN</name> <properties> |