diff options
| author | Bernardo Gomez Palacio <bernardo.gomezpalacio@gmail.com> | 2014-05-12 11:10:28 -0700 |
|---|---|---|
| committer | Patrick Wendell <pwendell@gmail.com> | 2014-05-12 11:10:28 -0700 |
| commit | d9c97ba39723d36d3dcb53a75a3a2565357c0955 (patch) | |
| tree | 0569444960ad9d41d94582e53d3f216957dc3d75 | |
| parent | 3af1f386439cdddd42e545ad63d089f4dfdf9f8a (diff) | |
| download | spark-d9c97ba39723d36d3dcb53a75a3a2565357c0955.tar.gz spark-d9c97ba39723d36d3dcb53a75a3a2565357c0955.tar.bz2 spark-d9c97ba39723d36d3dcb53a75a3a2565357c0955.zip | |
SPARK-1806: Upgrade Mesos dependency to 0.18.1
Enabled Mesos (0.18.1) dependency with shaded protobuf
Why is this needed?
Avoids any protobuf version collision between Mesos and any other
dependency in Spark e.g. Hadoop HDFS 2.2+ or 1.0.4.
Ticket: https://issues.apache.org/jira/browse/SPARK-1806
* Should close https://issues.apache.org/jira/browse/SPARK-1433
Author berngp
Author: Bernardo Gomez Palacio <bernardo.gomezpalacio@gmail.com>
Closes #741 from berngp/feature/SPARK-1806 and squashes the following commits:
5d70646 [Bernardo Gomez Palacio] SPARK-1806: Upgrade Mesos dependency to 0.18.1
| -rw-r--r-- | core/pom.xml | 3 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala | 2 | ||||
| -rw-r--r-- | core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala | 2 | ||||
| -rw-r--r-- | pom.xml | 10 | ||||
| -rw-r--r-- | project/SparkBuild.scala | 2 |
5 files changed, 14 insertions, 5 deletions
diff --git a/core/pom.xml b/core/pom.xml index 8fe215ab24..bab50f5ce2 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -140,6 +140,7 @@ <dependency> <groupId>org.apache.mesos</groupId> <artifactId>mesos</artifactId> + <classifier>${mesos.classifier}</classifier> </dependency> <dependency> <groupId>io.netty</groupId> @@ -322,7 +323,7 @@ </configuration> </plugin> </plugins> - + <resources> <resource> <directory>src/main/resources</directory> diff --git a/core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala index 9b56f711e0..74100498bb 100644 --- a/core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala +++ b/core/src/main/scala/org/apache/spark/executor/MesosExecutorBackend.scala @@ -19,7 +19,7 @@ package org.apache.spark.executor import java.nio.ByteBuffer -import com.google.protobuf.ByteString +import org.apache.mesos.protobuf.ByteString import org.apache.mesos.{Executor => MesosExecutor, ExecutorDriver, MesosExecutorDriver, MesosNativeLibrary} import org.apache.mesos.Protos.{TaskStatus => MesosTaskStatus, _} diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala index c975f31232..2cea1c8cf1 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackend.scala @@ -24,7 +24,7 @@ import java.util.Collections import scala.collection.JavaConversions._ import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet} -import com.google.protobuf.ByteString +import org.apache.mesos.protobuf.ByteString import org.apache.mesos.{Scheduler => MScheduler} import org.apache.mesos._ import org.apache.mesos.Protos.{TaskInfo => MesosTaskInfo, TaskState => MesosTaskState, _} @@ -113,7 +113,8 @@ <scala.version>2.10.4</scala.version> <scala.binary.version>2.10</scala.binary.version> - <mesos.version>0.13.0</mesos.version> + <mesos.version>0.18.1</mesos.version> + <mesos.classifier>shaded-protobuf</mesos.classifier> <akka.group>org.spark-project.akka</akka.group> <akka.version>2.2.3-shaded-protobuf</akka.version> <slf4j.version>1.7.5</slf4j.version> @@ -349,6 +350,13 @@ <groupId>org.apache.mesos</groupId> <artifactId>mesos</artifactId> <version>${mesos.version}</version> + <classifier>${mesos.classifier}</classifier> + <exclusions> + <exclusion> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>commons-net</groupId> diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 12791e490a..57b3e22f81 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -343,7 +343,7 @@ object SparkBuild extends Build { "org.spark-project.akka" %% "akka-testkit" % akkaVersion % "test", "org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap), "colt" % "colt" % "1.2.0", - "org.apache.mesos" % "mesos" % "0.13.0", + "org.apache.mesos" % "mesos" % "0.18.1" classifier("shaded-protobuf") exclude("com.google.protobuf", "protobuf-java"), "commons-net" % "commons-net" % "2.2", "net.java.dev.jets3t" % "jets3t" % jets3tVersion excludeAll(excludeCommonsLogging), "org.apache.derby" % "derby" % "10.4.2.0" % "test", |