diff options
Diffstat (limited to 'project')
| -rw-r--r-- | project/SparkBuild.scala | 32 |
1 files changed, 23 insertions, 9 deletions
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index bccf36c00f..3dd7c8c962 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -30,6 +30,10 @@ object SparkBuild extends Build { val DEFAULT_HADOOP_VERSION = "1.0.4" val DEFAULT_YARN = false + // Whether the Hadoop version to build against is 2.2.x, or a variant of it. This can be set + // through the SPARK_IS_NEW_YARN environment variable. + val DEFAULT_IS_NEW_YARN = false + // HBase version; set as appropriate. val HBASE_VERSION = "0.94.6" @@ -55,8 +59,6 @@ object SparkBuild extends Build { lazy val mllib = Project("mllib", file("mllib"), settings = mllibSettings) dependsOn(core) - lazy val yarn = Project("yarn", file("yarn"), settings = yarnSettings) dependsOn(core) - lazy val assemblyProj = Project("assembly", file("assembly"), settings = assemblyProjSettings) .dependsOn(core, bagel, mllib, repl, streaming) dependsOn(maybeYarn: _*) @@ -68,14 +70,26 @@ object SparkBuild extends Build { // Allows build configuration to be set through environment variables lazy val hadoopVersion = scala.util.Properties.envOrElse("SPARK_HADOOP_VERSION", DEFAULT_HADOOP_VERSION) + lazy val isNewYarn = scala.util.Properties.envOrNone("SPARK_IS_NEW_YARN") match { + case None => { + val isNewHadoopVersion = "2.[2-9]*".r.findFirstIn(hadoopVersion).isDefined + (isNewHadoopVersion|| DEFAULT_IS_NEW_YARN) + } + case Some(v) => v.toBoolean + } lazy val isYarnEnabled = scala.util.Properties.envOrNone("SPARK_YARN") match { case None => DEFAULT_YARN case Some(v) => v.toBoolean } + // Build against a protobuf-2.5 compatible Akka if Hadoop 2 is used. + lazy val protobufVersion = if (isNewYarn) "2.5.0" else "2.4.1" + lazy val akkaVersion = if (isNewYarn) "2.0.5-protobuf-2.5" else "2.0.5" + // Conditionally include the yarn sub-project - lazy val maybeYarn = if(isYarnEnabled) Seq[ClasspathDependency](yarn) else Seq[ClasspathDependency]() - lazy val maybeYarnRef = if(isYarnEnabled) Seq[ProjectReference](yarn) else Seq[ProjectReference]() + lazy val yarn = Project("yarn", file(if (isNewYarn) "new-yarn" else "yarn"), settings = yarnSettings) dependsOn(core) + lazy val maybeYarn = if (isYarnEnabled) Seq[ClasspathDependency](yarn) else Seq[ClasspathDependency]() + lazy val maybeYarnRef = if (isYarnEnabled) Seq[ProjectReference](yarn) else Seq[ProjectReference]() // Everything except assembly, tools and examples belong to packageProjects lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib) ++ maybeYarnRef @@ -211,10 +225,10 @@ object SparkBuild extends Build { "com.ning" % "compress-lzf" % "0.8.4", "org.xerial.snappy" % "snappy-java" % "1.0.5", "org.ow2.asm" % "asm" % "4.0", - "com.google.protobuf" % "protobuf-java" % "2.4.1", - "com.typesafe.akka" % "akka-actor" % "2.0.5" excludeAll(excludeNetty), - "com.typesafe.akka" % "akka-remote" % "2.0.5" excludeAll(excludeNetty), - "com.typesafe.akka" % "akka-slf4j" % "2.0.5" excludeAll(excludeNetty), + "com.google.protobuf" % "protobuf-java" % protobufVersion, + "com.typesafe.akka" % "akka-actor" % akkaVersion excludeAll(excludeNetty), + "com.typesafe.akka" % "akka-remote" % akkaVersion excludeAll(excludeNetty), + "com.typesafe.akka" % "akka-slf4j" % akkaVersion excludeAll(excludeNetty), "it.unimi.dsi" % "fastutil" % "6.4.4", "colt" % "colt" % "1.2.0", "net.liftweb" % "lift-json_2.9.2" % "2.5", @@ -289,7 +303,7 @@ object SparkBuild extends Build { "org.eclipse.paho" % "mqtt-client" % "0.4.0", "org.apache.flume" % "flume-ng-sdk" % "1.2.0" % "compile" excludeAll(excludeNetty, excludeSnappy), "org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty), - "com.typesafe.akka" % "akka-zeromq" % "2.0.5" excludeAll(excludeNetty), + "com.typesafe.akka" % "akka-zeromq" % akkaVersion excludeAll(excludeNetty), "org.apache.kafka" % "kafka_2.9.2" % "0.8.0-beta1" exclude("com.sun.jdmk", "jmxtools") exclude("com.sun.jmx", "jmxri") |