From e21a707a13b437327cef25d44ef08ddb2e3931af Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Tue, 7 Jan 2014 00:21:43 -0800 Subject: Adding unit tests and some refactoring to promote testability. --- project/SparkBuild.scala | 1 + 1 file changed, 1 insertion(+) (limited to 'project/SparkBuild.scala') diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 051e5105f3..bd5f3f79c9 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -233,6 +233,7 @@ object SparkBuild extends Build { "org.ow2.asm" % "asm" % "4.0", "org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty), "org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty), + "org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test", "net.liftweb" %% "lift-json" % "2.5.1" excludeAll(excludeNetty), "it.unimi.dsi" % "fastutil" % "6.4.4", "colt" % "colt" % "1.2.0", -- cgit v1.2.3 From dd7fd0c4433398b2e0a86e008a4d64dafc2f11be Mon Sep 17 00:00:00 2001 From: jerryshao Date: Fri, 10 Jan 2014 08:53:59 +0800 Subject: Upgrade Kafka dependecy to 0.8.0 release version --- external/kafka/pom.xml | 4 ++-- project/SparkBuild.scala | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'project/SparkBuild.scala') diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index f782e0e126..23b2fead65 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -45,9 +45,9 @@ test - com.sksamuel.kafka + org.apache.kafka kafka_${scala.binary.version} - 0.8.0-beta1 + 0.8.0 com.sun.jmx diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index bd5816ea28..c1a5c4df6f 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -90,7 +90,7 @@ object SparkBuild extends Build { lazy val maybeYarn = if (isYarnEnabled) Seq[ClasspathDependency](if (isNewHadoop) yarn else yarnAlpha) else Seq[ClasspathDependency]() lazy val maybeYarnRef = if (isYarnEnabled) Seq[ProjectReference](if (isNewHadoop) yarn else yarnAlpha) else Seq[ProjectReference]() - lazy val externalTwitter = Project("external-twitter", file("external/twitter"), settings = twitterSettings) + lazy val externalTwitter = Project("external-twitter", file("external/twitter"), settings = twitterSettings) .dependsOn(streaming % "compile->compile;test->test") lazy val externalKafka = Project("external-kafka", file("external/kafka"), settings = kafkaSettings) @@ -98,23 +98,23 @@ object SparkBuild extends Build { lazy val externalFlume = Project("external-flume", file("external/flume"), settings = flumeSettings) .dependsOn(streaming % "compile->compile;test->test") - + lazy val externalZeromq = Project("external-zeromq", file("external/zeromq"), settings = zeromqSettings) .dependsOn(streaming % "compile->compile;test->test") - + lazy val externalMqtt = Project("external-mqtt", file("external/mqtt"), settings = mqttSettings) .dependsOn(streaming % "compile->compile;test->test") lazy val allExternal = Seq[ClasspathDependency](externalTwitter, externalKafka, externalFlume, externalZeromq, externalMqtt) lazy val allExternalRefs = Seq[ProjectReference](externalTwitter, externalKafka, externalFlume, externalZeromq, externalMqtt) - + lazy val examples = Project("examples", file("examples"), settings = examplesSettings) .dependsOn(core, mllib, bagel, streaming, externalTwitter) dependsOn(allExternal: _*) // Everything except assembly, tools and examples belong to packageProjects lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib) ++ maybeYarnRef - lazy val allProjects = packageProjects ++ allExternalRefs ++ Seq[ProjectReference](examples, tools, assemblyProj) + lazy val allProjects = packageProjects ++ allExternalRefs ++ Seq[ProjectReference](examples, tools, assemblyProj) def sharedSettings = Defaults.defaultSettings ++ Seq( organization := "org.apache.spark", @@ -320,7 +320,7 @@ object SparkBuild extends Build { def streamingSettings = sharedSettings ++ Seq( name := "spark-streaming", libraryDependencies ++= Seq( - "commons-io" % "commons-io" % "2.4" + "commons-io" % "commons-io" % "2.4" ) ) @@ -387,19 +387,19 @@ object SparkBuild extends Build { "org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty) ) ) - + def kafkaSettings() = sharedSettings ++ Seq( name := "spark-streaming-kafka", libraryDependencies ++= Seq( "com.github.sgroschupf" % "zkclient" % "0.1" excludeAll(excludeNetty), - "com.sksamuel.kafka" %% "kafka" % "0.8.0-beta1" + "org.apache.kafka" %% "kafka" % "0.8.0" exclude("com.sun.jdmk", "jmxtools") exclude("com.sun.jmx", "jmxri") exclude("net.sf.jopt-simple", "jopt-simple") excludeAll(excludeNetty) ) ) - + def flumeSettings() = sharedSettings ++ Seq( name := "spark-streaming-flume", libraryDependencies ++= Seq( -- cgit v1.2.3