diff options
| author | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-10-18 20:32:39 -0700 |
|---|---|---|
| committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2013-10-18 20:32:39 -0700 |
| commit | 8de9706b86f41a37464f55e1ffe5a246adc712d1 (patch) | |
| tree | e805c8b08124282e5480e99394d5636d5b091d91 /project/SparkBuild.scala | |
| parent | e5316d0685c41a40e54a064cf271f3d62df6c8e8 (diff) | |
| parent | 0a4b76fcc22568dd62e6be0301e3bda916bc6bc8 (diff) | |
| download | spark-8de9706b86f41a37464f55e1ffe5a246adc712d1.tar.gz spark-8de9706b86f41a37464f55e1ffe5a246adc712d1.tar.bz2 spark-8de9706b86f41a37464f55e1ffe5a246adc712d1.zip | |
Merge pull request #66 from shivaram/sbt-assembly-deps
Add SBT target to assemble dependencies
This pull request is an attempt to address the long assembly build times during development. Instead of rebuilding the assembly jar for every Spark change, this pull request adds a new SBT target `spark` that packages all the Spark modules and builds an assembly of the dependencies.
So the work flow that should work now would be something like
```
./sbt/sbt spark # Doing this once should suffice
## Make changes
./sbt/sbt compile
./sbt/sbt test or ./spark-shell
```
Diffstat (limited to 'project/SparkBuild.scala')
| -rw-r--r-- | project/SparkBuild.scala | 13 |
1 files changed, 10 insertions, 3 deletions
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f2bbe5358f..965c4f3a63 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -60,6 +60,8 @@ object SparkBuild extends Build { lazy val assemblyProj = Project("assembly", file("assembly"), settings = assemblyProjSettings) .dependsOn(core, bagel, mllib, repl, streaming) dependsOn(maybeYarn: _*) + lazy val assembleDeps = TaskKey[Unit]("assemble-deps", "Build assembly of dependencies and packages Spark projects") + // A configuration to set an alternative publishLocalConfiguration lazy val MavenCompile = config("m2r") extend(Compile) lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy") @@ -74,8 +76,11 @@ object SparkBuild extends Build { // Conditionally include the yarn sub-project lazy val maybeYarn = if(isYarnEnabled) Seq[ClasspathDependency](yarn) else Seq[ClasspathDependency]() lazy val maybeYarnRef = if(isYarnEnabled) Seq[ProjectReference](yarn) else Seq[ProjectReference]() - lazy val allProjects = Seq[ProjectReference]( - core, repl, examples, bagel, streaming, mllib, tools, assemblyProj) ++ maybeYarnRef + + // Everything except assembly, tools and examples belong to packageProjects + lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib) ++ maybeYarnRef + + lazy val allProjects = packageProjects ++ Seq[ProjectReference](examples, tools, assemblyProj) def sharedSettings = Defaults.defaultSettings ++ Seq( organization := "org.apache.spark", @@ -303,7 +308,9 @@ object SparkBuild extends Build { def assemblyProjSettings = sharedSettings ++ Seq( name := "spark-assembly", - jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" } + assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, + jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, + jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings def extraAssemblySettings() = Seq( |