From 484166d520e239d0da9501c3863435a59e235242 Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Wed, 9 Oct 2013 04:24:34 -0700 Subject: Add new SBT target for dependency assembly --- project/SparkBuild.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'project/SparkBuild.scala') diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index d038a4f479..6d6a8ef992 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -56,6 +56,8 @@ object SparkBuild extends Build { lazy val assemblyProj = Project("assembly", file("assembly"), settings = assemblyProjSettings) .dependsOn(core, bagel, mllib, repl, streaming) dependsOn(maybeYarn: _*) + lazy val spark = TaskKey[Unit]("spark", "Build assembly of dependencies and spark packages") + // A configuration to set an alternative publishLocalConfiguration lazy val MavenCompile = config("m2r") extend(Compile) lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy") @@ -73,6 +75,8 @@ object SparkBuild extends Build { lazy val allProjects = Seq[ProjectReference]( core, repl, examples, bagel, streaming, mllib, tools, assemblyProj) ++ maybeYarnRef + lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib, tools) ++ maybeYarnRef + def sharedSettings = Defaults.defaultSettings ++ Seq( organization := "org.apache.spark", version := "0.8.0-SNAPSHOT", @@ -288,7 +292,9 @@ object SparkBuild extends Build { def assemblyProjSettings = sharedSettings ++ Seq( name := "spark-assembly", - jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" } + spark in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, + jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, + jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings def extraAssemblySettings() = Seq( -- cgit v1.2.3 From c441904bcec27d8cca49d174b18ee67da70dfdff Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Fri, 11 Oct 2013 18:23:15 -0700 Subject: Add a comment and exclude tools --- project/SparkBuild.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'project/SparkBuild.scala') diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 6d6a8ef992..bc2cec0553 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -75,7 +75,8 @@ object SparkBuild extends Build { lazy val allProjects = Seq[ProjectReference]( core, repl, examples, bagel, streaming, mllib, tools, assemblyProj) ++ maybeYarnRef - lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib, tools) ++ maybeYarnRef + // Everything except assembly and examples belongs to packageProjects + lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib) ++ maybeYarnRef def sharedSettings = Defaults.defaultSettings ++ Seq( organization := "org.apache.spark", -- cgit v1.2.3 From 0a4b76fcc22568dd62e6be0301e3bda916bc6bc8 Mon Sep 17 00:00:00 2001 From: Shivaram Venkataraman Date: Wed, 16 Oct 2013 17:05:46 -0700 Subject: Rename SBT target to assemble-deps. --- project/SparkBuild.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'project/SparkBuild.scala') diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index d7e82079cf..dd26d7e60b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -60,7 +60,7 @@ object SparkBuild extends Build { lazy val assemblyProj = Project("assembly", file("assembly"), settings = assemblyProjSettings) .dependsOn(core, bagel, mllib, repl, streaming) dependsOn(maybeYarn: _*) - lazy val spark = TaskKey[Unit]("spark", "Build assembly of dependencies and spark packages") + lazy val assembleDeps = TaskKey[Unit]("assemble-deps", "Build assembly of dependencies and packages Spark projects") // A configuration to set an alternative publishLocalConfiguration lazy val MavenCompile = config("m2r") extend(Compile) @@ -76,12 +76,12 @@ object SparkBuild extends Build { // Conditionally include the yarn sub-project lazy val maybeYarn = if(isYarnEnabled) Seq[ClasspathDependency](yarn) else Seq[ClasspathDependency]() lazy val maybeYarnRef = if(isYarnEnabled) Seq[ProjectReference](yarn) else Seq[ProjectReference]() - lazy val allProjects = Seq[ProjectReference]( - core, repl, examples, bagel, streaming, mllib, tools, assemblyProj) ++ maybeYarnRef - // Everything except assembly and examples belongs to packageProjects + // Everything except assembly, tools and examples belong to packageProjects lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib) ++ maybeYarnRef + lazy val allProjects = packageProjects ++ Seq[ProjectReference](examples, tools, assemblyProj) + def sharedSettings = Defaults.defaultSettings ++ Seq( organization := "org.apache.spark", version := "0.9.0-incubating-SNAPSHOT", @@ -308,7 +308,7 @@ object SparkBuild extends Build { def assemblyProjSettings = sharedSettings ++ Seq( name := "spark-assembly", - spark in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, + assembleDeps in Compile <<= (packageProjects.map(packageBin in Compile in _) ++ Seq(packageDependency in Compile)).dependOn, jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }, jarName in packageDependency <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" } ) ++ assemblySettings ++ extraAssemblySettings -- cgit v1.2.3