diff options
author | Denny <dennybritz@gmail.com> | 2012-08-01 14:06:09 -0700 |
---|---|---|
committer | Denny <dennybritz@gmail.com> | 2012-08-01 14:06:09 -0700 |
commit | 1b29e90a7914155565d09e6c3477ad265969df00 (patch) | |
tree | be4ceefffafbe5c1c0f3f8339f4d911f666fd469 /project | |
parent | 011220fa557ed98c4bf2cfaa7f61de44c1054ea9 (diff) | |
parent | 545165e8153bb516d4364f9b3df1440c6b44c01b (diff) | |
download | spark-1b29e90a7914155565d09e6c3477ad265969df00.tar.gz spark-1b29e90a7914155565d09e6c3477ad265969df00.tar.bz2 spark-1b29e90a7914155565d09e6c3477ad265969df00.zip |
merge dev branch
Diffstat (limited to 'project')
-rw-r--r-- | project/SparkBuild.scala | 57 |
1 files changed, 55 insertions, 2 deletions
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index d1445f2ade..684108677f 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1,5 +1,7 @@ import sbt._ import Keys._ +import classpath.ClasspathUtilities.isArchive +import java.io.FileOutputStream import sbtassembly.Plugin._ import AssemblyKeys._ import twirl.sbt.TwirlPlugin._ @@ -70,12 +72,12 @@ object SparkBuild extends Build { "cc.spray" % "spray-can" % "1.0-M2.1", "cc.spray" % "spray-server" % "1.0-M2.1" ) - ) ++ assemblySettings ++ extraAssemblySettings ++ Seq(Twirl.settings: _*) + ) ++ assemblySettings ++ extraAssemblySettings ++ mergeSettings ++ Seq(Twirl.settings: _*) def replSettings = sharedSettings ++ Seq( name := "spark-repl", libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _) - ) ++ assemblySettings ++ extraAssemblySettings + ) ++ assemblySettings ++ extraAssemblySettings ++ mergeSettings def examplesSettings = sharedSettings ++ Seq( name := "spark-examples" @@ -83,6 +85,57 @@ object SparkBuild extends Build { def bagelSettings = sharedSettings ++ Seq(name := "spark-bagel") + // Fix for "No configuration setting found for key 'akka.version'" exception + // when running Spark from the jar generated by the "assembly" task; see + // http://letitcrash.com/post/21025950392/howto-sbt-assembly-vs-reference-conf + lazy val merge = TaskKey[File]("merge-reference", + "merge all reference.conf") + + lazy val mergeSettings: Seq[Project.Setting[_]] = Seq( + merge <<= (fullClasspath in assembly) map { + c => + // collect from all elements of the full classpath + val (libs, dirs) = + c map (_.data) partition (isArchive) + // goal is to simply concatenate files here + val dest = file("reference.conf") + val out = new FileOutputStream(dest) + val append = IO.transfer(_: File, out) + try { + // first collect from managed sources + (dirs * "reference.conf").get foreach append + // then from dependency jars by unzipping and + // collecting reference.conf if present + for (lib <- libs) { + IO withTemporaryDirectory { + dir => + IO.unzip(lib, dir, "reference.conf") + (dir * "reference.conf").get foreach append + } + } + // return merged file location as task result + dest + } finally { + out.close() + } + }, + + // get rid of the individual files from jars + excludedFiles in assembly <<= + (excludedFiles in assembly) { + (old) => (bases) => + old(bases) ++ (bases flatMap (base => + (base / "reference.conf").get)) + }, + + // tell sbt-assembly to include our merged file + assembledMappings in assembly <<= + (assembledMappings in assembly, merge) map { + (old, merged) => (f) => + old(f) :+(merged, "reference.conf") + } + ) + def extraAssemblySettings() = Seq(test in assembly := {}) ++ Seq( mergeStrategy in assembly := { case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard |