diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2012-07-30 22:37:00 -0700 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2012-07-30 22:37:00 -0700 |
commit | 545165e8153bb516d4364f9b3df1440c6b44c01b (patch) | |
tree | 4e5f7f51d638a7ba6414ece70a7fcadc0e826377 /project | |
parent | 3ee2530c0c40f7670151f55c05232728a12c23e2 (diff) | |
parent | 509b721d12c909d9298127637a9391bfef6e91b4 (diff) | |
download | spark-545165e8153bb516d4364f9b3df1440c6b44c01b.tar.gz spark-545165e8153bb516d4364f9b3df1440c6b44c01b.tar.bz2 spark-545165e8153bb516d4364f9b3df1440c6b44c01b.zip |
Merge pull request #158 from JoshRosen/fix/assembly-akka-config
Merge Akka reference.conf files in sbt assembly task
Diffstat (limited to 'project')
-rw-r--r-- | project/SparkBuild.scala | 57 |
1 files changed, 55 insertions, 2 deletions
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 726d490738..3603d42a23 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1,5 +1,7 @@ import sbt._ import Keys._ +import classpath.ClasspathUtilities.isArchive +import java.io.FileOutputStream import sbtassembly.Plugin._ import AssemblyKeys._ @@ -69,12 +71,12 @@ object SparkBuild extends Build { "cc.spray" % "spray-can" % "1.0-M2.1", "cc.spray" % "spray-server" % "1.0-M2.1" ) - ) ++ assemblySettings ++ extraAssemblySettings + ) ++ assemblySettings ++ extraAssemblySettings ++ mergeSettings def replSettings = sharedSettings ++ Seq( name := "spark-repl", libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _) - ) ++ assemblySettings ++ extraAssemblySettings + ) ++ assemblySettings ++ extraAssemblySettings ++ mergeSettings def examplesSettings = sharedSettings ++ Seq( name := "spark-examples" @@ -82,6 +84,57 @@ object SparkBuild extends Build { def bagelSettings = sharedSettings ++ Seq(name := "spark-bagel") + // Fix for "No configuration setting found for key 'akka.version'" exception + // when running Spark from the jar generated by the "assembly" task; see + // http://letitcrash.com/post/21025950392/howto-sbt-assembly-vs-reference-conf + lazy val merge = TaskKey[File]("merge-reference", + "merge all reference.conf") + + lazy val mergeSettings: Seq[Project.Setting[_]] = Seq( + merge <<= (fullClasspath in assembly) map { + c => + // collect from all elements of the full classpath + val (libs, dirs) = + c map (_.data) partition (isArchive) + // goal is to simply concatenate files here + val dest = file("reference.conf") + val out = new FileOutputStream(dest) + val append = IO.transfer(_: File, out) + try { + // first collect from managed sources + (dirs * "reference.conf").get foreach append + // then from dependency jars by unzipping and + // collecting reference.conf if present + for (lib <- libs) { + IO withTemporaryDirectory { + dir => + IO.unzip(lib, dir, "reference.conf") + (dir * "reference.conf").get foreach append + } + } + // return merged file location as task result + dest + } finally { + out.close() + } + }, + + // get rid of the individual files from jars + excludedFiles in assembly <<= + (excludedFiles in assembly) { + (old) => (bases) => + old(bases) ++ (bases flatMap (base => + (base / "reference.conf").get)) + }, + + // tell sbt-assembly to include our merged file + assembledMappings in assembly <<= + (assembledMappings in assembly, merge) map { + (old, merged) => (f) => + old(f) :+(merged, "reference.conf") + } + ) + def extraAssemblySettings() = Seq(test in assembly := {}) ++ Seq( mergeStrategy in assembly := { case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard |