merge dev branch

1 files changed, 55 insertions, 2 deletions

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index d1445f2ade..684108677f 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1,5 +1,7 @@
 import sbt._
 import Keys._
+import classpath.ClasspathUtilities.isArchive
+import java.io.FileOutputStream
 import sbtassembly.Plugin._
 import AssemblyKeys._
 import twirl.sbt.TwirlPlugin._
@@ -70,12 +72,12 @@ object SparkBuild extends Build {
       "cc.spray" % "spray-can" % "1.0-M2.1",
       "cc.spray" % "spray-server" % "1.0-M2.1"
     )
-  ) ++ assemblySettings ++ extraAssemblySettings ++ Seq(Twirl.settings: _*)
+  ) ++ assemblySettings ++ extraAssemblySettings ++ mergeSettings ++ Seq(Twirl.settings: _*)
 
   def replSettings = sharedSettings ++ Seq(
     name := "spark-repl",
     libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _)
-  ) ++ assemblySettings ++ extraAssemblySettings 
+  ) ++ assemblySettings ++ extraAssemblySettings ++ mergeSettings
 
   def examplesSettings = sharedSettings ++ Seq(
     name := "spark-examples"
@@ -83,6 +85,57 @@ object SparkBuild extends Build {
 
   def bagelSettings = sharedSettings ++ Seq(name := "spark-bagel")
 
+  // Fix for "No configuration setting found for key 'akka.version'" exception
+  // when running Spark from the jar generated by the "assembly" task; see
+  // http://letitcrash.com/post/21025950392/howto-sbt-assembly-vs-reference-conf
+  lazy val merge = TaskKey[File]("merge-reference",
+    "merge all reference.conf")
+
+  lazy val mergeSettings: Seq[Project.Setting[_]] = Seq(
+    merge <<= (fullClasspath in assembly) map {
+      c =>
+      // collect from all elements of the full classpath
+        val (libs, dirs) =
+          c map (_.data) partition (isArchive)
+        // goal is to simply concatenate files here
+        val dest = file("reference.conf")
+        val out = new FileOutputStream(dest)
+        val append = IO.transfer(_: File, out)
+        try {
+          // first collect from managed sources
+          (dirs * "reference.conf").get foreach append
+          // then from dependency jars by unzipping and
+          // collecting reference.conf if present
+          for (lib <- libs) {
+            IO withTemporaryDirectory {
+              dir =>
+                IO.unzip(lib, dir, "reference.conf")
+                (dir * "reference.conf").get foreach append
+            }
+          }
+          // return merged file location as task result
+          dest
+        } finally {
+          out.close()
+        }
+    },
+
+    // get rid of the individual files from jars
+    excludedFiles in assembly <<=
+      (excludedFiles in assembly) {
+        (old) => (bases) =>
+          old(bases) ++ (bases flatMap (base =>
+            (base / "reference.conf").get))
+      },
+
+    // tell sbt-assembly to include our merged file
+    assembledMappings in assembly <<=
+      (assembledMappings in assembly, merge) map {
+        (old, merged) => (f) =>
+          old(f) :+(merged, "reference.conf")
+      }
+  )
+
   def extraAssemblySettings() = Seq(test in assembly := {}) ++ Seq(
     mergeStrategy in assembly := { 
       case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard