diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-08-29 16:13:26 -0700 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-08-29 16:13:26 -0700 |
commit | 4cc5e51e28049c7f622006238aadfc3a3928cbfb (patch) | |
tree | b4b4c472da3745b829b8105ec83fae93dbb3ff62 | |
parent | a161f00610bce0a44bad5977400c849e363cd056 (diff) | |
download | spark-4cc5e51e28049c7f622006238aadfc3a3928cbfb.tar.gz spark-4cc5e51e28049c7f622006238aadfc3a3928cbfb.tar.bz2 spark-4cc5e51e28049c7f622006238aadfc3a3928cbfb.zip |
Use the new SBT 0.10 assembly plugin instead of the DepJar code we had.
-rw-r--r-- | project/DepJar.scala | 108 | ||||
-rw-r--r-- | project/SparkBuild.scala | 8 | ||||
-rw-r--r-- | project/plugins/build.sbt | 2 |
3 files changed, 6 insertions, 112 deletions
diff --git a/project/DepJar.scala b/project/DepJar.scala deleted file mode 100644 index 6ca00423be..0000000000 --- a/project/DepJar.scala +++ /dev/null @@ -1,108 +0,0 @@ -import sbt._ -import Keys._ -import java.io.PrintWriter -import scala.collection.mutable -import scala.io.Source -import Project.Initialize - -/* - * This is based on the AssemblyPlugin. For now it was easier to copy and modify than to wait for - * the required changes needed for us to customise it so that it does what we want. We may revisit - * this in the future. - */ -object DepJarPlugin extends Plugin { - val DepJar = config("dep-jar") extend(Runtime) - val depJar = TaskKey[File]("dep-jar", "Builds a single-file jar of all dependencies.") - - val jarName = SettingKey[String]("jar-name") - val outputPath = SettingKey[File]("output-path") - val excludedFiles = SettingKey[Seq[File] => Seq[File]]("excluded-files") - val conflictingFiles = SettingKey[Seq[File] => Seq[File]]("conflicting-files") - - private def assemblyTask: Initialize[Task[File]] = - (test, packageOptions, cacheDirectory, outputPath, - fullClasspath, excludedFiles, conflictingFiles, streams) map { - (test, options, cacheDir, jarPath, cp, exclude, conflicting, s) => - IO.withTemporaryDirectory { tempDir => - val srcs = assemblyPaths(tempDir, cp, exclude, conflicting, s.log) - val config = new Package.Configuration(srcs, jarPath, options) - Package(config, cacheDir, s.log) - jarPath - } - } - - private def assemblyPackageOptionsTask: Initialize[Task[Seq[PackageOption]]] = - (packageOptions in Compile, mainClass in DepJar) map { (os, mainClass) => - mainClass map { s => - os find { o => o.isInstanceOf[Package.MainClass] } map { _ => os - } getOrElse { Package.MainClass(s) +: os } - } getOrElse {os} - } - - private def assemblyExcludedFiles(base: Seq[File]): Seq[File] = { - ((base / "scala" ** "*") +++ // exclude scala library - (base / "spark" ** "*") +++ // exclude Spark classes - ((base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF - (base / "META-INF" / "services" ** "*") --- // include all service providers - (base / "META-INF" / "maven" ** "*"))).get // include all Maven POMs and such - } - - private def assemblyPaths(tempDir: File, classpath: Classpath, - exclude: Seq[File] => Seq[File], conflicting: Seq[File] => Seq[File], log: Logger) = { - import sbt.classpath.ClasspathUtilities - - val (libs, directories) = classpath.map(_.data).partition(ClasspathUtilities.isArchive) - val services = mutable.Map[String, mutable.ArrayBuffer[String]]() - for(jar <- libs) { - val jarName = jar.asFile.getName - log.info("Including %s".format(jarName)) - IO.unzip(jar, tempDir) - IO.delete(conflicting(Seq(tempDir))) - val servicesDir = tempDir / "META-INF" / "services" - if (servicesDir.asFile.exists) { - for (service <- (servicesDir ** "*").get) { - val serviceFile = service.asFile - if (serviceFile.exists && serviceFile.isFile) { - val entries = services.getOrElseUpdate(serviceFile.getName, new mutable.ArrayBuffer[String]()) - for (provider <- Source.fromFile(serviceFile).getLines) { - if (!entries.contains(provider)) { - entries += provider - } - } - } - } - } - } - - for ((service, providers) <- services) { - log.debug("Merging providers for %s".format(service)) - val serviceFile = (tempDir / "META-INF" / "services" / service).asFile - val writer = new PrintWriter(serviceFile) - for (provider <- providers.map { _.trim }.filter { !_.isEmpty }) { - log.debug("- %s".format(provider)) - writer.println(provider) - } - writer.close() - } - - val base = tempDir +: directories - val descendants = ((base ** (-DirectoryFilter)) --- exclude(base)).get - descendants x relativeTo(base) - } - - lazy val depJarSettings: Seq[Setting[_]] = inConfig(DepJar)(Seq( - depJar <<= packageBin.identity, - packageBin <<= assemblyTask, - jarName <<= (name, version) { (name, version) => name + "-dep-" + version + ".jar" }, - outputPath <<= (target, jarName) { (t, s) => t / s }, - test <<= (test in Test).identity, - mainClass <<= (mainClass in Runtime).identity, - fullClasspath <<= (fullClasspath in Runtime).identity, - packageOptions <<= assemblyPackageOptionsTask, - excludedFiles := assemblyExcludedFiles _, - conflictingFiles := assemblyExcludedFiles _ - )) ++ - Seq( - depJar <<= (depJar in DepJar).identity - ) -} diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 8392c63d74..4f68ca5fd9 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1,5 +1,6 @@ import sbt._ import Keys._ +import sbtassembly.Plugin.{assemblySettings, Assembly} object SparkBuild extends Build { @@ -42,13 +43,12 @@ object SparkBuild extends Build { "asm" % "asm-all" % "3.3.1", "com.google.protobuf" % "protobuf-java" % "2.3.0", "de.javakaffee" % "kryo-serializers" % "0.9" - )) ++ DepJarPlugin.depJarSettings + )) ++ assemblySettings ++ Seq(test in Assembly := {}) def replSettings = sharedSettings ++ - Seq(libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _)) ++ - DepJarPlugin.depJarSettings + Seq(libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _)) def examplesSettings = sharedSettings ++ Seq(libraryDependencies += "colt" % "colt" % "1.2.0") - def bagelSettings = sharedSettings ++ DepJarPlugin.depJarSettings + def bagelSettings = sharedSettings } diff --git a/project/plugins/build.sbt b/project/plugins/build.sbt index 91c6cb4df1..1e4d170f0e 100644 --- a/project/plugins/build.sbt +++ b/project/plugins/build.sbt @@ -11,3 +11,5 @@ libraryDependencies += "com.github.mpeltonen" %% "sbt-idea" % "0.10.0" libraryDependencies <<= (libraryDependencies, sbtVersion) { (deps, version) => deps :+ ("com.typesafe.sbteclipse" %% "sbteclipse" % "1.2" extra("sbtversion" -> version)) } + +libraryDependencies <+= (sbtVersion) { sv => "com.eed3si9n" %% "sbt-assembly" % ("sbt" + sv + "_0.6") } |