Merge branch 'scala-2.9'

Conflicts:
	project/build/SparkProject.scala

7 files changed, 182 insertions, 126 deletions

diff --git a/project/DepJar.scala b/project/DepJar.scala
new file mode 100644
index 0000000000..1d54005690
--- /dev/null
+++ b/project/DepJar.scala
@@ -0,0 +1,108 @@
+import sbt._
+import Keys._
+import java.io.PrintWriter
+import scala.collection.mutable
+import scala.io.Source
+import Project.Initialize
+
+/*
+ * This is based on the AssemblyPlugin. For now it was easier to copy and modify than to wait for
+ * the required changes needed for us to customise it so that it does what we want. We may revisit
+ * this in the future.
+ */
+object DepJarPlugin extends Plugin {
+  val DepJar = config("dep-jar") extend(Runtime)
+  val depJar = TaskKey[File]("dep-jar", "Builds a single-file jar of all dependencies.")
+
+  val jarName           = SettingKey[String]("jar-name")
+  val outputPath        = SettingKey[File]("output-path")
+  val excludedFiles     = SettingKey[Seq[File] => Seq[File]]("excluded-files")
+  val conflictingFiles  = SettingKey[Seq[File] => Seq[File]]("conflicting-files")
+
+  private def assemblyTask: Initialize[Task[File]] =
+    (test, packageOptions, cacheDirectory, outputPath,
+        fullClasspath, excludedFiles, conflictingFiles, streams) map {
+      (test, options, cacheDir, jarPath, cp, exclude, conflicting, s) =>
+        IO.withTemporaryDirectory { tempDir =>
+          val srcs = assemblyPaths(tempDir, cp, exclude, conflicting, s.log)
+          val config = new Package.Configuration(srcs, jarPath, options)
+          Package(config, cacheDir, s.log)
+          jarPath
+        }
+    }
+
+  private def assemblyPackageOptionsTask: Initialize[Task[Seq[PackageOption]]] =
+    (packageOptions in Compile, mainClass in DepJar) map { (os, mainClass) =>
+      mainClass map { s =>
+        os find { o => o.isInstanceOf[Package.MainClass] } map { _ => os
+        } getOrElse { Package.MainClass(s) +: os }
+      } getOrElse {os}
+    }
+
+  private def assemblyExcludedFiles(base: Seq[File]): Seq[File] = {
+    ((base / "scala" ** "*") +++ // exclude scala library
+      (base / "spark" ** "*") +++ // exclude Spark classes
+      ((base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF
+        (base / "META-INF" / "services" ** "*") --- // include all service providers
+        (base / "META-INF" / "maven" ** "*"))).get // include all Maven POMs and such
+  }
+
+  private def assemblyPaths(tempDir: File, classpath: Classpath,
+      exclude: Seq[File] => Seq[File], conflicting: Seq[File] => Seq[File], log: Logger) = {
+    import sbt.classpath.ClasspathUtilities
+
+    val (libs, directories) = classpath.map(_.data).partition(ClasspathUtilities.isArchive)
+    val services = mutable.Map[String, mutable.ArrayBuffer[String]]()
+    for(jar <- libs) {
+      val jarName = jar.asFile.getName
+      log.info("Including %s".format(jarName))
+      IO.unzip(jar, tempDir)
+      IO.delete(conflicting(Seq(tempDir)))
+      val servicesDir = tempDir / "META-INF" / "services"
+      if (servicesDir.asFile.exists) {
+       for (service <- (servicesDir ** "*").get) {
+         val serviceFile = service.asFile
+         if (serviceFile.exists && serviceFile.isFile) {
+           val entries = services.getOrElseUpdate(serviceFile.getName, new mutable.ArrayBuffer[String]())
+           for (provider <- Source.fromFile(serviceFile).getLines) {
+             if (!entries.contains(provider)) {
+               entries += provider
+             }
+           }
+         }
+       }
+     }
+    }
+
+    for ((service, providers) <- services) {
+      log.debug("Merging providers for %s".format(service))
+      val serviceFile = (tempDir / "META-INF" / "services" / service).asFile
+      val writer = new PrintWriter(serviceFile)
+      for (provider <- providers.map { _.trim }.filter { !_.isEmpty }) {
+        log.debug("-  %s".format(provider))
+        writer.println(provider)
+      }
+      writer.close()
+    }
+
+    val base = tempDir +: directories
+    val descendants = ((base ** (-DirectoryFilter)) --- exclude(base)).get
+    descendants x relativeTo(base)
+  }
+
+  lazy val depJarSettings = inConfig(DepJar)(Seq(
+    depJar <<= packageBin.identity,
+    packageBin <<= assemblyTask,
+    jarName <<= (name, version) { (name, version) => name + "-dep-" + version + ".jar" },
+    outputPath <<= (target, jarName) { (t, s) => t / s },
+    test <<= (test in Test).identity,
+    mainClass <<= (mainClass in Runtime).identity,
+    fullClasspath <<= (fullClasspath in Runtime).identity,
+    packageOptions <<= assemblyPackageOptionsTask,
+    excludedFiles := assemblyExcludedFiles _,
+    conflictingFiles := assemblyExcludedFiles _
+  )) ++
+  Seq(
+    depJar <<= (depJar in DepJar).identity
+  )
+}
\ No newline at end of file
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
new file mode 100644
index 0000000000..23d13a8179
--- /dev/null
+++ b/project/SparkBuild.scala
@@ -0,0 +1,53 @@
+import sbt._
+import Keys._
+
+object SparkBuild extends Build {
+
+  lazy val root = Project("root", file("."), settings = sharedSettings) aggregate(core, repl, examples, bagel)
+
+  lazy val core = Project("core", file("core"), settings = coreSettings)
+
+  lazy val repl = Project("repl", file("repl"), settings = replSettings) dependsOn (core)
+
+  lazy val examples = Project("examples", file("examples"), settings = examplesSettings) dependsOn (core)
+
+  lazy val bagel = Project("bagel", file("bagel"), settings = bagelSettings) dependsOn (core)
+
+  def sharedSettings = Defaults.defaultSettings ++ Seq(
+    organization := "org.spark-project",
+    version := "0.4-SNAPSHOT",
+    scalaVersion := "2.9.0-1",
+    scalacOptions := Seq(/*"-deprecation",*/ "-unchecked"), // TODO Enable -deprecation and fix all warnings
+    unmanagedJars in Compile <<= baseDirectory map { base => (base ** "*.jar").classpath },
+    retrieveManaged := true,
+    transitiveClassifiers in Scope.GlobalScope := Seq("sources"),
+    testListeners <<= target.map(t => Seq(new eu.henkelmann.sbt.JUnitXmlTestsListener(t.getAbsolutePath))),
+    libraryDependencies ++= Seq(
+      "org.eclipse.jetty" % "jetty-server" % "7.4.2.v20110526",
+      "org.scalatest" % "scalatest_2.9.0" % "1.6.1" % "test",
+      "org.scala-tools.testing" % "scalacheck_2.9.0-1" % "1.9" % "test"
+    )
+  )
+
+  val slf4jVersion = "1.6.1"
+
+  def coreSettings = sharedSettings ++ Seq(libraryDependencies ++= Seq(
+    "com.google.guava" % "guava" % "r09",
+    "log4j" % "log4j" % "1.2.16",
+    "org.slf4j" % "slf4j-api" % slf4jVersion,
+    "org.slf4j" % "slf4j-log4j12" % slf4jVersion,
+    "com.ning" % "compress-lzf" % "0.7.0",
+    "org.apache.hadoop" % "hadoop-core" % "0.20.2",
+    "asm" % "asm-all" % "3.3.1",
+    "com.google.protobuf" % "protobuf-java" % "2.3.0",
+    "de.javakaffee" % "kryo-serializers" % "0.9"
+  )) ++ DepJarPlugin.depJarSettings
+
+  def replSettings = sharedSettings ++
+      Seq(libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _)) ++
+      DepJarPlugin.depJarSettings
+
+  def examplesSettings = sharedSettings ++ Seq(libraryDependencies += "colt" % "colt" % "1.2.0")
+
+  def bagelSettings = sharedSettings ++ DepJarPlugin.depJarSettings
+}
diff --git a/project/build.properties b/project/build.properties
index 1b5f8c4ea9..f47a3009ec 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -1,8 +1 @@
-#Project properties
-#Sat Nov 13 21:57:32 PST 2010
-project.organization=org.spark-project
-project.name=spark
-sbt.version=0.7.7
-project.version=0.4-SNAPSHOT
-build.scala.versions=2.8.1
-project.initialize=false
+sbt.version=0.10.1
diff --git a/project/build/SparkProject.scala b/project/build/SparkProject.scala
deleted file mode 100644
index 2354f47a76..0000000000
--- a/project/build/SparkProject.scala
+++ /dev/null
@@ -1,107 +0,0 @@
-import sbt._
-import sbt.Process._
-
-import assembly._
-
-import de.element34.sbteclipsify._
-
-
-class SparkProject(info: ProjectInfo) extends ParentProject(info) with IdeaProject {
-
-  lazy val core = project("core", "Spark Core", new CoreProject(_))
-
-  lazy val repl = project("repl", "Spark REPL", new ReplProject(_), core)
-
-  lazy val examples = project("examples", "Spark Examples", new ExamplesProject(_), core)
-
-  lazy val bagel = project("bagel", "Bagel", new BagelProject(_), core)
-
-  trait BaseProject extends BasicScalaProject with ScalaPaths with BasicPackagePaths with Eclipsify with IdeaProject {
-    override def compileOptions = super.compileOptions ++ Seq(Unchecked)
-
-    lazy val jettyServer = "org.eclipse.jetty" % "jetty-server" % "7.4.2.v20110526"
-    
-    override def packageDocsJar = defaultJarPath("-javadoc.jar")
-    override def packageSrcJar= defaultJarPath("-sources.jar")
-    lazy val sourceArtifact = Artifact.sources(artifactID)
-    lazy val docsArtifact = Artifact.javadoc(artifactID)
-    override def packageToPublishActions = super.packageToPublishActions ++ Seq(packageDocs, packageSrc)
-  }
-
-  class CoreProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport {
-    val guava = "com.google.guava" % "guava" % "r09"
-    val log4j = "log4j" % "log4j" % "1.2.16"
-    val slf4jVersion = "1.6.1"
-    val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion
-    val slf4jLog4j = "org.slf4j" % "slf4j-log4j12" % slf4jVersion
-    val compressLzf = "com.ning" % "compress-lzf" % "0.7.0"
-    val hadoop = "org.apache.hadoop" % "hadoop-core" % "0.20.2"
-    val asm = "asm" % "asm-all" % "3.3.1"
-    val scalaTest = "org.scalatest" % "scalatest" % "1.3" % "test"
-    val scalaCheck = "org.scala-tools.testing" %% "scalacheck" % "1.7" % "test"
-    val protobuf = "com.google.protobuf" % "protobuf-java" % "2.3.0"
-    val kryoSerializers = "de.javakaffee" % "kryo-serializers" % "0.9"
-  }
-
-  class ReplProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport
-
-  class ExamplesProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject {
-    val colt = "colt" % "colt" % "1.2.0"
-  }
-
-  class BagelProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport
-
-  override def managedStyle = ManagedStyle.Maven
-}
-
-
-// Project mixin for an XML-based ScalaTest report. Unfortunately
-// there is currently no way to call this directly from SBT without
-// executing a subprocess.
-trait XmlTestReport extends BasicScalaProject {
-  def testReportDir = outputPath / "test-report"
-
-  lazy val testReport = task {
-    log.info("Creating " + testReportDir + "...")
-    if (!testReportDir.exists) {
-      testReportDir.asFile.mkdirs()
-    }
-    log.info("Executing org.scalatest.tools.Runner...")
-    val command = ("scala -classpath " + testClasspath.absString +
-                   " org.scalatest.tools.Runner -o " +
-                   " -u " + testReportDir.absolutePath +
-                   " -p " + (outputPath / "test-classes").absolutePath)
-    Process(command, path("."), "JAVA_OPTS" -> "-Xmx500m") !
-
-    None
-  }.dependsOn(compile, testCompile).describedAs("Generate XML test report.")
-}
-
-
-// Project mixin for creating a JAR with  a project's dependencies. This is based
-// on the AssemblyBuilder plugin, but because this plugin attempts to package Scala
-// and our project too, we leave that out using our own exclude filter (depJarExclude).
-trait DepJar extends AssemblyBuilder {
-  def depJarExclude(base: PathFinder) = {
-    (base / "scala" ** "*") +++ // exclude scala library
-    (base / "spark" ** "*") +++ // exclude Spark classes
-    ((base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF
-     (base / "META-INF" / "services" ** "*") --- // include all service providers
-     (base / "META-INF" / "maven" ** "*")) // include all Maven POMs and such
-  }
-
-  def depJarTempDir = outputPath / "dep-classes"
-
-  def depJarOutputPath =
-    outputPath / (name.toLowerCase.replace(" ", "-") + "-dep-" + version.toString + ".jar")
-
-  lazy val depJar = {
-    packageTask(
-      Path.lazyPathFinder(assemblyPaths(depJarTempDir,
-                                        assemblyClasspath,
-                                        assemblyExtraJars,
-                                        depJarExclude)),
-      depJarOutputPath,
-      packageOptions)
-  }.dependsOn(compile).describedAs("Bundle project's dependencies into a JAR.")
-}
diff --git a/project/plugins/SparkProjectPlugins.scala b/project/plugins/SparkProjectPlugins.scala
deleted file mode 100644
index 565f160829..0000000000
--- a/project/plugins/SparkProjectPlugins.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-import sbt._
-
-class SparkProjectPlugins(info: ProjectInfo) extends PluginDefinition(info) {
-  val eclipse = "de.element34" % "sbt-eclipsify" % "0.7.0"
-
-  val sbtIdeaRepo = "sbt-idea-repo" at "http://mpeltonen.github.com/maven/"
-  val sbtIdea = "com.github.mpeltonen" % "sbt-idea-plugin" % "0.4.0"
-
-  val codaRepo = "Coda Hale's Repository" at "http://repo.codahale.com/"
-  val assemblySBT = "com.codahale" % "assembly-sbt" % "0.1.1"
-}
diff --git a/project/plugins/build.sbt b/project/plugins/build.sbt
new file mode 100644
index 0000000000..91c6cb4df1
--- /dev/null
+++ b/project/plugins/build.sbt
@@ -0,0 +1,13 @@
+resolvers += {
+  val typesafeRepoUrl = new java.net.URL("http://repo.typesafe.com/typesafe/releases")
+  val pattern = Patterns(false, "[organisation]/[module]/[sbtversion]/[revision]/[type]s/[module](-[classifier])-[revision].[ext]")
+  Resolver.url("Typesafe Repository", typesafeRepoUrl)(pattern)
+}
+
+resolvers += "sbt-idea-repo" at "http://mpeltonen.github.com/maven/"
+
+libraryDependencies += "com.github.mpeltonen" %% "sbt-idea" % "0.10.0"
+
+libraryDependencies <<= (libraryDependencies, sbtVersion) { (deps, version) =>
+  deps :+ ("com.typesafe.sbteclipse" %% "sbteclipse" % "1.2" extra("sbtversion" -> version))
+}
diff --git a/project/plugins/project/SparkPluginBuild.scala b/project/plugins/project/SparkPluginBuild.scala
new file mode 100644
index 0000000000..999611982a
--- /dev/null
+++ b/project/plugins/project/SparkPluginBuild.scala
@@ -0,0 +1,7 @@
+import sbt._
+
+object SparkPluginDef extends Build {
+  lazy val root = Project("plugins", file(".")) dependsOn(junitXmlListener)
+  /* This is not published in a Maven repository, so we get it from GitHub directly */
+  lazy val junitXmlListener = uri("git://github.com/ijuma/junit_xml_listener.git#fe434773255b451a38e8d889536ebc260f4225ce")
+}
\ No newline at end of file