diff options
author | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-08-01 15:25:26 -0700 |
---|---|---|
committer | Matei Zaharia <matei@eecs.berkeley.edu> | 2011-08-01 15:25:26 -0700 |
commit | 711575391df632627cd9eb91d84d5b6f0d4db6f2 (patch) | |
tree | 37655fceac82dbb33c3a987fbd9bf8cd7c9a7627 /project | |
parent | 4050d661c5f0a48a5a043ec932d98707f4606dd5 (diff) | |
parent | ecb8b69fa088764c109c74153f05ba8d2fc698d1 (diff) | |
download | spark-711575391df632627cd9eb91d84d5b6f0d4db6f2.tar.gz spark-711575391df632627cd9eb91d84d5b6f0d4db6f2.tar.bz2 spark-711575391df632627cd9eb91d84d5b6f0d4db6f2.zip |
Merge branch 'scala-2.9'
Conflicts:
project/build/SparkProject.scala
Diffstat (limited to 'project')
-rw-r--r-- | project/DepJar.scala | 108 | ||||
-rw-r--r-- | project/SparkBuild.scala | 53 | ||||
-rw-r--r-- | project/build.properties | 9 | ||||
-rw-r--r-- | project/build/SparkProject.scala | 107 | ||||
-rw-r--r-- | project/plugins/SparkProjectPlugins.scala | 11 | ||||
-rw-r--r-- | project/plugins/build.sbt | 13 | ||||
-rw-r--r-- | project/plugins/project/SparkPluginBuild.scala | 7 |
7 files changed, 182 insertions, 126 deletions
diff --git a/project/DepJar.scala b/project/DepJar.scala new file mode 100644 index 0000000000..1d54005690 --- /dev/null +++ b/project/DepJar.scala @@ -0,0 +1,108 @@ +import sbt._ +import Keys._ +import java.io.PrintWriter +import scala.collection.mutable +import scala.io.Source +import Project.Initialize + +/* + * This is based on the AssemblyPlugin. For now it was easier to copy and modify than to wait for + * the required changes needed for us to customise it so that it does what we want. We may revisit + * this in the future. + */ +object DepJarPlugin extends Plugin { + val DepJar = config("dep-jar") extend(Runtime) + val depJar = TaskKey[File]("dep-jar", "Builds a single-file jar of all dependencies.") + + val jarName = SettingKey[String]("jar-name") + val outputPath = SettingKey[File]("output-path") + val excludedFiles = SettingKey[Seq[File] => Seq[File]]("excluded-files") + val conflictingFiles = SettingKey[Seq[File] => Seq[File]]("conflicting-files") + + private def assemblyTask: Initialize[Task[File]] = + (test, packageOptions, cacheDirectory, outputPath, + fullClasspath, excludedFiles, conflictingFiles, streams) map { + (test, options, cacheDir, jarPath, cp, exclude, conflicting, s) => + IO.withTemporaryDirectory { tempDir => + val srcs = assemblyPaths(tempDir, cp, exclude, conflicting, s.log) + val config = new Package.Configuration(srcs, jarPath, options) + Package(config, cacheDir, s.log) + jarPath + } + } + + private def assemblyPackageOptionsTask: Initialize[Task[Seq[PackageOption]]] = + (packageOptions in Compile, mainClass in DepJar) map { (os, mainClass) => + mainClass map { s => + os find { o => o.isInstanceOf[Package.MainClass] } map { _ => os + } getOrElse { Package.MainClass(s) +: os } + } getOrElse {os} + } + + private def assemblyExcludedFiles(base: Seq[File]): Seq[File] = { + ((base / "scala" ** "*") +++ // exclude scala library + (base / "spark" ** "*") +++ // exclude Spark classes + ((base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF + (base / "META-INF" / "services" ** "*") --- // include all service providers + (base / "META-INF" / "maven" ** "*"))).get // include all Maven POMs and such + } + + private def assemblyPaths(tempDir: File, classpath: Classpath, + exclude: Seq[File] => Seq[File], conflicting: Seq[File] => Seq[File], log: Logger) = { + import sbt.classpath.ClasspathUtilities + + val (libs, directories) = classpath.map(_.data).partition(ClasspathUtilities.isArchive) + val services = mutable.Map[String, mutable.ArrayBuffer[String]]() + for(jar <- libs) { + val jarName = jar.asFile.getName + log.info("Including %s".format(jarName)) + IO.unzip(jar, tempDir) + IO.delete(conflicting(Seq(tempDir))) + val servicesDir = tempDir / "META-INF" / "services" + if (servicesDir.asFile.exists) { + for (service <- (servicesDir ** "*").get) { + val serviceFile = service.asFile + if (serviceFile.exists && serviceFile.isFile) { + val entries = services.getOrElseUpdate(serviceFile.getName, new mutable.ArrayBuffer[String]()) + for (provider <- Source.fromFile(serviceFile).getLines) { + if (!entries.contains(provider)) { + entries += provider + } + } + } + } + } + } + + for ((service, providers) <- services) { + log.debug("Merging providers for %s".format(service)) + val serviceFile = (tempDir / "META-INF" / "services" / service).asFile + val writer = new PrintWriter(serviceFile) + for (provider <- providers.map { _.trim }.filter { !_.isEmpty }) { + log.debug("- %s".format(provider)) + writer.println(provider) + } + writer.close() + } + + val base = tempDir +: directories + val descendants = ((base ** (-DirectoryFilter)) --- exclude(base)).get + descendants x relativeTo(base) + } + + lazy val depJarSettings = inConfig(DepJar)(Seq( + depJar <<= packageBin.identity, + packageBin <<= assemblyTask, + jarName <<= (name, version) { (name, version) => name + "-dep-" + version + ".jar" }, + outputPath <<= (target, jarName) { (t, s) => t / s }, + test <<= (test in Test).identity, + mainClass <<= (mainClass in Runtime).identity, + fullClasspath <<= (fullClasspath in Runtime).identity, + packageOptions <<= assemblyPackageOptionsTask, + excludedFiles := assemblyExcludedFiles _, + conflictingFiles := assemblyExcludedFiles _ + )) ++ + Seq( + depJar <<= (depJar in DepJar).identity + ) +}
\ No newline at end of file diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala new file mode 100644 index 0000000000..23d13a8179 --- /dev/null +++ b/project/SparkBuild.scala @@ -0,0 +1,53 @@ +import sbt._ +import Keys._ + +object SparkBuild extends Build { + + lazy val root = Project("root", file("."), settings = sharedSettings) aggregate(core, repl, examples, bagel) + + lazy val core = Project("core", file("core"), settings = coreSettings) + + lazy val repl = Project("repl", file("repl"), settings = replSettings) dependsOn (core) + + lazy val examples = Project("examples", file("examples"), settings = examplesSettings) dependsOn (core) + + lazy val bagel = Project("bagel", file("bagel"), settings = bagelSettings) dependsOn (core) + + def sharedSettings = Defaults.defaultSettings ++ Seq( + organization := "org.spark-project", + version := "0.4-SNAPSHOT", + scalaVersion := "2.9.0-1", + scalacOptions := Seq(/*"-deprecation",*/ "-unchecked"), // TODO Enable -deprecation and fix all warnings + unmanagedJars in Compile <<= baseDirectory map { base => (base ** "*.jar").classpath }, + retrieveManaged := true, + transitiveClassifiers in Scope.GlobalScope := Seq("sources"), + testListeners <<= target.map(t => Seq(new eu.henkelmann.sbt.JUnitXmlTestsListener(t.getAbsolutePath))), + libraryDependencies ++= Seq( + "org.eclipse.jetty" % "jetty-server" % "7.4.2.v20110526", + "org.scalatest" % "scalatest_2.9.0" % "1.6.1" % "test", + "org.scala-tools.testing" % "scalacheck_2.9.0-1" % "1.9" % "test" + ) + ) + + val slf4jVersion = "1.6.1" + + def coreSettings = sharedSettings ++ Seq(libraryDependencies ++= Seq( + "com.google.guava" % "guava" % "r09", + "log4j" % "log4j" % "1.2.16", + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion, + "com.ning" % "compress-lzf" % "0.7.0", + "org.apache.hadoop" % "hadoop-core" % "0.20.2", + "asm" % "asm-all" % "3.3.1", + "com.google.protobuf" % "protobuf-java" % "2.3.0", + "de.javakaffee" % "kryo-serializers" % "0.9" + )) ++ DepJarPlugin.depJarSettings + + def replSettings = sharedSettings ++ + Seq(libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _)) ++ + DepJarPlugin.depJarSettings + + def examplesSettings = sharedSettings ++ Seq(libraryDependencies += "colt" % "colt" % "1.2.0") + + def bagelSettings = sharedSettings ++ DepJarPlugin.depJarSettings +} diff --git a/project/build.properties b/project/build.properties index 1b5f8c4ea9..f47a3009ec 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,8 +1 @@ -#Project properties -#Sat Nov 13 21:57:32 PST 2010 -project.organization=org.spark-project -project.name=spark -sbt.version=0.7.7 -project.version=0.4-SNAPSHOT -build.scala.versions=2.8.1 -project.initialize=false +sbt.version=0.10.1 diff --git a/project/build/SparkProject.scala b/project/build/SparkProject.scala deleted file mode 100644 index 2354f47a76..0000000000 --- a/project/build/SparkProject.scala +++ /dev/null @@ -1,107 +0,0 @@ -import sbt._ -import sbt.Process._ - -import assembly._ - -import de.element34.sbteclipsify._ - - -class SparkProject(info: ProjectInfo) extends ParentProject(info) with IdeaProject { - - lazy val core = project("core", "Spark Core", new CoreProject(_)) - - lazy val repl = project("repl", "Spark REPL", new ReplProject(_), core) - - lazy val examples = project("examples", "Spark Examples", new ExamplesProject(_), core) - - lazy val bagel = project("bagel", "Bagel", new BagelProject(_), core) - - trait BaseProject extends BasicScalaProject with ScalaPaths with BasicPackagePaths with Eclipsify with IdeaProject { - override def compileOptions = super.compileOptions ++ Seq(Unchecked) - - lazy val jettyServer = "org.eclipse.jetty" % "jetty-server" % "7.4.2.v20110526" - - override def packageDocsJar = defaultJarPath("-javadoc.jar") - override def packageSrcJar= defaultJarPath("-sources.jar") - lazy val sourceArtifact = Artifact.sources(artifactID) - lazy val docsArtifact = Artifact.javadoc(artifactID) - override def packageToPublishActions = super.packageToPublishActions ++ Seq(packageDocs, packageSrc) - } - - class CoreProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport { - val guava = "com.google.guava" % "guava" % "r09" - val log4j = "log4j" % "log4j" % "1.2.16" - val slf4jVersion = "1.6.1" - val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion - val slf4jLog4j = "org.slf4j" % "slf4j-log4j12" % slf4jVersion - val compressLzf = "com.ning" % "compress-lzf" % "0.7.0" - val hadoop = "org.apache.hadoop" % "hadoop-core" % "0.20.2" - val asm = "asm" % "asm-all" % "3.3.1" - val scalaTest = "org.scalatest" % "scalatest" % "1.3" % "test" - val scalaCheck = "org.scala-tools.testing" %% "scalacheck" % "1.7" % "test" - val protobuf = "com.google.protobuf" % "protobuf-java" % "2.3.0" - val kryoSerializers = "de.javakaffee" % "kryo-serializers" % "0.9" - } - - class ReplProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport - - class ExamplesProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject { - val colt = "colt" % "colt" % "1.2.0" - } - - class BagelProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport - - override def managedStyle = ManagedStyle.Maven -} - - -// Project mixin for an XML-based ScalaTest report. Unfortunately -// there is currently no way to call this directly from SBT without -// executing a subprocess. -trait XmlTestReport extends BasicScalaProject { - def testReportDir = outputPath / "test-report" - - lazy val testReport = task { - log.info("Creating " + testReportDir + "...") - if (!testReportDir.exists) { - testReportDir.asFile.mkdirs() - } - log.info("Executing org.scalatest.tools.Runner...") - val command = ("scala -classpath " + testClasspath.absString + - " org.scalatest.tools.Runner -o " + - " -u " + testReportDir.absolutePath + - " -p " + (outputPath / "test-classes").absolutePath) - Process(command, path("."), "JAVA_OPTS" -> "-Xmx500m") ! - - None - }.dependsOn(compile, testCompile).describedAs("Generate XML test report.") -} - - -// Project mixin for creating a JAR with a project's dependencies. This is based -// on the AssemblyBuilder plugin, but because this plugin attempts to package Scala -// and our project too, we leave that out using our own exclude filter (depJarExclude). -trait DepJar extends AssemblyBuilder { - def depJarExclude(base: PathFinder) = { - (base / "scala" ** "*") +++ // exclude scala library - (base / "spark" ** "*") +++ // exclude Spark classes - ((base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF - (base / "META-INF" / "services" ** "*") --- // include all service providers - (base / "META-INF" / "maven" ** "*")) // include all Maven POMs and such - } - - def depJarTempDir = outputPath / "dep-classes" - - def depJarOutputPath = - outputPath / (name.toLowerCase.replace(" ", "-") + "-dep-" + version.toString + ".jar") - - lazy val depJar = { - packageTask( - Path.lazyPathFinder(assemblyPaths(depJarTempDir, - assemblyClasspath, - assemblyExtraJars, - depJarExclude)), - depJarOutputPath, - packageOptions) - }.dependsOn(compile).describedAs("Bundle project's dependencies into a JAR.") -} diff --git a/project/plugins/SparkProjectPlugins.scala b/project/plugins/SparkProjectPlugins.scala deleted file mode 100644 index 565f160829..0000000000 --- a/project/plugins/SparkProjectPlugins.scala +++ /dev/null @@ -1,11 +0,0 @@ -import sbt._ - -class SparkProjectPlugins(info: ProjectInfo) extends PluginDefinition(info) { - val eclipse = "de.element34" % "sbt-eclipsify" % "0.7.0" - - val sbtIdeaRepo = "sbt-idea-repo" at "http://mpeltonen.github.com/maven/" - val sbtIdea = "com.github.mpeltonen" % "sbt-idea-plugin" % "0.4.0" - - val codaRepo = "Coda Hale's Repository" at "http://repo.codahale.com/" - val assemblySBT = "com.codahale" % "assembly-sbt" % "0.1.1" -} diff --git a/project/plugins/build.sbt b/project/plugins/build.sbt new file mode 100644 index 0000000000..91c6cb4df1 --- /dev/null +++ b/project/plugins/build.sbt @@ -0,0 +1,13 @@ +resolvers += { + val typesafeRepoUrl = new java.net.URL("http://repo.typesafe.com/typesafe/releases") + val pattern = Patterns(false, "[organisation]/[module]/[sbtversion]/[revision]/[type]s/[module](-[classifier])-[revision].[ext]") + Resolver.url("Typesafe Repository", typesafeRepoUrl)(pattern) +} + +resolvers += "sbt-idea-repo" at "http://mpeltonen.github.com/maven/" + +libraryDependencies += "com.github.mpeltonen" %% "sbt-idea" % "0.10.0" + +libraryDependencies <<= (libraryDependencies, sbtVersion) { (deps, version) => + deps :+ ("com.typesafe.sbteclipse" %% "sbteclipse" % "1.2" extra("sbtversion" -> version)) +} diff --git a/project/plugins/project/SparkPluginBuild.scala b/project/plugins/project/SparkPluginBuild.scala new file mode 100644 index 0000000000..999611982a --- /dev/null +++ b/project/plugins/project/SparkPluginBuild.scala @@ -0,0 +1,7 @@ +import sbt._ + +object SparkPluginDef extends Build { + lazy val root = Project("plugins", file(".")) dependsOn(junitXmlListener) + /* This is not published in a Maven repository, so we get it from GitHub directly */ + lazy val junitXmlListener = uri("git://github.com/ijuma/junit_xml_listener.git#fe434773255b451a38e8d889536ebc260f4225ce") +}
\ No newline at end of file |