From 1d75c6060a2a759dc6ca2e356925df6dcaaf75f4 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Fri, 27 May 2011 11:34:07 +0100 Subject: Update to Scala 2.9.0-1 and disable repl module for now. The repl module requires more complex work. --- run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'run') diff --git a/run b/run index efcf63cbbc..13c40af4c2 100755 --- a/run +++ b/run @@ -1,6 +1,6 @@ #!/bin/bash -SCALA_VERSION=2.8.1 +SCALA_VERSION=2.9.0 # Figure out where the Scala framework is installed FWDIR="$(cd `dirname $0`; pwd)" -- cgit v1.2.3 From 2c691295bf7811d0a4345db0871b38578955053e Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Fri, 27 May 2011 11:47:02 +0100 Subject: SCALA_VERSION in run should be 2.9.0-1 --- run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'run') diff --git a/run b/run index 13c40af4c2..8350874c92 100755 --- a/run +++ b/run @@ -1,6 +1,6 @@ #!/bin/bash -SCALA_VERSION=2.9.0 +SCALA_VERSION=2.9.0-1 # Figure out where the Scala framework is installed FWDIR="$(cd `dirname $0`; pwd)" -- cgit v1.2.3 From 3854d23dd4eed8bb9b8f0e9ae80c142b02868518 Mon Sep 17 00:00:00 2001 From: Matei Zaharia Date: Tue, 31 May 2011 22:17:48 -0700 Subject: Pass quoted arguments properly to run --- run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'run') diff --git a/run b/run index 8350874c92..2ea4cac9a7 100755 --- a/run +++ b/run @@ -66,4 +66,4 @@ else SCALA=scala fi -exec $SCALA -cp $CLASSPATH $@ +exec $SCALA -cp $CLASSPATH "$@" -- cgit v1.2.3 From f686e3dacb02d42dd2bb9695a96cecd85786d7b5 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Fri, 15 Jul 2011 03:38:25 +0100 Subject: Initial work on converting build to SBT 0.10.1 --- project/SparkBuild.scala | 101 ++++++++++++++++++++++++++++ project/build.properties | 9 +-- project/build/SparkProject.scala | 105 ------------------------------ project/plugins/SparkProjectPlugins.scala | 11 ---- project/plugins/build.sbt | 16 +++++ run | 17 ++--- sbt/sbt-launch-0.10.1.jar | Bin 0 -> 937683 bytes sbt/sbt-launch-0.7.5.jar | Bin 948813 -> 0 bytes 8 files changed, 127 insertions(+), 132 deletions(-) create mode 100644 project/SparkBuild.scala delete mode 100644 project/build/SparkProject.scala delete mode 100644 project/plugins/SparkProjectPlugins.scala create mode 100644 project/plugins/build.sbt create mode 100644 sbt/sbt-launch-0.10.1.jar delete mode 100644 sbt/sbt-launch-0.7.5.jar (limited to 'run') diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala new file mode 100644 index 0000000000..b6191561a0 --- /dev/null +++ b/project/SparkBuild.scala @@ -0,0 +1,101 @@ +import sbt._ +import Keys._ + +object SparkBuild extends Build { + + lazy val root = Project("root", file("."), settings = sharedSettings) aggregate(core, repl, examples, bagel) + + lazy val core = Project("core", file("core"), settings = coreSettings) + + lazy val repl = Project("repl", file("repl"), settings = replSettings) dependsOn (core) + + lazy val examples = Project("examples", file("examples"), settings = examplesSettings) dependsOn (core) + + lazy val bagel = Project("bagel", file("bagel"), settings = bagelSettings) dependsOn (core) + + def sharedSettings = Defaults.defaultSettings ++ Seq( + organization := "org.spark-project", + version := "version=0.4-SNAPSHOT", + scalaVersion := "2.9.0-1", + scalacOptions := Seq(/*"-deprecation",*/ "-unchecked"), // TODO Enable -deprecation and fix all warnings + unmanagedJars in Compile <<= baseDirectory map { base => (base ** "*.jar").classpath }, + retrieveManaged := true, + transitiveClassifiers in Scope.GlobalScope := Seq("sources"), + libraryDependencies ++= Seq( + "org.eclipse.jetty" % "jetty-server" % "7.4.2.v20110526", + "org.scalatest" % "scalatest_2.9.0" % "1.4.1" % "test", + "org.scala-tools.testing" % "scalacheck_2.9.0" % "1.9" % "test" + ) + ) + + val slf4jVersion = "1.6.1" + + //FIXME DepJar and XmlTestReport + def coreSettings = sharedSettings ++ Seq(libraryDependencies ++= Seq( + "com.google.guava" % "guava" % "r09", + "log4j" % "log4j" % "1.2.16", + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.slf4j" % "slf4j-log4j12" % slf4jVersion, + "com.ning" % "compress-lzf" % "0.7.0", + "org.apache.hadoop" % "hadoop-core" % "0.20.2", + "asm" % "asm-all" % "3.3.1" + )) + + //FIXME DepJar and XmlTestReport + def replSettings = sharedSettings ++ Seq(libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _)) + + def examplesSettings = sharedSettings ++ Seq(libraryDependencies += "colt" % "colt" % "1.2.0") + + //FIXME DepJar and XmlTestReport + def bagelSettings = sharedSettings +} + +// Project mixin for an XML-based ScalaTest report. Unfortunately +// there is currently no way to call this directly from SBT without +// executing a subprocess. +//trait XmlTestReport extends BasicScalaProject { +// def testReportDir = outputPath / "test-report" +// +// lazy val testReport = task { +// log.info("Creating " + testReportDir + "...") +// if (!testReportDir.exists) { +// testReportDir.asFile.mkdirs() +// } +// log.info("Executing org.scalatest.tools.Runner...") +// val command = ("scala -classpath " + testClasspath.absString + +// " org.scalatest.tools.Runner -o " + +// " -u " + testReportDir.absolutePath + +// " -p " + (outputPath / "test-classes").absolutePath) +// Process(command, path("."), "JAVA_OPTS" -> "-Xmx500m") ! +// +// None +// }.dependsOn(compile, testCompile).describedAs("Generate XML test report.") +//} + +// Project mixin for creating a JAR with a project's dependencies. This is based +// on the AssemblyBuilder plugin, but because this plugin attempts to package Scala +// and our project too, we leave that out using our own exclude filter (depJarExclude). +//trait DepJar extends AssemblyBuilder { +// def depJarExclude(base: PathFinder) = { +// (base / "scala" ** "*") +++ // exclude scala library +// (base / "spark" ** "*") +++ // exclude Spark classes +// ((base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF +// (base / "META-INF" / "services" ** "*") --- // include all service providers +// (base / "META-INF" / "maven" ** "*")) // include all Maven POMs and such +// } +// +// def depJarTempDir = outputPath / "dep-classes" +// +// def depJarOutputPath = +// outputPath / (name.toLowerCase.replace(" ", "-") + "-dep-" + version.toString + ".jar") +// +// lazy val depJar = { +// packageTask( +// Path.lazyPathFinder(assemblyPaths(depJarTempDir, +// assemblyClasspath, +// assemblyExtraJars, +// depJarExclude)), +// depJarOutputPath, +// packageOptions) +// }.dependsOn(compile).describedAs("Bundle project's dependencies into a JAR.") +//} diff --git a/project/build.properties b/project/build.properties index 9cef4f0d44..f47a3009ec 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1,8 +1 @@ -#Project properties -#Sat Nov 13 21:57:32 PST 2010 -project.organization=org.spark-project -project.name=spark -sbt.version=0.7.7 -project.version=0.4-SNAPSHOT -build.scala.versions=2.9.0-1 -project.initialize=false +sbt.version=0.10.1 diff --git a/project/build/SparkProject.scala b/project/build/SparkProject.scala deleted file mode 100644 index 1ffcc3ef34..0000000000 --- a/project/build/SparkProject.scala +++ /dev/null @@ -1,105 +0,0 @@ -import sbt._ -import sbt.Process._ - -import assembly._ - -import de.element34.sbteclipsify._ - - -class SparkProject(info: ProjectInfo) extends ParentProject(info) with IdeaProject { - - lazy val core = project("core", "Spark Core", new CoreProject(_)) - - lazy val repl = project("repl", "Spark REPL", new ReplProject(_), core) - - lazy val examples = project("examples", "Spark Examples", new ExamplesProject(_), core) - - lazy val bagel = project("bagel", "Bagel", new BagelProject(_), core) - - trait BaseProject extends BasicScalaProject with ScalaPaths with BasicPackagePaths with Eclipsify with IdeaProject { - override def compileOptions = super.compileOptions ++ Seq(Unchecked) - - lazy val jettyServer = "org.eclipse.jetty" % "jetty-server" % "7.4.2.v20110526" - - override def packageDocsJar = defaultJarPath("-javadoc.jar") - override def packageSrcJar= defaultJarPath("-sources.jar") - lazy val sourceArtifact = Artifact.sources(artifactID) - lazy val docsArtifact = Artifact.javadoc(artifactID) - override def packageToPublishActions = super.packageToPublishActions ++ Seq(packageDocs, packageSrc) - } - - class CoreProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport { - val guava = "com.google.guava" % "guava" % "r09" - val log4j = "log4j" % "log4j" % "1.2.16" - val slf4jVersion = "1.6.1" - val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion - val slf4jLog4j = "org.slf4j" % "slf4j-log4j12" % slf4jVersion - val compressLzf = "com.ning" % "compress-lzf" % "0.7.0" - val hadoop = "org.apache.hadoop" % "hadoop-core" % "0.20.2" - val asm = "asm" % "asm-all" % "3.3.1" - val scalaTest = "org.scalatest" % "scalatest_2.9.0" % "1.4.1" % "test" - val scalaCheck = "org.scala-tools.testing" % "scalacheck_2.9.0" % "1.9" % "test" - } - - class ReplProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport - - class ExamplesProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject { - val colt = "colt" % "colt" % "1.2.0" - } - - class BagelProject(info: ProjectInfo) extends DefaultProject(info) with BaseProject with DepJar with XmlTestReport - - override def managedStyle = ManagedStyle.Maven -} - - -// Project mixin for an XML-based ScalaTest report. Unfortunately -// there is currently no way to call this directly from SBT without -// executing a subprocess. -trait XmlTestReport extends BasicScalaProject { - def testReportDir = outputPath / "test-report" - - lazy val testReport = task { - log.info("Creating " + testReportDir + "...") - if (!testReportDir.exists) { - testReportDir.asFile.mkdirs() - } - log.info("Executing org.scalatest.tools.Runner...") - val command = ("scala -classpath " + testClasspath.absString + - " org.scalatest.tools.Runner -o " + - " -u " + testReportDir.absolutePath + - " -p " + (outputPath / "test-classes").absolutePath) - Process(command, path("."), "JAVA_OPTS" -> "-Xmx500m") ! - - None - }.dependsOn(compile, testCompile).describedAs("Generate XML test report.") -} - - -// Project mixin for creating a JAR with a project's dependencies. This is based -// on the AssemblyBuilder plugin, but because this plugin attempts to package Scala -// and our project too, we leave that out using our own exclude filter (depJarExclude). -trait DepJar extends AssemblyBuilder { - def depJarExclude(base: PathFinder) = { - (base / "scala" ** "*") +++ // exclude scala library - (base / "spark" ** "*") +++ // exclude Spark classes - ((base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF - (base / "META-INF" / "services" ** "*") --- // include all service providers - (base / "META-INF" / "maven" ** "*")) // include all Maven POMs and such - } - - def depJarTempDir = outputPath / "dep-classes" - - def depJarOutputPath = - outputPath / (name.toLowerCase.replace(" ", "-") + "-dep-" + version.toString + ".jar") - - lazy val depJar = { - packageTask( - Path.lazyPathFinder(assemblyPaths(depJarTempDir, - assemblyClasspath, - assemblyExtraJars, - depJarExclude)), - depJarOutputPath, - packageOptions) - }.dependsOn(compile).describedAs("Bundle project's dependencies into a JAR.") -} diff --git a/project/plugins/SparkProjectPlugins.scala b/project/plugins/SparkProjectPlugins.scala deleted file mode 100644 index 565f160829..0000000000 --- a/project/plugins/SparkProjectPlugins.scala +++ /dev/null @@ -1,11 +0,0 @@ -import sbt._ - -class SparkProjectPlugins(info: ProjectInfo) extends PluginDefinition(info) { - val eclipse = "de.element34" % "sbt-eclipsify" % "0.7.0" - - val sbtIdeaRepo = "sbt-idea-repo" at "http://mpeltonen.github.com/maven/" - val sbtIdea = "com.github.mpeltonen" % "sbt-idea-plugin" % "0.4.0" - - val codaRepo = "Coda Hale's Repository" at "http://repo.codahale.com/" - val assemblySBT = "com.codahale" % "assembly-sbt" % "0.1.1" -} diff --git a/project/plugins/build.sbt b/project/plugins/build.sbt new file mode 100644 index 0000000000..c13449ae03 --- /dev/null +++ b/project/plugins/build.sbt @@ -0,0 +1,16 @@ +resolvers += { + val typesafeRepoUrl = new java.net.URL("http://repo.typesafe.com/typesafe/releases") + val pattern = Patterns(false, "[organisation]/[module]/[sbtversion]/[revision]/[type]s/[module](-[classifier])-[revision].[ext]") + Resolver.url("Typesafe Repository", typesafeRepoUrl)(pattern) +} + +resolvers += "sbt-idea-repo" at "http://mpeltonen.github.com/maven/" + +libraryDependencies ++= Seq( + "com.github.mpeltonen" %% "sbt-idea" % "0.10.0-SNAPSHOT" +// FIXME Uncomment once version for SBT 0.10.1 is available "com.eed3si9n" %% "sbt-assembly" % "0.2" +) + +libraryDependencies <<= (libraryDependencies, sbtVersion) { (deps, version) => + deps :+ ("com.typesafe.sbteclipse" %% "sbteclipse" % "1.2" extra("sbtversion" -> version)) +} \ No newline at end of file diff --git a/run b/run index 2ea4cac9a7..78ed51110f 100755 --- a/run +++ b/run @@ -1,6 +1,6 @@ #!/bin/bash -SCALA_VERSION=2.9.0-1 +SCALA_VERSION=2.9.0.1 # Figure out where the Scala framework is installed FWDIR="$(cd `dirname $0`; pwd)" @@ -41,23 +41,23 @@ EXAMPLES_DIR=$FWDIR/examples BAGEL_DIR=$FWDIR/bagel # Build up classpath -CLASSPATH="$SPARK_CLASSPATH:$CORE_DIR/target/scala_$SCALA_VERSION/classes:$MESOS_CLASSPATH" +CLASSPATH="$SPARK_CLASSPATH:$CORE_DIR/target/scala-$SCALA_VERSION/classes:$MESOS_CLASSPATH" CLASSPATH+=:$FWDIR/conf -CLASSPATH+=:$REPL_DIR/target/scala_$SCALA_VERSION/classes -CLASSPATH+=:$EXAMPLES_DIR/target/scala_$SCALA_VERSION/classes +CLASSPATH+=:$REPL_DIR/target/scala-$SCALA_VERSION/classes +CLASSPATH+=:$EXAMPLES_DIR/target/scala-$SCALA_VERSION/classes for jar in `find $CORE_DIR/lib -name '*jar'`; do CLASSPATH+=:$jar done -for jar in $CORE_DIR/lib_managed/scala_$SCALA_VERSION/compile/*.jar; do +for jar in `find $FWDIR/lib_managed/jars -name '*jar'`; do CLASSPATH+=:$jar done -for jar in `find $REPL_DIR/lib -name '*jar'`; do +for jar in `find $FWDIR/lib_managed/bundles -name '*jar'`; do CLASSPATH+=:$jar done -for jar in $REPL_DIR/lib_managed/scala_$SCALA_VERSION/compile/*.jar; do +for jar in `find $REPL_DIR/lib -name '*jar'`; do CLASSPATH+=:$jar done -CLASSPATH+=:$BAGEL_DIR/target/scala_$SCALA_VERSION/classes +CLASSPATH+=:$BAGEL_DIR/target/scala-$SCALA_VERSION/classes export CLASSPATH # Needed for spark-shell if [ -n "$SCALA_HOME" ]; then @@ -66,4 +66,5 @@ else SCALA=scala fi +echo $CLASSPATH >> tmp exec $SCALA -cp $CLASSPATH "$@" diff --git a/sbt/sbt-launch-0.10.1.jar b/sbt/sbt-launch-0.10.1.jar new file mode 100644 index 0000000000..673495f78a Binary files /dev/null and b/sbt/sbt-launch-0.10.1.jar differ diff --git a/sbt/sbt-launch-0.7.5.jar b/sbt/sbt-launch-0.7.5.jar deleted file mode 100644 index 052c1e1e56..0000000000 Binary files a/sbt/sbt-launch-0.7.5.jar and /dev/null differ -- cgit v1.2.3