Merge pull request #111 from cvogt/reproducible-builds

Reproducible builds

12 files changed, 324 insertions, 164 deletions

diff --git a/stage2/AdminStage2.scala b/stage2/AdminStage2.scala
index 9d7dcb2..f4e61d0 100644
--- a/stage2/AdminStage2.scala
+++ b/stage2/AdminStage2.scala
@@ -4,7 +4,7 @@ object AdminStage2 extends Stage2Base{
   def run( _args: Stage2Args ): Unit = {
     val args = _args.args.dropWhile(Seq("admin","direct") contains _)
     val lib = new Lib(_args.logger)
-    val adminTasks = new AdminTasks(lib, args, _args.cwd, _args.classLoaderCache)
+    val adminTasks = new AdminTasks(lib, args, _args.cwd, _args.classLoaderCache, _args.cache, _args.cbtHome, _args.cbtHasChanged)
     new lib.ReflectObject(adminTasks){
       def usage: String = "Available methods: " ++ lib.taskNames(adminTasks.getClass).mkString("  ")
     }.callNullary(args.lift(0))
diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala
index f189805..9086470 100644
--- a/stage2/AdminTasks.scala
+++ b/stage2/AdminTasks.scala
@@ -2,14 +2,25 @@ package cbt
 import scala.collection.immutable.Seq
 import java.io.{Console=>_,_}
 import java.nio.file._
-class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: ClassLoaderCache){
+class AdminTasks(
+  lib: Lib,
+  args: Seq[String],
+  cwd: File,
+  classLoaderCache: ClassLoaderCache,
+  cache: File,
+  cbtHome: File,
+  cbtHasChanged: Boolean
+){
+  private val paths = Paths(cbtHome, cache)
+  import paths._
+  private val mavenCentral = MavenResolver(cbtHasChanged,mavenCache,MavenResolver.central)
   implicit val logger: Logger = lib.logger
   def resolve = {
     ClassPath.flatten(
       args(1).split(",").toVector.map{
         d =>
           val v = d.split(":")
-          MavenRepository.central.resolveOne(MavenDependency(v(0),v(1),v(2))).classpath
+          mavenCentral.resolveOne(MavenDependency(v(0),v(1),v(2))).classpath
       }
     )
   }
@@ -17,14 +28,14 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class
     args(1).split(",").toVector.map{
       d =>
         val v = d.split(":")
-        MavenRepository.central.resolveOne(MavenDependency(v(0),v(1),v(2))).dependencyTree
+        mavenCentral.resolveOne(MavenDependency(v(0),v(1),v(2))).dependencyTree
     }.mkString("\n\n")
   }
   def amm = ammonite
   def ammonite = {
     val version = args.lift(1).getOrElse(constants.scalaVersion)
-    val scalac = new ScalaCompilerDependency( version )
-    val d = MavenRepository.central.resolveOne(
+    val scalac = new ScalaCompilerDependency( cbtHasChanged,mavenCache, version )
+    val d = mavenCentral.resolveOne(
       MavenDependency(
         "com.lihaoyi","ammonite-repl_2.11.7",args.lift(1).getOrElse("0.5.7")
       )
@@ -36,7 +47,7 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class
   }
   def scala = {
     val version = args.lift(1).getOrElse(constants.scalaVersion)
-    val scalac = new ScalaCompilerDependency( version )
+    val scalac = new ScalaCompilerDependency( cbtHasChanged, mavenCache, version )
     lib.runMain(
       "scala.tools.nsc.MainGenericRunner", Seq("-cp", scalac.classpath.string), scalac.classLoader(classLoaderCache)
     )
@@ -49,16 +60,16 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class
     val scalaXmlVersion = args.lift(2).getOrElse(constants.scalaXmlVersion)
     val zincVersion = args.lift(3).getOrElse(constants.zincVersion)
     val scalaDeps = Seq(
-      MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)),
-      MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion))
+      mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)),
+      mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion))
     )
     
     val scalaXml = Dependencies(
-      MavenRepository.central.resolveOne(MavenDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion)),
-      MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-library",scalaVersion))
+      mavenCentral.resolveOne(MavenDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion)),
+      mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-library",scalaVersion))
     )
 
-    val zinc = MavenRepository.central.resolveOne(MavenDependency("com.typesafe.zinc","zinc",zincVersion))
+    val zinc = mavenCentral.resolveOne(MavenDependency("com.typesafe.zinc","zinc",zincVersion))
 
     def valName(dep: BoundMavenDependency) = {
       val words = dep.artifactId.split("_").head.split("-")
@@ -66,24 +77,28 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class
     }
 
     def jarVal(dep: BoundMavenDependency) = "_" + valName(dep) +"Jar"
-    def transitive(dep: Dependency) = (dep +: dep.transitiveDependencies.reverse).collect{case d: BoundMavenDependency => d}
+    def transitive(dep: Dependency) = (dep +: lib.transitiveDependencies(dep).reverse).collect{case d: BoundMavenDependency => d}
     def codeEach(dep: Dependency) = {    
       transitive(dep).tails.map(_.reverse).toVector.reverse.drop(1).map{
         deps =>
           val d = deps.last
           val parents = deps.dropRight(1)
-          val parentString = if(parents.isEmpty) ""  else ( ", " ++ valName(parents.last) )
+          val parentString = if(parents.isEmpty) "rootClassLoader"  else ( valName(parents.last) )
           val n = valName(d)
           s"""
     // ${d.groupId}:${d.artifactId}:${d.version}
-    download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}");
-    ClassLoader $n = cachePut(
-      classLoader( ${n}File$parentString ),
-      ${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")}
-    );"""
+    download(new URL(mavenUrl + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}");
+
+    String[] ${n}ClasspathArray = new String[]{${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")}};
+    String ${n}Classpath = classpath( ${n}ClasspathArray );
+    ClassLoader $n =
+      classLoaderCache.contains( ${n}Classpath )
+      ? classLoaderCache.get( ${n}Classpath )
+      : classLoaderCache.put( classLoader( ${n}File, $parentString ), ${n}Classpath );"""
       }
     }
     val assignments = codeEach(zinc) ++ codeEach(scalaXml)
+    val files = scalaDeps ++ transitive(scalaXml) ++ transitive(zinc)
     //{ case (name, dep) => s"$name =\n      ${tree(dep, 4)};" }.mkString("\n\n    ")
     val code = s"""// This file was auto-generated using `cbt admin cbtEarlyDependencies`
 package cbt;
@@ -97,23 +112,29 @@ import static cbt.NailgunLauncher.*;
 class EarlyDependencies{
 
   /** ClassLoader for stage1 */
-  ClassLoader stage1;
+  ClassLoader classLoader;
+  String[] classpathArray;
   /** ClassLoader for zinc */
   ClassLoader zinc;
 
-${(scalaDeps ++ transitive(scalaXml) ++ transitive(zinc)).map(d => s"""  String ${valName(d)}File = MAVEN_CACHE + "${d.basePath}.jar";""").mkString("\n")}
+${files.map(d => s"""  String ${valName(d)}File;""").mkString("\n")}
+
+  public EarlyDependencies(
+    String mavenCache, String mavenUrl, ClassLoaderCache2<ClassLoader> classLoaderCache, ClassLoader rootClassLoader
+  ) throws Exception {
+${files.map(d => s"""    ${valName(d)}File = mavenCache + "${d.basePath}.jar";""").mkString("\n")}
 
-  public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{
-${scalaDeps.map(d => s"""    download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")}
+${scalaDeps.map(d => s"""    download(new URL(mavenUrl + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")}
 ${assignments.mkString("\n")}
   
-    stage1 = scalaXml_${scalaXmlVersion.replace(".","_")}_;
+    classLoader = scalaXml_${scalaXmlVersion.replace(".","_")}_;
+    classpathArray = scalaXml_${scalaXmlVersion.replace(".","_")}_ClasspathArray;
 
     zinc = zinc_${zincVersion.replace(".","_")}_;
   }
 }
 """
-    val file = paths.nailgun ++ ("/" ++ "EarlyDependencies.java")
+    val file = nailgun ++ ("/" ++ "EarlyDependencies.java")
     Files.write( file.toPath, code.getBytes )
     println( Console.GREEN ++ "Wrote " ++ file.string ++ Console.RESET )
   }
diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala
index 9f9dbdc..fb5e652 100644
--- a/stage2/BasicBuild.scala
+++ b/stage2/BasicBuild.scala
@@ -1,5 +1,4 @@
 package cbt
-import cbt.paths._
 
 import java.io._
 import java.net._
@@ -11,21 +10,30 @@ import java.util.jar._
 import scala.collection.immutable.Seq
 import scala.util._
 
-class BasicBuild( context: Context ) extends Build( context )
-class Build(val context: Context) extends Dependency with TriggerLoop with SbtDependencyDsl{
+trait Recommended extends BasicBuild{
+  override def scalacOptions = super.scalacOptions ++ Seq(
+    "-feature",
+    "-deprecation",
+    "-unchecked",
+    "-language:postfixOps",
+    "-language:implicitConversions",
+    "-language:higherKinds",
+    "-language:existentials"
+  )
+}
+class BasicBuild(val context: Context) extends DependencyImplementation with BuildInterface with TriggerLoop with SbtDependencyDsl{
   // library available to builds
-  implicit final val logger: Logger = context.logger
-  implicit final val classLoaderCache: ClassLoaderCache = context.classLoaderCache
-  implicit final val _context = context
-  override final protected val lib: Lib = new Lib(logger)
+  implicit protected final val logger: Logger = context.logger
+  implicit protected final val classLoaderCache: ClassLoaderCache = context.classLoaderCache
+  implicit protected final val _context = context
+  override protected final val lib: Lib = new Lib(logger)
 
   // ========== general stuff ==========
 
-  override def canBeCached = false
   def enableConcurrency = false
   final def projectDirectory: File = lib.realpath(context.projectDirectory)
   assert( projectDirectory.exists, "projectDirectory does not exist: " ++ projectDirectory.string )
-  final def usage: String = lib.usage(this.getClass, context)
+  final def usage: String = lib.usage(this.getClass, show)
 
   // ========== meta data ==========
 
@@ -33,11 +41,15 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe
   final def scalaVersion = context.scalaVersion getOrElse defaultScalaVersion
   final def scalaMajorVersion: String = lib.scalaMajorVersion(scalaVersion)
   def crossScalaVersions: Seq[String] = Seq(scalaVersion, "2.10.6")
-  def copy(context: Context) = lib.copy(this.getClass, context).asInstanceOf[Build]
+  final def crossScalaVersionsArray: Array[String] = crossScalaVersions.to
+
+  // TODO: this should probably provide a nice error message if class has constructor signature
+  def copy(context: Context): BuildInterface = lib.copy(this.getClass, context).asInstanceOf[BuildInterface]
   def zincVersion = "0.3.9"
 
   def dependencies: Seq[Dependency] = Seq(
-    MavenRepository.central.resolve(
+    // FIXME: this should probably be removed
+    MavenResolver(context.cbtHasChanged, context.paths.mavenCache, MavenResolver.central).resolve(
       "org.scala-lang" % "scala-library" % scalaVersion
     )
   )
@@ -63,16 +75,10 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe
   def compileStatusFile: File = compileTarget ++ ".last-success"
 
   /** Source directories and files. Defaults to .scala and .java files in src/ and top-level. */
-  def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(sourceFileFilter)
-
-  /** Which file endings to consider being source files. */
-  def sourceFileFilter(file: File): Boolean = file.toString.endsWith(".scala") || file.toString.endsWith(".java")
+  def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(lib.sourceFileFilter)
 
   /** Absolute path names for all individual files found in sources directly or contained in directories. */
-  final def sourceFiles: Seq[File] = for {
-    base <- sources.filter(_.exists).map(lib.realpath)
-    file <- lib.listFilesRecursive(base) if file.isFile && sourceFileFilter(file)
-  } yield file
+  final def sourceFiles: Seq[File] = lib.sourceFiles(sources)
 
   protected def assertSourceDirectories(): Unit = {
     val nonExisting =
@@ -103,38 +109,87 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe
       .flatMap(_.listFiles)
       .filter(_.toString.endsWith(".jar"))
 
-  //def cacheJar = false
   override def dependencyClasspath : ClassPath = ClassPath(localJars) ++ super.dependencyClasspath
-  override def dependencyJars      : Seq[File] = localJars ++ super.dependencyJars
 
   def exportedClasspath   : ClassPath = ClassPath(compile.toSeq:_*)
   def targetClasspath = ClassPath(Seq(compileTarget))
-  def exportedJars: Seq[File] = Seq()
   // ========== compile, run, test ==========
 
   /** scalac options used for zinc and scaladoc */
-  def scalacOptions: Seq[String] = Seq( "-feature", "-deprecation", "-unchecked" )
+  def scalacOptions: Seq[String] = Seq()
 
   private object needsUpdateCache extends Cache[Boolean]
   def needsUpdate: Boolean = needsUpdateCache(
     context.cbtHasChanged
     || lib.needsUpdate( sourceFiles, compileStatusFile )
-    || transitiveDependencies.exists(_.needsUpdate)
+    || transitiveDependencies.filterNot(_ == context.parentBuild).exists(_.needsUpdate)
   )
 
   private object compileCache extends Cache[Option[File]]
   def compile: Option[File] = compileCache{
     lib.compile(
-      needsUpdate,
-      sourceFiles, compileTarget, compileStatusFile, dependencyClasspath, scalacOptions,
-      context.classLoaderCache, zincVersion = zincVersion, scalaVersion = scalaVersion
+      context.cbtHasChanged,
+      needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false),
+      sourceFiles, compileTarget, compileStatusFile, dependencyClasspath,
+      context.paths.mavenCache, scalacOptions, context.classLoaderCache,
+      zincVersion = zincVersion, scalaVersion = scalaVersion
     )
   }
 
   def runClass: String = "Main"
   def run: ExitCode = lib.runMainIfFound( runClass, context.args, classLoader(context.classLoaderCache) )
 
-  def test: ExitCode = lib.test(context)
+  def test: Option[ExitCode] = {
+    lib.test(context)
+  }
+
+  def recursiveSafe(_run: BuildInterface => Any): ExitCode = {
+    val builds = (this +: transitiveDependencies).collect{
+      case b: BuildInterface => b
+    }
+    val results = builds.map(_run)
+    if(
+      results.forall{
+        case Some(_:ExitCode) => true
+        case None => true
+        case _:ExitCode => true
+        case other => false
+      }
+    ){
+      if(
+        results.collect{
+          case Some(c:ExitCode) => c
+          case c:ExitCode => c
+        }.filter(_ != 0)
+         .nonEmpty
+      ) ExitCode.Failure
+      else ExitCode.Success
+    } else ExitCode.Success
+  }
+
+  def recursive: ExitCode = {
+    recursiveUnsafe(context.args.lift(1))
+  }
+
+  def recursiveUnsafe(taskName: Option[String]): ExitCode = {
+    recursiveSafe{
+      b =>
+      System.err.println(b.show)
+      lib.trapExitCode{ // FIXME: trapExitCode does not seem to work here
+        try{
+          new lib.ReflectBuild(b).callNullary(taskName)
+          ExitCode.Success
+        } catch {
+          case e: Throwable => println(e.getClass); throw e
+        }        
+      }
+      ExitCode.Success
+    }
+  }
+
+  def c = compile
+  def t = test
+  def rt = recursiveUnsafe(Some("test"))
 
   /*
   context.logger.composition(">"*80)
@@ -148,6 +203,6 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe
   */
 
   // ========== cbt internals ==========
-  private[cbt] def finalBuild = this
+  def finalBuild: BuildInterface = this
   override def show = this.getClass.getSimpleName ++ "(" ++ projectDirectory.string ++ ")"
 }
diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala
index c52c3c6..bab8d88 100644
--- a/stage2/BuildBuild.scala
+++ b/stage2/BuildBuild.scala
@@ -1,25 +1,63 @@
 package cbt
 import java.io.File
+import java.nio.file._
 import scala.collection.immutable.Seq
 
-class BuildBuild(context: Context) extends Build(context){
-  override def dependencies = Seq( CbtDependency()(context.logger) ) ++ super.dependencies
+class BuildBuild(context: Context) extends BasicBuild(context){
+  override def dependencies =
+    super.dependencies :+ context.cbtDependency
   def managedBuildDirectory: File = lib.realpath( projectDirectory.parent )
-  val managedBuild = try{
-    val managedContext = context.copy( projectDirectory = managedBuildDirectory )
-    val cl = classLoader(context.classLoaderCache)
-    logger.composition("Loading build at "+managedContext.projectDirectory)
-      cl
-        .loadClass(lib.buildClassName)
-        .getConstructor(classOf[Context])
-        .newInstance(managedContext)
-        .asInstanceOf[Build]
-  } catch {
-    case e: ClassNotFoundException if e.getMessage == lib.buildClassName => 
-      throw new Exception("You need to remove the directory or define a class Build in: "+context.projectDirectory)
-    case e: Exception =>
-      throw new Exception("during build: "+context.projectDirectory, e)
+  private object managedBuildCache extends Cache[BuildInterface]
+  def managedBuild = managedBuildCache{
+    try{
+      val managedContext = context.copy(
+        projectDirectory = managedBuildDirectory,
+        parentBuild=Some(this)
+      )
+      val managedBuildFile = projectDirectory++"/build.scala"
+      logger.composition("Loading build at "++managedContext.projectDirectory.toString)
+      (
+        if(managedBuildFile.exists){
+          val contents = new String(Files.readAllBytes(managedBuildFile.toPath))
+          val cbtUrl = ("cbt:"++GitDependency.GitUrl.regex++"#[a-z0-9A-Z]+").r
+          cbtUrl
+            .findFirstIn(contents)
+            .flatMap{
+              url =>
+              val Array(base,hash) = url.drop(4).split("#")
+              if(context.cbtHome.string.contains(hash))
+                None
+              else Some{
+                val checkoutDirectory = new GitDependency(base, hash).checkout
+                val build = new BasicBuild( context.copy( projectDirectory = checkoutDirectory ++ "/nailgun_launcher" ) )
+                val cl = build
+                  .classLoader(classLoaderCache)
+                // Note: cbt can't use an old version of itself for building,
+                // otherwise we'd have to recursively build all versions since
+                // the beginning. Instead CBT always needs to build the pure Java
+                // Launcher in the checkout with itself and then run it via reflection.
+                cl
+                  .loadClass( "cbt.NailgunLauncher" )
+                  .getMethod( "getBuild", classOf[AnyRef] )
+                  .invoke( null, managedContext.copy(cbtHome=checkoutDirectory) )
+              }
+            }.getOrElse{
+              classLoader(context.classLoaderCache)
+                .loadClass(lib.buildClassName)
+                .getConstructors.head
+                .newInstance(managedContext)
+            }
+        } else {
+          new BasicBuild(managedContext)
+        }
+      ).asInstanceOf[BuildInterface]
+    } catch {
+      case e: ClassNotFoundException if e.getMessage == lib.buildClassName => 
+        throw new Exception("You need to remove the directory or define a class Build in: "+context.projectDirectory)
+      case e: Exception =>
+        throw new Exception("during build: "+context.projectDirectory, e)
+    }
   }
   override def triggerLoopFiles = super.triggerLoopFiles ++ managedBuild.triggerLoopFiles
-  override def finalBuild = if( context.projectDirectory == context.cwd ) this else managedBuild.finalBuild
+  override def finalBuild: BuildInterface = if( context.projectDirectory == context.cwd ) this else managedBuild.finalBuild
 }
diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala
index 8965cee..f6b6911 100644
--- a/stage2/BuildDependency.scala
+++ b/stage2/BuildDependency.scala
@@ -11,7 +11,8 @@ sealed abstract class ProjectProxy extends Ha{
   def dependencies = Seq(delegate)
 }
 */
-trait TriggerLoop extends Dependency{
+trait TriggerLoop extends DependencyImplementation{
+  final def triggerLoopFilesArray = triggerLoopFiles.toArray
   def triggerLoopFiles: Seq[File]
 }
 /** You likely want to use the factory method in the BasicBuild class instead of this. */
@@ -21,9 +22,7 @@ case class BuildDependency(context: Context) extends TriggerLoop{
   final override lazy val lib: Lib = new Lib(logger)
   private val root = lib.loadRoot( context.copy(args=Seq()) )
   lazy val build = root.finalBuild
-  override def canBeCached = build.canBeCached
   def exportedClasspath = ClassPath(Seq())
-  def exportedJars = Seq()
   def dependencies = Seq(build)
   def triggerLoopFiles = root.triggerLoopFiles
   override final val needsUpdate = build.needsUpdate
diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala
index 16423df..174f9ff 100644
--- a/stage2/GitDependency.scala
+++ b/stage2/GitDependency.scala
@@ -5,25 +5,25 @@ import scala.collection.immutable.Seq
 import org.eclipse.jgit.api._
 import org.eclipse.jgit.lib.Ref
 
+object GitDependency{
+  val GitUrl = "(git:|https:|file:/)//([^/]+)/(.+)".r
+}
 case class GitDependency(
   url: String, ref: String // example: git://github.com/cvogt/cbt.git#<some-hash>
-)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends Dependency{
+)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends DependencyImplementation{
+  import GitDependency._
   override def lib = new Lib(logger)
 
-  override def canBeCached = true
   // TODO: add support for authentication via ssh and/or https
   // See http://www.codeaffine.com/2014/12/09/jgit-authentication/
+  private val GitUrl( _, domain, path ) = url  
+
 
-  private val GitUrl = "(git|https)://([^/]+)/(.+)".r
-  private val GitUrl( _, domain, path ) = url
-  
-  private object dependenciesCache extends Cache[Seq[Dependency]]
-  def dependencies = dependenciesCache{
-    val checkoutDirectory = paths.cbtHome ++ s"/cache/git/$domain/$path/$ref"
+  def checkout: File = {
+    val checkoutDirectory = context.cache ++ s"/git/$domain/$path/$ref"
     if(checkoutDirectory.exists){
       logger.git(s"Found existing checkout of $url#$ref in $checkoutDirectory")
     } else {
-
       logger.git(s"Cloning $url into $checkoutDirectory")
       val git =
         Git.cloneRepository()
@@ -35,16 +35,17 @@ case class GitDependency(
       git.checkout()
           .setName(ref)
           .call()
-
     }
-    val managedBuild = lib.loadDynamic(
-      context.copy( projectDirectory = checkoutDirectory, args = Seq() )
-    )    
-    Seq( managedBuild )
+    checkoutDirectory
+  }
+  private object dependencyCache extends Cache[Dependency]
+  def dependency = dependencyCache{
+    BuildDependency( context.copy( projectDirectory = checkout ) )
   }
 
+  def dependencies = Seq(dependency)
+
   def exportedClasspath = ClassPath(Seq())
-  def exportedJars = Seq()
   private[cbt] def targetClasspath = exportedClasspath
   def needsUpdate: Boolean = false
 }
diff --git a/stage2/Lib.scala b/stage2/Lib.scala
index cebdb92..1b19a5e 100644
--- a/stage2/Lib.scala
+++ b/stage2/Lib.scala
@@ -1,5 +1,4 @@
 package cbt
-import cbt.paths._
 
 import java.io._
 import java.net._
@@ -30,7 +29,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
     .newInstance(context)
 
   /** Loads Build for given Context */
-  def loadDynamic(context: Context, default: Context => Build = new Build(_)): Build = {
+  def loadDynamic(context: Context, default: Context => BuildInterface = new BasicBuild(_)): BuildInterface = {
     context.logger.composition( context.logger.showInvocation("Build.loadDynamic",context) )
     loadRoot(context, default).finalBuild
   }
@@ -38,7 +37,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
   Loads whatever Build needs to be executed first in order to eventually build the build for the given context.
   This can either the Build itself, of if exists a BuildBuild or a BuildBuild for a BuildBuild and so on.
   */
-  def loadRoot(context: Context, default: Context => Build = new Build(_)): Build = {
+  def loadRoot(context: Context, default: Context => BuildInterface = new BasicBuild(_)): BuildInterface = {
     context.logger.composition( context.logger.showInvocation("Build.loadRoot",context.projectDirectory) )
     def findStartDir(projectDirectory: File): File = {
       val buildDir = realpath( projectDirectory ++ "/build" )
@@ -73,6 +72,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
   }
 
   def docJar(
+    cbtHasChanged: Boolean,
     scalaVersion: String,
     sourceFiles: Seq[File],
     dependencyClasspath: ClassPath,
@@ -82,7 +82,8 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
     scalaMajorVersion: String,
     version: String,
     compileArgs: Seq[String],
-    classLoaderCache: ClassLoaderCache
+    classLoaderCache: ClassLoaderCache,
+    mavenCache: File
   ): Option[File] = {
     if(sourceFiles.isEmpty){
       None
@@ -98,7 +99,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
         runMain(
           "scala.tools.nsc.ScalaDoc",
           args,
-          ScalaDependencies(scalaVersion)(logger).classLoader(classLoaderCache)
+          ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion)(logger).classLoader(classLoaderCache)
         )
       }
       lib.jarFile(
@@ -108,19 +109,24 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
     }
   }
 
-  def test( context: Context ): ExitCode = {
-    val loggers = logger.enabledLoggers.mkString(",")
-    // FIXME: this is a hack to pass logger args on to the tests.
-    // should probably have a more structured way
-    val loggerArg = if(loggers != "") Some("-Dlog="++loggers) else None
-
-    logger.lib(s"invoke testDefault( $context )")
-    val exitCode: ExitCode = loadDynamic(
-      context.copy( projectDirectory = context.projectDirectory ++ "/test", args = loggerArg.toVector ++ context.args ),
-      new Build(_) with mixins.Test
-    ).run
-    logger.lib(s"return testDefault( $context )")
-    exitCode
+  def test( context: Context ): Option[ExitCode] = {
+    if((context.projectDirectory ++ "/test").exists){
+      val loggers = logger.enabledLoggers.mkString(",")
+      // FIXME: this is a hack to pass logger args on to the tests.
+      // should probably have a more structured way
+      val loggerArg = if(loggers != "") Some("-Dlog="++loggers) else None
+
+      logger.lib(s"invoke testDefault( $context )")
+      val exitCode: ExitCode =
+        new ReflectBuild(
+          loadDynamic(
+            context.copy( projectDirectory = context.projectDirectory ++ "/test", args = loggerArg.toVector ++ context.args ),
+            new BasicBuild(_) with mixins.Test
+          )
+        ).callNullary( Some("run") )
+      logger.lib(s"return testDefault( $context )")
+      Some(exitCode)
+    } else None
   }
 
   // task reflection helpers
@@ -145,13 +151,18 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
 
   def taskNames(cls: Class[_]): Seq[String] = tasks(cls).keys.toVector.sorted
 
-  def usage(buildClass: Class[_], context: Context): String = {
-    val baseTasks = lib.taskNames(classOf[Build])
+  def usage(buildClass: Class[_], show: String): String = {
+    val baseTasks = Seq(
+      classOf[BasicBuild],
+      classOf[PackageBuild],
+      classOf[PublishBuild],
+      classOf[Recommended]
+    ).flatMap(lib.taskNames).distinct.sorted
     val thisTasks = lib.taskNames(buildClass) diff baseTasks
     (
       (
         if( thisTasks.nonEmpty ){
-          s"""Methods provided by Build ${context.projectDirectory}
+          s"""Methods provided by Build ${show}
 
   ${thisTasks.mkString("  ")}
 
@@ -163,12 +174,13 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
       ) ++ "\n"
   }
 
-  class ReflectBuild[T:scala.reflect.ClassTag](build: Build) extends ReflectObject(build){
-    def usage = lib.usage(build.getClass, build.context)
+  class ReflectBuild[T:scala.reflect.ClassTag](build: BuildInterface) extends ReflectObject(build){
+    def usage = lib.usage(build.getClass, build.show)
   }
-  abstract class ReflectObject[T:scala.reflect.ClassTag](obj: T){
+  abstract class ReflectObject[T](obj: T){
     def usage: String
-    def callNullary( taskName: Option[String] ): Unit = {
+    def callNullary( taskName: Option[String] ): ExitCode = {
+      logger.lib("Calling task " ++ taskName.toString)
       val ts = tasks(obj.getClass)
       taskName.map( NameTransformer.encode ).flatMap(ts.get).map{ method =>
         val result: Option[Any] = Option(method.invoke(obj)) // null in case of Unit
@@ -181,26 +193,30 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
           scala.util.Try( value.getClass.getDeclaredMethod("toConsole") ) match {
             case scala.util.Success(toConsole) =>
               println(toConsole.invoke(value))
+              ExitCode.Success
 
             case scala.util.Failure(e) if Option(e.getMessage).getOrElse("") contains "toConsole" =>
               value match {
-                case ExitCode(code) => System.exit(code)
-                case other => println( other.toString ) // no method .toConsole, using to String
+                case code:ExitCode =>
+                  code
+                case other =>
+                  println( other.toString ) // no method .toConsole, using to String
+                  ExitCode.Success
               }
 
             case scala.util.Failure(e) =>
               throw e
           }
-        }.getOrElse("")
+        }.getOrElse(ExitCode.Success)
       }.getOrElse{
         taskName.foreach{ n =>
           System.err.println(s"Method not found: $n")
           System.err.println("")
         }
         System.err.println(usage)
-        taskName.foreach{ _ =>
+        taskName.map{ _ =>
           ExitCode.Failure
-        }
+        }.getOrElse( ExitCode.Success )
       }
     }
   }
@@ -208,7 +224,17 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
   // file system helpers
   def basename(path: File): String = path.toString.stripSuffix("/").split("/").last
   def dirname(path: File): File = new File(realpath(path).string.stripSuffix("/").split("/").dropRight(1).mkString("/"))
-  def nameAndContents(file: File) = basename(file) -> readAllBytes(Paths.get(file.toString))
+  def nameAndContents(file: File) = basename(file) -> readAllBytes(file.toPath)
+
+  /** Which file endings to consider being source files. */
+  def sourceFileFilter(file: File): Boolean = file.toString.endsWith(".scala") || file.toString.endsWith(".java")
+
+  def sourceFiles( sources: Seq[File], sourceFileFilter: File => Boolean = sourceFileFilter ): Seq[File] = {
+    for {
+      base <- sources.filter(_.exists).map(lib.realpath)
+      file <- lib.listFilesRecursive(base) if file.isFile && sourceFileFilter(file)
+    } yield file    
+  }
 
   def jarFile( jarFile: File, files: Seq[File] ): Option[File] = {
     if( files.isEmpty ){
@@ -229,7 +255,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
           val entry = new JarEntry( name )
           entry.setTime(file.lastModified)
           jar.putNextEntry(entry)
-          jar.write( readAllBytes( Paths.get(file.toString) ) )
+          jar.write( readAllBytes( file.toPath ) )
           jar.closeEntry
           name
       }
@@ -340,14 +366,14 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
     else items.map(projection)
   }
 
-  def publishSnapshot( sourceFiles: Seq[File], artifacts: Seq[File], url: URL ): Unit = {
+  def publishSnapshot( sourceFiles: Seq[File], artifacts: Seq[File], url: URL, credentials: String ): Unit = {
     if(sourceFiles.nonEmpty){
       val files = artifacts.map(nameAndContents)
-      uploadAll(url, files)
+      uploadAll(url, files, credentials)
     }
   }
 
-  def publishSigned( sourceFiles: Seq[File], artifacts: Seq[File], url: URL ): Unit = {
+  def publishSigned( sourceFiles: Seq[File], artifacts: Seq[File], url: URL, credentials: String ): Unit = {
     // TODO: make concurrency configurable here
     if(sourceFiles.nonEmpty){
       val files = (artifacts ++ artifacts.map(sign)).map(nameAndContents)
@@ -358,15 +384,15 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
         )
       }
       val all = (files ++ checksums)
-      uploadAll(url, all)
+      uploadAll(url, all, credentials)
     }
   }
 
 
-  def uploadAll(url: URL, nameAndContents: Seq[(String, Array[Byte])]): Unit =
-    nameAndContents.map{ case(name, content) => upload(name, content, url) }
+  def uploadAll(url: URL, nameAndContents: Seq[(String, Array[Byte])], credentials: String ): Unit =
+    nameAndContents.map{ case(name, content) => upload(name, content, url, credentials: String ) }
 
-  def upload(fileName: String, fileContents: Array[Byte], baseUrl: URL): Unit = {
+  def upload(fileName: String, fileContents: Array[Byte], baseUrl: URL, credentials: String): Unit = {
     import java.net._
     import java.io._
     logger.task("uploading "++fileName)
@@ -374,8 +400,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{
     val httpCon = url.openConnection.asInstanceOf[HttpURLConnection]
     httpCon.setDoOutput(true)
     httpCon.setRequestMethod("PUT")
-    val userPassword = new String(readAllBytes(sonatypeLogin.toPath)).trim
-    val encoding = new sun.misc.BASE64Encoder().encode(userPassword.getBytes)
+    val encoding = new sun.misc.BASE64Encoder().encode(credentials.getBytes)
     httpCon.setRequestProperty("Authorization", "Basic " ++ encoding)
     httpCon.setRequestProperty("Content-Type", "application/binary")
     httpCon.getOutputStream.write(
diff --git a/stage2/PackageBuild.scala b/stage2/PackageBuild.scala
index d24bf38..869af0e 100644
--- a/stage2/PackageBuild.scala
+++ b/stage2/PackageBuild.scala
@@ -20,9 +20,11 @@ abstract class PackageBuild(context: Context) extends BasicBuild(context) with A
 
   private object cacheDocBasicBuild extends Cache[Option[File]]
   def docJar: Option[File] = cacheDocBasicBuild{
-    lib.docJar( scalaVersion, sourceFiles, dependencyClasspath, apiTarget, jarTarget, artifactId, scalaMajorVersion, version, scalacOptions, context.classLoaderCache )
+    lib.docJar(
+      context.cbtHasChanged,
+      scalaVersion, sourceFiles, dependencyClasspath, apiTarget,
+      jarTarget, artifactId, scalaMajorVersion, version,
+      scalacOptions, context.classLoaderCache, context.paths.mavenCache
+    )
   }
-
-  override def jars = jar.toVector ++ dependencyJars
-  override def exportedJars: Seq[File] = jar.toVector
 }
diff --git a/stage2/PublishBuild.scala b/stage2/PublishBuild.scala
index 6b85b22..cc4f5e5 100644
--- a/stage2/PublishBuild.scala
+++ b/stage2/PublishBuild.scala
@@ -1,6 +1,7 @@
 package cbt
 import java.io.File
 import java.net.URL
+import java.nio.file.Files.readAllBytes
 import scala.collection.immutable.Seq
 
 abstract class PublishBuild(context: Context) extends PackageBuild(context){
@@ -37,10 +38,22 @@ abstract class PublishBuild(context: Context) extends PackageBuild(context){
   def snapshotUrl = new URL("https://oss.sonatype.org/content/repositories/snapshots")
   def releaseUrl = new URL("https://oss.sonatype.org/service/local/staging/deploy/maven2")
   override def copy(context: Context) = super.copy(context).asInstanceOf[PublishBuild]
+
+  protected def sonatypeCredentials = {
+    // FIXME: this should probably not use cbtHome, but some reference to the system's host cbt
+    new String(readAllBytes((context.cbtHome ++ "/sonatype.login").toPath)).trim
+  }
+
   def publishSnapshot: Unit = {
     val snapshotBuild = copy( context.copy(version = Some(version+"-SNAPSHOT")) )
     val files = snapshotBuild.pom +: snapshotBuild.`package`
-    lib.publishSnapshot(sourceFiles, files, snapshotUrl ++ releaseFolder )
+    lib.publishSnapshot(
+      sourceFiles, files, snapshotUrl ++ releaseFolder, sonatypeCredentials
+    )
+  }
+  def publishSigned: Unit = {
+    lib.publishSigned(
+      sourceFiles, pom +: `package`, releaseUrl ++ releaseFolder, sonatypeCredentials
+    )
   }
-  def publishSigned: Unit = lib.publishSigned(sourceFiles, pom +: `package`, releaseUrl ++ releaseFolder )
 }
diff --git a/stage2/SbtDependencyDsl.scala b/stage2/SbtDependencyDsl.scala
index 4fd4250..d8c0786 100644
--- a/stage2/SbtDependencyDsl.scala
+++ b/stage2/SbtDependencyDsl.scala
@@ -1,5 +1,5 @@
 package cbt
-trait SbtDependencyDsl{ self: Build =>
+trait SbtDependencyDsl{ self: BasicBuild =>
   /** SBT-like dependency builder DSL for syntax compatibility */
   class DependencyBuilder2( groupId: String, artifactId: String, scalaVersion: Option[String] ){
     def %(version: String) = scalaVersion.map(
@@ -13,7 +13,7 @@ trait SbtDependencyDsl{ self: Build =>
     def  %(artifactId: String) = new DependencyBuilder2( groupId, artifactId, None )
   }
   implicit class DependencyBuilder3(d: MavenDependency){
-    def  %(classifier: String) = d.copy(classifier = Classifier(Some(classifier)))
+    def  %(classifier: String): MavenDependency = d//.copy(classifier = Classifier(Some(classifier)))
   }
 
   /*
diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala
index fa41d79..5316e78 100644
--- a/stage2/Stage2.scala
+++ b/stage2/Stage2.scala
@@ -4,12 +4,23 @@ import java.io._
 
 import scala.collection.immutable.Seq
 
-import cbt.paths._
 
 object Stage2 extends Stage2Base{
+  def getBuild(__context: java.lang.Object, _cbtChanged: java.lang.Boolean) = {
+    val cl1 = __context.getClass.getClassLoader
+    val cl2 = classOf[Context].getClassLoader
+    val _context = __context.asInstanceOf[Context]
+    val context = _context.copy(
+      cbtHasChanged = _context.cbtHasChanged || _cbtChanged
+    )
+    val first = new Lib(context.logger).loadRoot( context )
+    first.finalBuild
+  }
+
   def run( args: Stage2Args ): Unit = {
     import args.logger
-
+    val paths = Paths(args.cbtHome,args.cache)
+    import paths._
     val lib = new Lib(args.logger)
 
     logger.stage2(s"Stage2 start")
@@ -24,11 +35,26 @@ object Stage2 extends Stage2Base{
     }
     val task = args.args.lift( taskIndex )
     
-    val context = Context( args.cwd, args.cwd, args.args.drop( taskIndex ), logger, args.cbtHasChanged, args.classLoaderCache )
+    val context: Context = ContextImplementation(
+      args.cwd,
+      args.cwd,
+      args.args.drop( taskIndex ).toArray,
+      logger.enabledLoggers.toArray,
+      logger.start,
+      args.cbtHasChanged,
+      null,
+      null,
+      args.permanentKeys,
+      args.permanentClassLoaders,
+      args.cache,
+      args.cbtHome,
+      compatibilityTarget,
+      null
+    )
     val first = lib.loadRoot( context )
     val build = first.finalBuild
 
-    def call(build: Build) = {
+    def call(build: BuildInterface) = {
       if(cross){
         build.crossScalaVersions.foreach{
           v => new lib.ReflectBuild(
@@ -57,7 +83,7 @@ object Stage2 extends Stage2Base{
 
           case file if triggerFiles.exists(file.toString startsWith _.toString) =>
             val build = lib.loadDynamic(context)
-            logger.loop(s"Re-running $task for " ++ build.projectDirectory.toString)
+            logger.loop(s"Re-running $task for " ++ build.show)
             call(build)
         }
       } else {
diff --git a/stage2/mixins.scala b/stage2/mixins.scala
index fcffd97..1383324 100644
--- a/stage2/mixins.scala
+++ b/stage2/mixins.scala
@@ -2,31 +2,11 @@ package cbt
 package mixins
 import scala.collection.immutable.Seq
 import java.io._
-trait Test extends Build{
+trait Test extends BasicBuild{
   lazy val testedBuild = BuildDependency( projectDirectory.parent )
   override def dependencies = Seq( testedBuild ) ++ super.dependencies
   override def defaultScalaVersion = testedBuild.build.scalaVersion
 }
-trait Sbt extends Build{
-  override def sources = Seq( projectDirectory ++ "/src/main/scala" )
-}
 trait SbtTest extends Test{
   override def sources = Vector( projectDirectory.parent ++ "/src/test/scala" )
 }
-trait ScalaTest extends Build with Test{
-  def scalaTestVersion: String
-
-  override def dependencies = super.dependencies :+ MavenRepository.central.resolve(
-    "org.scalatest" %% "scalatest" % scalaTestVersion
-  )
-
-  override def run: ExitCode = {
-    val discoveryPath = compile.toString++"/"
-    context.logger.lib("discoveryPath: " ++ discoveryPath)
-    lib.runMain(
-      "org.scalatest.tools.Runner",
-      Seq("-R", discoveryPath, "-oF") ++ context.args.drop(1),
-      classLoader(context.classLoaderCache)
-    )
-  }
-}