Early classloading improvements

- Changed launcher to already load zinc
- use code generation to generate necessary dependencies
- changed resolver to linearize dependency DAG in a way that guarantees that every transitive dependee of a node in the DAG is a transitive dependee of that node in the linear sequence
- move exit code trapping code into java so it can be used for zinc early

There seems to be a bug in this version, where CBT crashes about half of the time with a "object is not an instance of declaring class" Exception during running the task from the build object via reflection.

4 files changed, 48 insertions, 58 deletions

diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala
index 69fc372..7278fcf 100644
--- a/stage1/Stage1.scala
+++ b/stage1/Stage1.scala
@@ -53,32 +53,29 @@ object Stage1{
     
     val classLoaderCache = new ClassLoaderCache(logger)
 
-    val deps = ClassPath.flatten(
-      Seq(
-        JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
-        JavaDependency("org.scala-lang","scala-reflect",constants.scalaVersion),
-        ScalaDependency(
-          "org.scala-lang.modules", "scala-xml", "1.0.5", scalaVersion=constants.scalaMajorVersion
-        ),
-        JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r")
-      ).map(_.classpath)
+    val deps = Dependencies(
+      JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
+      JavaDependency("org.scala-lang","scala-reflect",constants.scalaVersion),
+      JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r")
     )
 
+    val scalaXml = JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion)
+
     logger.stage1("before conditionally running zinc to recompile CBT")
     if( src.exists(newerThan(_, changeIndicator)) ) {
       logger.stage1("cbt.lib has changed. Recompiling.")
-      zinc( true, src, stage2Target, nailgunTarget +: stage1Target +: deps, classLoaderCache, Seq("-deprecation") )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion )
+      zinc( true, src, stage2Target, nailgunTarget +: stage1Target +: Dependencies(deps, scalaXml).classpath, classLoaderCache, Seq("-deprecation") )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion )
     }
     logger.stage1(s"[$now] calling CbtDependency.classLoader")
 
     val cp = stage2Target
     val cl = classLoaderCache.transient.get(
-      (stage2Target +: deps).string,
+      (stage2Target +: deps.classpath).string,
       cbt.URLClassLoader(
         ClassPath(Seq(stage2Target)),
         classLoaderCache.persistent.get(
-          deps.string,
-          cbt.URLClassLoader( deps, classLoader )
+          deps.classpath.string,
+          cbt.URLClassLoader( deps.classpath, classLoader )
         )
       )
     )
diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala
index 0259cb0..c8af672 100644
--- a/stage1/Stage1Lib.scala
+++ b/stage1/Stage1Lib.scala
@@ -20,14 +20,14 @@ object ExitCode{
   val Failure = ExitCode(1)
 }
 
-class TrappedExitCode(private val exitCode: Int) extends Exception
-object TrappedExitCode{
-  def unapply(e: Throwable): Option[ExitCode] =
+object CatchTrappedExitCode{
+  def unapply(e: Throwable): Option[ExitCode] = {
     Option(e) flatMap {
       case i: InvocationTargetException => unapply(i.getTargetException)
       case e: TrappedExitCode => Some( ExitCode(e.exitCode) )
       case _ => None
     }
+  }
 }
 
 case class Context( cwd: File, args: Seq[String], logger: Logger, classLoaderCache: ClassLoaderCache )
@@ -205,30 +205,12 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{
     }
   }
 
-  private val trapSecurityManager = new SecurityManager {
-    override def checkPermission( permission: Permission ) = {
-      /*
-      NOTE: is it actually ok, to just make these empty?
-      Calling .super leads to ClassNotFound exteption for a lambda.
-      Calling to the previous SecurityManager leads to a stack overflow
-      */
-    }
-    override def checkPermission( permission: Permission, context: Any ) = {
-      /* Does this methods need to be overidden? */
-    }
-    override def checkExit( status: Int ) = {
-      super.checkExit(status)
-      logger.lib(s"checkExit($status)")
-      throw new TrappedExitCode(status)
-    }
-  }
-
   def trapExitCode( code: => ExitCode ): ExitCode = {
     try{
-      System.setSecurityManager( trapSecurityManager )
+      System.setSecurityManager( new TrapSecurityManager )
       code
     } catch {
-      case TrappedExitCode(exitCode) =>
+      case CatchTrappedExitCode(exitCode) =>
         exitCode
     } finally {
       System.setSecurityManager(NailgunLauncher.defaultSecurityManager)
diff --git a/stage1/constants.scala b/stage1/constants.scala
index 147e10c..4c39237 100644
--- a/stage1/constants.scala
+++ b/stage1/constants.scala
@@ -1,5 +1,7 @@
 package cbt
 object constants{
+  val scalaXmlVersion = NailgunLauncher.SCALA_XML_VERSION
   val scalaVersion = NailgunLauncher.SCALA_VERSION
+  val zincVersion = NailgunLauncher.ZINC_VERSION
   val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".")
 }
diff --git a/stage1/resolver.scala b/stage1/resolver.scala
index e5cd027..b8e6544 100644
--- a/stage1/resolver.scala
+++ b/stage1/resolver.scala
@@ -154,16 +154,20 @@ abstract class Dependency{
     new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents)))
   }
 
+  def linearize(deps: Seq[Dependency]): Seq[Dependency] =
+    if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) )
+  
   private object transitiveDependenciesCache extends Cache[Seq[Dependency]]
+  /** return dependencies in order of linearized dependence. this is a bit tricky. */
   def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{
-    val deps = (dependencies ++ dependencies.flatMap(_.transitiveDependencies)).distinct
+    val deps = linearize(dependencies)
     val hasInfo = deps.collect{ case d:ArtifactInfo => d }
     val noInfo  = deps.filter{
       case _:ArtifactInfo => false
       case _ => true
     }
-    noInfo ++ JavaDependency.removeOutdated( hasInfo )
-  }.sortBy(_.targetClasspath.string)
+    noInfo ++ JavaDependency.updateOutdated( hasInfo ).reverse.distinct
+  }
 
   def show: String = this.getClass.getSimpleName
   // ========== debug ==========
@@ -172,8 +176,8 @@ abstract class Dependency{
     ( " " * indent )
     ++ (if(updated) lib.red(show) else show)
     ++ dependencies.map(
-      _.dependencyTreeRecursion(indent + 1)
-    ).map( "\n" ++ _.toString ).mkString("")
+      "\n" ++ _.dependencyTreeRecursion(indent + 1)
+    ).mkString
   )
 }
 
@@ -202,6 +206,15 @@ case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implici
   def targetClasspath = exportedClasspath
 }
 
+/** Allows to easily assemble a bunch of dependencies */
+case class Dependencies( _dependencies: Dependency* )(implicit val logger: Logger) extends Dependency{
+  override def dependencies = _dependencies.to
+  def updated = dependencies.exists(_.updated)
+  def exportedClasspath = ClassPath(Seq())
+  def exportedJars = Seq()
+  def targetClasspath = ClassPath(Seq())
+}
+
 case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{
   def updated = false // FIXME: think this through, might allow simplifications and/or optimizations
   override def canBeCached = false
@@ -262,7 +275,7 @@ case class JavaDependency(
   private def jarFile: File = baseFile ++ ".jar"
   //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar"
   private def pomUrl: URL = baseUrl ++ ".pom"
-  private def jarUrl: URL = baseUrl ++ ".jar"    
+  private[cbt] def jarUrl: URL = baseUrl ++ ".jar"    
 
   def exportedJars = Seq( jar )
   def exportedClasspath = ClassPath( exportedJars )
@@ -271,31 +284,27 @@ case class JavaDependency(
   
   def jarSha1 = {
     val file = jarFile ++ ".sha1"
-    scala.util.Try{
-      lib.download( jarUrl ++ ".sha1"  , file, None )
-      // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
-      Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
-    }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible)
+    lib.download( jarUrl ++ ".sha1"  , file, None )
+    // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
+    Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
   }
 
   def pomSha1 = {
     val file = pomFile++".sha1"
-    scala.util.Try{ 
-      lib.download( pomUrl++".sha1" , file, None )
-      // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
-      Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
-    }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible)
+    lib.download( pomUrl++".sha1" , file, None )
+    // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
+    Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
   }
 
   private object jarCache extends Cache[File]
   def jar = jarCache{
-    lib.download( jarUrl, jarFile, jarSha1 )
+    lib.download( jarUrl, jarFile, Some(jarSha1) )
     jarFile
   }
   def pomXml = XML.loadFile(pom.toString)
 
   def pom = {
-    lib.download( pomUrl, pomFile, pomSha1 )
+    lib.download( pomUrl, pomFile, Some(pomSha1) )
     pomFile
   }
 
@@ -381,7 +390,7 @@ object JavaDependency{
     case e: NumberFormatException => Right(str)
   }
   /* this obviously should be overridable somehow */
-  def removeOutdated(
+  def updateOutdated(
     deps: Seq[ArtifactInfo],
     versionLessThan: (String, String) => Boolean = semanticVersionLessThan
   )(implicit logger: Logger): Seq[ArtifactInfo] = {
@@ -391,11 +400,11 @@ object JavaDependency{
         _.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) )
          .last
       )
-    deps.flatMap{
+    deps.map{
       d => 
-        val l = latest.get((d.groupId,d.artifactId))
+        val l = latest((d.groupId,d.artifactId))
         if(d != l) logger.resolver("outdated: "++d.show)
         l
-    }.distinct
+    }
   }
 }