Early classloading improvements

- Changed launcher to already load zinc
- use code generation to generate necessary dependencies
- changed resolver to linearize dependency DAG in a way that guarantees that every transitive dependee of a node in the DAG is a transitive dependee of that node in the linear sequence
- move exit code trapping code into java so it can be used for zinc early

There seems to be a bug in this version, where CBT crashes about half of the time with a "object is not an instance of declaring class" Exception during running the task from the build object via reflection.

12 files changed, 394 insertions, 203 deletions

diff --git a/cbt b/cbt
index 372c145..5224744 100755
--- a/cbt
+++ b/cbt
@@ -141,7 +141,7 @@ stage1 () {
 	done
 	compiles=0
 	if [ $changed -eq 1 ]; then
-		rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files
+		#rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files
 		echo "Compiling cbt/nailgun_launcher" 1>&2
 		javac -Xlint:deprecation -Xlint:unchecked -d $NAILGUN$TARGET `ls $NAILGUN*.java`
 		compiles=$?
diff --git a/nailgun_launcher/CBTUrlClassLoader.java b/nailgun_launcher/CBTUrlClassLoader.java
index 72423a0..15440c7 100644
--- a/nailgun_launcher/CBTUrlClassLoader.java
+++ b/nailgun_launcher/CBTUrlClassLoader.java
@@ -13,6 +13,10 @@ class CbtURLClassLoader extends URLClassLoader{
       + "\n)"
     );
   }
+  public Class loadClass(String name) throws ClassNotFoundException{
+    //System.out.println("loadClass("+name+") on \n"+this);
+    return super.loadClass(name);
+  }
   void assertExist(URL[] urls){
     for(URL url: urls){
       if(!new File(url.getPath()).exists()){
diff --git a/nailgun_launcher/Dependency.java b/nailgun_launcher/Dependency.java
deleted file mode 100644
index 93f4785..0000000
--- a/nailgun_launcher/Dependency.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package cbt;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-class EarlyDependency {
-
-  final URL    url;
-  final Path   path;
-  final String hash;
-
-  public EarlyDependency(String folder, String file, String hash) throws MalformedURLException {
-    this.path = Paths.get(NailgunLauncher.CBT_HOME + "/cache/maven/" + folder + "/" + file + ".jar");
-    this.url  = new URL("https://repo1.maven.org/maven2/" + folder + "/" + file + ".jar");
-    this.hash = hash;
-  }
-
-  // scala-lang dependency
-  public static EarlyDependency scala(String scalaModule, String hash) 
-    throws MalformedURLException {
-    return new EarlyDependency(
-      "org/scala-lang/scala-" + scalaModule + "/" + NailgunLauncher.SCALA_VERSION,
-      "scala-" + scalaModule + "-" + NailgunLauncher.SCALA_VERSION,
-      hash
-    );
-  }
-
-}
diff --git a/nailgun_launcher/EarlyDependencies.java b/nailgun_launcher/EarlyDependencies.java
new file mode 100644
index 0000000..1e129c7
--- /dev/null
+++ b/nailgun_launcher/EarlyDependencies.java
@@ -0,0 +1,99 @@
+// This file was auto-generated using `cbt admin cbtEarlyDependencies`
+package cbt;
+import java.io.*;
+import java.nio.file.*;
+import java.net.*;
+import java.security.*;
+import static cbt.NailgunLauncher.*;
+
+class EarlyDependencies{
+
+  /** ClassLoader for stage1 */
+  ClassLoader stage1;
+  /** ClassLoader for zinc */
+  ClassLoader zinc;
+
+  String scalaReflect_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar";
+  String scalaCompiler_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar";
+  String scalaXml_1_0_5_File = MAVEN_CACHE + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar";
+  String scalaLibrary_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar";
+  String zinc_0_3_9_File = MAVEN_CACHE + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar";
+  String incrementalCompiler_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar";
+  String compilerInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar";
+  String scalaCompiler_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar";
+  String sbtInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar";
+  String scalaReflect_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar";
+  String scalaLibrary_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar";
+
+  public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{
+    download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"), Paths.get(scalaReflect_2_11_8_File), "b74530deeba742ab4f3134de0c2da0edc49ca361");
+    download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a");
+
+    // org.scala-lang:scala-library:2.10.5
+    download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"), Paths.get(scalaLibrary_2_10_5_File), "57ac67a6cf6fd591e235c62f8893438e8d10431d");
+    ClassLoader scalaLibrary_2_10_5_ = cachePut(
+      classLoader( scalaLibrary_2_10_5_File ),
+      scalaLibrary_2_10_5_File
+    );
+
+    // org.scala-lang:scala-reflect:2.10.5
+    download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"), Paths.get(scalaReflect_2_10_5_File), "7392facb48876c67a89fcb086112b195f5f6bbc3");
+    ClassLoader scalaReflect_2_10_5_ = cachePut(
+      classLoader( scalaReflect_2_10_5_File, scalaLibrary_2_10_5_ ),
+      scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File
+    );
+
+    // com.typesafe.sbt:sbt-interface:0.13.9
+    download(new URL(MAVEN_URL + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"), Paths.get(sbtInterface_0_13_9_File), "29848631415402c81b732e919be88f268df37250");
+    ClassLoader sbtInterface_0_13_9_ = cachePut(
+      classLoader( sbtInterface_0_13_9_File, scalaReflect_2_10_5_ ),
+      sbtInterface_0_13_9_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File
+    );
+
+    // org.scala-lang:scala-compiler:2.10.5
+    download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"), Paths.get(scalaCompiler_2_10_5_File), "f0f5bb444ca26a6e489af3dd35e24f7e2d2d118e");
+    ClassLoader scalaCompiler_2_10_5_ = cachePut(
+      classLoader( scalaCompiler_2_10_5_File, sbtInterface_0_13_9_ ),
+      sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File
+    );
+
+    // com.typesafe.sbt:compiler-interface:0.13.9
+    download(new URL(MAVEN_URL + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"), Paths.get(compilerInterface_0_13_9_File), "2311addbed1182916ad00f83c57c0eeca1af382b");
+    ClassLoader compilerInterface_0_13_9_ = cachePut(
+      classLoader( compilerInterface_0_13_9_File, scalaCompiler_2_10_5_ ),
+      compilerInterface_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File
+    );
+
+    // com.typesafe.sbt:incremental-compiler:0.13.9
+    download(new URL(MAVEN_URL + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"), Paths.get(incrementalCompiler_0_13_9_File), "fbbf1cadbed058aa226643e83543c35de43b13f0");
+    ClassLoader incrementalCompiler_0_13_9_ = cachePut(
+      classLoader( incrementalCompiler_0_13_9_File, compilerInterface_0_13_9_ ),
+      compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File
+    );
+
+    // com.typesafe.zinc:zinc:0.3.9
+    download(new URL(MAVEN_URL + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"), Paths.get(zinc_0_3_9_File), "46a4556d1f36739879f4b2cc19a73d12b3036e9a");
+    ClassLoader zinc_0_3_9_ = cachePut(
+      classLoader( zinc_0_3_9_File, incrementalCompiler_0_13_9_ ),
+      compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, zinc_0_3_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File
+    );
+
+    // org.scala-lang:scala-library:2.11.8
+    download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573");
+    ClassLoader scalaLibrary_2_11_8_ = cachePut(
+      classLoader( scalaLibrary_2_11_8_File ),
+      scalaLibrary_2_11_8_File
+    );
+
+    // org.scala-lang.modules:scala-xml_2.11:1.0.5
+    download(new URL(MAVEN_URL + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459");
+    ClassLoader scalaXml_1_0_5_ = cachePut(
+      classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ),
+      scalaXml_1_0_5_File, scalaLibrary_2_11_8_File
+    );
+  
+    stage1 = scalaXml_1_0_5_;
+
+    zinc = zinc_0_3_9_;
+  }
+}
diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java
index 9121797..c213518 100644
--- a/nailgun_launcher/NailgunLauncher.java
+++ b/nailgun_launcher/NailgunLauncher.java
@@ -4,6 +4,7 @@ import java.lang.reflect.*;
 import java.net.*;
 import java.nio.*;
 import java.nio.file.*;
+import static java.io.File.pathSeparator;
 import java.security.*;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
@@ -15,6 +16,16 @@ import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
  * dependencies outside the JDK.
  */
 public class NailgunLauncher{
+  public static String SCALA_VERSION = "2.11.8";
+  public static String SCALA_XML_VERSION = "1.0.5";
+  public static String ZINC_VERSION = "0.3.9";
+
+  public static String CBT_HOME = System.getenv("CBT_HOME");
+  public static String NAILGUN = System.getenv("NAILGUN");
+  public static String TARGET = System.getenv("TARGET");
+  public static String STAGE1 = CBT_HOME + "/stage1/";
+  public static String MAVEN_CACHE = CBT_HOME + "/cache/maven";
+  public static String MAVEN_URL = "https://repo1.maven.org/maven2";
 
   /**
    * Persistent cache for caching classloaders for the JVM life time. Can be used as needed by user
@@ -25,18 +36,8 @@ public class NailgunLauncher{
 
   public static SecurityManager defaultSecurityManager = System.getSecurityManager();
 
-  public static String CBT_HOME = System.getenv("CBT_HOME");
-  public static String NAILGUN = System.getenv("NAILGUN");
-  public static String STAGE1 = CBT_HOME + "/stage1/";
-  public static String TARGET = System.getenv("TARGET");
-
-  public static String SCALA_VERSION = "2.11.8";
-
-  public static void _assert(Boolean condition, Object msg){
-    if(!condition){
-      throw new AssertionError("Assertion failed: "+msg);
-    }
-  }
+  public static long lastSuccessfullCompile = 0;
+  static ClassLoader stage1classLoader = null;
 
   public static void main(String[] args) throws ClassNotFoundException,
                                                 NoSuchMethodException,
@@ -45,125 +46,136 @@ public class NailgunLauncher{
                                                 MalformedURLException,
                                                 IOException,
                                                 NoSuchAlgorithmException {
+    long now = System.currentTimeMillis();
+    //System.err.println("ClassLoader: "+stage1classLoader);
+    //System.err.println("lastSuccessfullCompile: "+lastSuccessfullCompile);
+    //System.err.println("now: "+now);
+
     _assert(CBT_HOME != null, CBT_HOME);
     _assert(NAILGUN != null, NAILGUN);
     _assert(TARGET != null, TARGET);
     _assert(STAGE1 != null, STAGE1);
 
-    File f2 = new File(STAGE1);
-    _assert(f2.listFiles() != null, f2);
-    long lastCompiled = new File(STAGE1 + TARGET + "/cbt/Stage1.class").lastModified();
-    for( File file: f2.listFiles() ){
-      if( file.isFile() && file.toString().endsWith(".scala")
-          && file.lastModified() > lastCompiled ){
-
-        EarlyDependency[] dependencies = new EarlyDependency[]{
-          EarlyDependency.scala("library", "DDD5A8BCED249BEDD86FB4578A39B9FB71480573"),
-          EarlyDependency.scala("compiler","FE1285C9F7B58954C5EF6D80B59063569C065E9A"),
-          EarlyDependency.scala("reflect", "B74530DEEBA742AB4F3134DE0C2DA0EDC49CA361"),
-          new EarlyDependency("org/scala-lang/modules/scala-xml_2.11/1.0.5", "scala-xml_2.11-1.0.5", "77ac9be4033768cf03cc04fbd1fc5e5711de2459")
-        };
-
-        ArrayList<String> scalaClassPath = new ArrayList<String>();
-
-        for (EarlyDependency d: dependencies) {
-          download( d.url, d.path, d.hash );
-          scalaClassPath.add( d.path.toString() );
-        }
-
-        File stage1ClassFiles = new File(STAGE1 + TARGET + "/cbt/");
-        if( stage1ClassFiles.exists() ){
-          for( File f: stage1ClassFiles.listFiles() ){
-          if( f.toString().endsWith(".class") ){
-            f.delete();
-          }
-        }
-        }
+    if(args[0].equals("check-alive")){
+      System.exit(33);
+      return;
+    }
 
-        new File(STAGE1 + TARGET).mkdirs();
-
-        String s = File.pathSeparator;
-        ArrayList<String> scalacArgsList = new ArrayList<String>(
-          Arrays.asList(
-            new String[]{
-              "-deprecation", "-feature",
-              "-cp", String.join( s, scalaClassPath.toArray(new String[scalaClassPath.size()])) + s + NAILGUN+TARGET,
-              "-d", STAGE1+TARGET
-            }
-          )
-        );
-
-        for( File f: new File(STAGE1).listFiles() ){
-          if( f.isFile() && f.toString().endsWith(".scala") ){
-            scalacArgsList.add( f.toString() );
-          }
-        }
+    List<File> stage1SourceFiles = new ArrayList<File>();
+    for( File f: new File(STAGE1).listFiles() ){
+      if( f.isFile() && f.toString().endsWith(".scala") ){
+        stage1SourceFiles.add(f);
+      }
+    }
 
-        ArrayList<URL> urls = new ArrayList<URL>();
-        for( String c: scalaClassPath ){
-          urls.add(new URL("file:"+c));
-        }
-        ClassLoader cl = new CbtURLClassLoader( (URL[]) urls.toArray(new URL[urls.size()]) );
-        cl.loadClass("scala.tools.nsc.Main")
-          .getMethod("main", String[].class)
-          .invoke( null/* _cls.newInstance()*/, (Object) scalacArgsList.toArray(new String[scalacArgsList.size()]));
+    Boolean stage1SourcesChanged = false;
+    for( File file: stage1SourceFiles ){
+      if( file.lastModified() > lastSuccessfullCompile ){
+        stage1SourcesChanged = true;
+        //System.err.println("File change: "+file.lastModified());
         break;
       }
     }
 
-    String library = CBT_HOME+"/cache/maven/org/scala-lang/scala-library/"+SCALA_VERSION+"/scala-library-"+SCALA_VERSION+".jar";
-    if(!classLoaderCacheKeys.containsKey(library)){
-      Object libraryKey = new Object();
-      classLoaderCacheKeys.put(library,libraryKey);
-      ClassLoader libraryClassLoader = new CbtURLClassLoader( new URL[]{ new URL("file:"+library) } );
-      classLoaderCacheValues.put(libraryKey, libraryClassLoader);
-
-      String xml = CBT_HOME+"/cache/maven/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar";
-      Object xmlKey = new Object();
-      classLoaderCacheKeys.put(xml,xmlKey);
-      ClassLoader xmlClassLoader = new CbtURLClassLoader(
-        new URL[]{ new URL("file:"+xml) },
-        libraryClassLoader
-      );
-      classLoaderCacheValues.put(xmlKey, xmlClassLoader);
-
-      Object nailgunKey = new Object();
-      classLoaderCacheKeys.put(NAILGUN+TARGET,nailgunKey);
-      ClassLoader nailgunClassLoader = new CbtURLClassLoader(
-        new URL[]{ new URL("file:"+NAILGUN+TARGET) },
-        xmlClassLoader
-      );
-      classLoaderCacheValues.put(nailgunKey, nailgunClassLoader);
+    if(stage1SourcesChanged || stage1classLoader == null){
+      EarlyDependencies earlyDeps = new EarlyDependencies();
+      int exitCode = zinc(earlyDeps, stage1SourceFiles);
+      if( exitCode == 0 ){
+        lastSuccessfullCompile = now;
+      } else {
+        System.exit( exitCode );
+      }
+
+      ClassLoader nailgunClassLoader;
+      if( classLoaderCacheKeys.containsKey( NAILGUN+TARGET ) ){
+        nailgunClassLoader = cacheGet( NAILGUN+TARGET );
+      } else {
+        nailgunClassLoader = cachePut( classLoader(NAILGUN+TARGET, earlyDeps.stage1), NAILGUN+TARGET ); // FIXME: key is wrong here, should be full CP
+      }
+
+      stage1classLoader = classLoader(STAGE1+TARGET, nailgunClassLoader);
+    }
+
+    try{
+      stage1classLoader
+        .loadClass("cbt.Stage1")
+        .getMethod("main", String[].class, ClassLoader.class)
+        .invoke( null, (Object) args, stage1classLoader);
+    }catch(Exception e){
+      System.err.println(stage1classLoader);
+      throw e;
     }
+  }
 
-    if(args[0].equals("check-alive")){
-      System.exit(33);
-      return;
+  public static void _assert(Boolean condition, Object msg){
+    if(!condition){
+      throw new AssertionError("Assertion failed: "+msg);
     }
+  }
+
+  public static void runMain(String cls, String[] args, ClassLoader cl) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{
+    cl.loadClass(cls)
+      .getMethod("main", String[].class)
+      .invoke( null, (Object) args);
+  }
 
-    ClassLoader cl = new CbtURLClassLoader(
-      new URL[]{ new URL("file:"+STAGE1+TARGET) }, 
-      classLoaderCacheValues.get(
-        classLoaderCacheKeys.get( NAILGUN+TARGET )
+  static int zinc( EarlyDependencies earlyDeps, List<File> sourceFiles ) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{
+    String cp = NAILGUN+TARGET + pathSeparator + earlyDeps.scalaXml_1_0_5_File + pathSeparator + earlyDeps.scalaLibrary_2_11_8_File;
+    List<String> zincArgs = new ArrayList<String>(
+      Arrays.asList(
+        new String[]{ 
+          "-scala-compiler", earlyDeps.scalaCompiler_2_11_8_File,
+          "-scala-library", earlyDeps.scalaLibrary_2_11_8_File,
+          "-scala-extra", earlyDeps.scalaReflect_2_11_8_File,
+          "-sbt-interface", earlyDeps.sbtInterface_0_13_9_File,
+          "-compiler-interface", earlyDeps.compilerInterface_0_13_9_File,
+          "-cp", cp,
+          "-d", STAGE1+TARGET
+        }
       )
     );
 
-    try{    
-      cl.loadClass("cbt.Stage1")
-        .getMethod("main", String[].class, ClassLoader.class)
-        .invoke( null/* _cls.newInstance()*/, (Object) args, cl);
-    }catch(ClassNotFoundException e){
-      System.err.println(cl);
-      throw e;
-    }catch(NoClassDefFoundError e){
-      System.err.println(cl);
-      throw e;
-    }catch(InvocationTargetException e){
-      System.err.println(cl);
-      throw e;
+    for( File f: sourceFiles ){
+      zincArgs.add(f.toString());
+    }
+
+    try{
+      System.setSecurityManager( new TrapSecurityManager() );
+      PrintStream oldOut = System.out;
+      System.setOut(System.err);
+      runMain( "com.typesafe.zinc.Main", zincArgs.toArray(new String[zincArgs.size()]), earlyDeps.zinc );
+      System.setOut(oldOut);
+      return 0;//throw new RuntimeException("zinc should have thrown an exit code");
+    }catch( TrappedExitCode trapped ){
+      return trapped.exitCode;
+    } finally {
+      System.setSecurityManager(NailgunLauncher.defaultSecurityManager);
     }
   }
 
+  static ClassLoader classLoader( String file ) throws MalformedURLException{
+    return new CbtURLClassLoader(
+      new URL[]{ new URL("file:"+file) }
+    );
+  }
+  static ClassLoader classLoader( String file, ClassLoader parent ) throws MalformedURLException{
+    return new CbtURLClassLoader(
+      new URL[]{ new URL("file:"+file) }, parent
+    );
+  }
+  static ClassLoader cacheGet( String key ){
+    return classLoaderCacheValues.get(
+      classLoaderCacheKeys.get( key )
+    );
+  }
+  static ClassLoader cachePut( ClassLoader classLoader, String... jars ){
+    String key = String.join( pathSeparator, jars );
+    Object keyObject = new Object();
+    classLoaderCacheKeys.put( key, keyObject );
+    classLoaderCacheValues.put( keyObject, classLoader );
+    return classLoader;
+  }
+
   public static void download(URL urlString, Path target, String sha1) throws IOException, NoSuchAlgorithmException {
     final Path unverified = Paths.get(target+".unverified");
     if(!Files.exists(target)) {
@@ -189,5 +201,3 @@ public class NailgunLauncher{
     return (new HexBinaryAdapter()).marshal(sha1.digest());
   }
 }
-
-
diff --git a/nailgun_launcher/TrapSecurityManager.java b/nailgun_launcher/TrapSecurityManager.java
new file mode 100644
index 0000000..ed00582
--- /dev/null
+++ b/nailgun_launcher/TrapSecurityManager.java
@@ -0,0 +1,19 @@
+package cbt;
+import java.security.*;
+public class TrapSecurityManager extends SecurityManager{
+  public void checkPermission( Permission permission ){
+    /*
+    NOTE: is it actually ok, to just make these empty?
+    Calling .super leads to ClassNotFound exteption for a lambda.
+    Calling to the previous SecurityManager leads to a stack overflow
+    */
+  }
+  public void checkPermission( Permission permission, Object context ){
+    /* Does this methods need to be overidden? */
+  }
+  @Override
+  public void checkExit( int status ){
+    super.checkExit(status);
+    throw new TrappedExitCode(status);
+  }
+}
diff --git a/nailgun_launcher/TrappedExitCode.java b/nailgun_launcher/TrappedExitCode.java
new file mode 100644
index 0000000..154db27
--- /dev/null
+++ b/nailgun_launcher/TrappedExitCode.java
@@ -0,0 +1,8 @@
+package cbt;
+import java.security.*;
+public class TrappedExitCode extends SecurityException{
+  public int exitCode;
+  public TrappedExitCode(int exitCode){
+    this.exitCode = exitCode;
+  }
+}
diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala
index 69fc372..7278fcf 100644
--- a/stage1/Stage1.scala
+++ b/stage1/Stage1.scala
@@ -53,32 +53,29 @@ object Stage1{
     
     val classLoaderCache = new ClassLoaderCache(logger)
 
-    val deps = ClassPath.flatten(
-      Seq(
-        JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
-        JavaDependency("org.scala-lang","scala-reflect",constants.scalaVersion),
-        ScalaDependency(
-          "org.scala-lang.modules", "scala-xml", "1.0.5", scalaVersion=constants.scalaMajorVersion
-        ),
-        JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r")
-      ).map(_.classpath)
+    val deps = Dependencies(
+      JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"),
+      JavaDependency("org.scala-lang","scala-reflect",constants.scalaVersion),
+      JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r")
     )
 
+    val scalaXml = JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion)
+
     logger.stage1("before conditionally running zinc to recompile CBT")
     if( src.exists(newerThan(_, changeIndicator)) ) {
       logger.stage1("cbt.lib has changed. Recompiling.")
-      zinc( true, src, stage2Target, nailgunTarget +: stage1Target +: deps, classLoaderCache, Seq("-deprecation") )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion )
+      zinc( true, src, stage2Target, nailgunTarget +: stage1Target +: Dependencies(deps, scalaXml).classpath, classLoaderCache, Seq("-deprecation") )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion )
     }
     logger.stage1(s"[$now] calling CbtDependency.classLoader")
 
     val cp = stage2Target
     val cl = classLoaderCache.transient.get(
-      (stage2Target +: deps).string,
+      (stage2Target +: deps.classpath).string,
       cbt.URLClassLoader(
         ClassPath(Seq(stage2Target)),
         classLoaderCache.persistent.get(
-          deps.string,
-          cbt.URLClassLoader( deps, classLoader )
+          deps.classpath.string,
+          cbt.URLClassLoader( deps.classpath, classLoader )
         )
       )
     )
diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala
index 0259cb0..c8af672 100644
--- a/stage1/Stage1Lib.scala
+++ b/stage1/Stage1Lib.scala
@@ -20,14 +20,14 @@ object ExitCode{
   val Failure = ExitCode(1)
 }
 
-class TrappedExitCode(private val exitCode: Int) extends Exception
-object TrappedExitCode{
-  def unapply(e: Throwable): Option[ExitCode] =
+object CatchTrappedExitCode{
+  def unapply(e: Throwable): Option[ExitCode] = {
     Option(e) flatMap {
       case i: InvocationTargetException => unapply(i.getTargetException)
       case e: TrappedExitCode => Some( ExitCode(e.exitCode) )
       case _ => None
     }
+  }
 }
 
 case class Context( cwd: File, args: Seq[String], logger: Logger, classLoaderCache: ClassLoaderCache )
@@ -205,30 +205,12 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{
     }
   }
 
-  private val trapSecurityManager = new SecurityManager {
-    override def checkPermission( permission: Permission ) = {
-      /*
-      NOTE: is it actually ok, to just make these empty?
-      Calling .super leads to ClassNotFound exteption for a lambda.
-      Calling to the previous SecurityManager leads to a stack overflow
-      */
-    }
-    override def checkPermission( permission: Permission, context: Any ) = {
-      /* Does this methods need to be overidden? */
-    }
-    override def checkExit( status: Int ) = {
-      super.checkExit(status)
-      logger.lib(s"checkExit($status)")
-      throw new TrappedExitCode(status)
-    }
-  }
-
   def trapExitCode( code: => ExitCode ): ExitCode = {
     try{
-      System.setSecurityManager( trapSecurityManager )
+      System.setSecurityManager( new TrapSecurityManager )
       code
     } catch {
-      case TrappedExitCode(exitCode) =>
+      case CatchTrappedExitCode(exitCode) =>
         exitCode
     } finally {
       System.setSecurityManager(NailgunLauncher.defaultSecurityManager)
diff --git a/stage1/constants.scala b/stage1/constants.scala
index 147e10c..4c39237 100644
--- a/stage1/constants.scala
+++ b/stage1/constants.scala
@@ -1,5 +1,7 @@
 package cbt
 object constants{
+  val scalaXmlVersion = NailgunLauncher.SCALA_XML_VERSION
   val scalaVersion = NailgunLauncher.SCALA_VERSION
+  val zincVersion = NailgunLauncher.ZINC_VERSION
   val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".")
 }
diff --git a/stage1/resolver.scala b/stage1/resolver.scala
index e5cd027..b8e6544 100644
--- a/stage1/resolver.scala
+++ b/stage1/resolver.scala
@@ -154,16 +154,20 @@ abstract class Dependency{
     new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents)))
   }
 
+  def linearize(deps: Seq[Dependency]): Seq[Dependency] =
+    if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) )
+  
   private object transitiveDependenciesCache extends Cache[Seq[Dependency]]
+  /** return dependencies in order of linearized dependence. this is a bit tricky. */
   def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{
-    val deps = (dependencies ++ dependencies.flatMap(_.transitiveDependencies)).distinct
+    val deps = linearize(dependencies)
     val hasInfo = deps.collect{ case d:ArtifactInfo => d }
     val noInfo  = deps.filter{
       case _:ArtifactInfo => false
       case _ => true
     }
-    noInfo ++ JavaDependency.removeOutdated( hasInfo )
-  }.sortBy(_.targetClasspath.string)
+    noInfo ++ JavaDependency.updateOutdated( hasInfo ).reverse.distinct
+  }
 
   def show: String = this.getClass.getSimpleName
   // ========== debug ==========
@@ -172,8 +176,8 @@ abstract class Dependency{
     ( " " * indent )
     ++ (if(updated) lib.red(show) else show)
     ++ dependencies.map(
-      _.dependencyTreeRecursion(indent + 1)
-    ).map( "\n" ++ _.toString ).mkString("")
+      "\n" ++ _.dependencyTreeRecursion(indent + 1)
+    ).mkString
   )
 }
 
@@ -202,6 +206,15 @@ case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implici
   def targetClasspath = exportedClasspath
 }
 
+/** Allows to easily assemble a bunch of dependencies */
+case class Dependencies( _dependencies: Dependency* )(implicit val logger: Logger) extends Dependency{
+  override def dependencies = _dependencies.to
+  def updated = dependencies.exists(_.updated)
+  def exportedClasspath = ClassPath(Seq())
+  def exportedJars = Seq()
+  def targetClasspath = ClassPath(Seq())
+}
+
 case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{
   def updated = false // FIXME: think this through, might allow simplifications and/or optimizations
   override def canBeCached = false
@@ -262,7 +275,7 @@ case class JavaDependency(
   private def jarFile: File = baseFile ++ ".jar"
   //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar"
   private def pomUrl: URL = baseUrl ++ ".pom"
-  private def jarUrl: URL = baseUrl ++ ".jar"    
+  private[cbt] def jarUrl: URL = baseUrl ++ ".jar"    
 
   def exportedJars = Seq( jar )
   def exportedClasspath = ClassPath( exportedJars )
@@ -271,31 +284,27 @@ case class JavaDependency(
   
   def jarSha1 = {
     val file = jarFile ++ ".sha1"
-    scala.util.Try{
-      lib.download( jarUrl ++ ".sha1"  , file, None )
-      // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
-      Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
-    }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible)
+    lib.download( jarUrl ++ ".sha1"  , file, None )
+    // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
+    Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
   }
 
   def pomSha1 = {
     val file = pomFile++".sha1"
-    scala.util.Try{ 
-      lib.download( pomUrl++".sha1" , file, None )
-      // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
-      Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
-    }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible)
+    lib.download( pomUrl++".sha1" , file, None )
+    // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c  jna-3.2.2.pom
+    Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim
   }
 
   private object jarCache extends Cache[File]
   def jar = jarCache{
-    lib.download( jarUrl, jarFile, jarSha1 )
+    lib.download( jarUrl, jarFile, Some(jarSha1) )
     jarFile
   }
   def pomXml = XML.loadFile(pom.toString)
 
   def pom = {
-    lib.download( pomUrl, pomFile, pomSha1 )
+    lib.download( pomUrl, pomFile, Some(pomSha1) )
     pomFile
   }
 
@@ -381,7 +390,7 @@ object JavaDependency{
     case e: NumberFormatException => Right(str)
   }
   /* this obviously should be overridable somehow */
-  def removeOutdated(
+  def updateOutdated(
     deps: Seq[ArtifactInfo],
     versionLessThan: (String, String) => Boolean = semanticVersionLessThan
   )(implicit logger: Logger): Seq[ArtifactInfo] = {
@@ -391,11 +400,11 @@ object JavaDependency{
         _.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) )
          .last
       )
-    deps.flatMap{
+    deps.map{
       d => 
-        val l = latest.get((d.groupId,d.artifactId))
+        val l = latest((d.groupId,d.artifactId))
         if(d != l) logger.resolver("outdated: "++d.show)
         l
-    }.distinct
+    }
   }
 }
diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala
index f825cc1..47fcce3 100644
--- a/stage2/AdminTasks.scala
+++ b/stage2/AdminTasks.scala
@@ -1,6 +1,7 @@
 package cbt
 import scala.collection.immutable.Seq
-import java.io._
+import java.io.{Console=>_,_}
+import java.nio.file._
 class AdminTasks(lib: Lib, args: Array[String], cwd: File){
   implicit val logger: Logger = lib.logger
   def resolve = {
@@ -8,7 +9,7 @@ class AdminTasks(lib: Lib, args: Array[String], cwd: File){
       args(1).split(",").toVector.map{
         d =>
           val v = d.split(":")
-          new JavaDependency(v(0),v(1),v(2))(lib.logger).classpath
+          new JavaDependency(v(0),v(1),v(2)).classpath
       }
     )
   }
@@ -16,7 +17,7 @@ class AdminTasks(lib: Lib, args: Array[String], cwd: File){
     args(1).split(",").toVector.map{
       d =>
         val v = d.split(":")
-        new JavaDependency(v(0),v(1),v(2))(lib.logger).dependencyTree
+        new JavaDependency(v(0),v(1),v(2)).dependencyTree
     }.mkString("\n\n")
   }
   def amm = ammonite
@@ -39,4 +40,93 @@ class AdminTasks(lib: Lib, args: Array[String], cwd: File){
     )
   }
   def scaffoldBasicBuild: Unit = lib.scaffoldBasicBuild( cwd )
+  def cbtEarlyDependencies = {
+    val scalaVersion = args.lift(1).getOrElse(constants.scalaVersion)
+    val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".")
+    val scalaXmlVersion = args.lift(2).getOrElse(constants.scalaXmlVersion)
+    val zincVersion = args.lift(3).getOrElse(constants.zincVersion)
+    /*
+    def tree(d: JavaDependency, indent: Int): String ={
+      val dependencies = {
+        if( d.dependencies.nonEmpty ){
+          d.dependencies.map{
+            case d: JavaDependency => tree(d,indent + 1)
+          }.mkString(",\n" ++ ( "  " * indent ),",\n" ++ ( "  " * indent ), "")
+        } else ""
+      }
+      (
+        s"""new EarlyDependency( "${d.groupId}", "${d.artifactId}", "${d.version}", "${d.jarSha1}"$dependencies)"""
+      )
+    }*/
+    val scalaDeps = Seq(
+      JavaDependency("org.scala-lang","scala-reflect",scalaVersion),
+      JavaDependency("org.scala-lang","scala-compiler",scalaVersion)
+    )
+    
+    val scalaXml = Dependencies(
+      JavaDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion),
+      JavaDependency("org.scala-lang","scala-library",scalaVersion)
+    )
+
+    val zinc = JavaDependency("com.typesafe.zinc","zinc",zincVersion)
+    println(zinc.dependencyTree)
+
+    def valName(dep: JavaDependency) = {
+      val words = dep.artifactId.split("_").head.split("-")
+      words(0) ++ words.drop(1).map(s => s(0).toString.toUpperCase ++ s.drop(1)).mkString ++ "_" ++ dep.version.replace(".","_") ++ "_"
+    }
+
+    def vals(d: JavaDependency) = s"""  """
+
+    def jarVal(dep: JavaDependency) = "_" + valName(dep) +"Jar"
+    def transitive(dep: Dependency) = (dep +: dep.transitiveDependencies.reverse).collect{case d: JavaDependency => d}
+    def codeEach(dep: Dependency) = {    
+      transitive(dep).tails.map(_.reverse).toVector.reverse.drop(1).map{
+        deps =>
+          val d = deps.last
+          val parents = deps.dropRight(1)
+          val parentString = if(parents.isEmpty) ""  else ( ", " ++ valName(parents.last) )
+          val n = valName(d)
+          s"""
+    // ${d.groupId}:${d.artifactId}:${d.version}
+    download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}");
+    ClassLoader $n = cachePut(
+      classLoader( ${n}File$parentString ),
+      ${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")}
+    );"""
+      }
+    }
+    val assignments = codeEach(zinc) ++ codeEach(scalaXml)
+    //{ case (name, dep) => s"$name =\n      ${tree(dep, 4)};" }.mkString("\n\n    ")
+    val code = s"""// This file was auto-generated using `cbt admin cbtEarlyDependencies`
+package cbt;
+import java.io.*;
+import java.nio.file.*;
+import java.net.*;
+import java.security.*;
+import static cbt.NailgunLauncher.*;
+
+class EarlyDependencies{
+
+  /** ClassLoader for stage1 */
+  ClassLoader stage1;
+  /** ClassLoader for zinc */
+  ClassLoader zinc;
+
+${(scalaDeps ++ transitive(scalaXml) ++ transitive(zinc)).map(d => s"""  String ${valName(d)}File = MAVEN_CACHE + "${d.basePath}.jar";""").mkString("\n")}
+
+  public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{
+${scalaDeps.map(d => s"""    download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")}
+${assignments.mkString("\n")}
+  
+    stage1 = scalaXml_${scalaXmlVersion.replace(".","_")}_;
+
+    zinc = zinc_${zincVersion.replace(".","_")}_;
+  }
+}
+"""
+    val file = paths.nailgun ++ ("/" ++ "EarlyDependencies.java")
+    Files.write( file.toPath, code.getBytes )
+    println( Console.GREEN ++ "Wrote " ++ file.string ++ Console.RESET )
+  }
 }