From bd75b5af0161013b26e2feda9cfcc1e152926071 Mon Sep 17 00:00:00 2001 From: Christopher Vogt Date: Sat, 26 Mar 2016 16:20:50 -0400 Subject: Early classloading improvements - Changed launcher to already load zinc - use code generation to generate necessary dependencies - changed resolver to linearize dependency DAG in a way that guarantees that every transitive dependee of a node in the DAG is a transitive dependee of that node in the linear sequence - move exit code trapping code into java so it can be used for zinc early There seems to be a bug in this version, where CBT crashes about half of the time with a "object is not an instance of declaring class" Exception during running the task from the build object via reflection. --- cbt | 2 +- nailgun_launcher/CBTUrlClassLoader.java | 4 + nailgun_launcher/Dependency.java | 29 ---- nailgun_launcher/EarlyDependencies.java | 99 +++++++++++++ nailgun_launcher/NailgunLauncher.java | 234 ++++++++++++++++-------------- nailgun_launcher/TrapSecurityManager.java | 19 +++ nailgun_launcher/TrappedExitCode.java | 8 + stage1/Stage1.scala | 23 ++- stage1/Stage1Lib.scala | 28 +--- stage1/constants.scala | 2 + stage1/resolver.scala | 53 ++++--- stage2/AdminTasks.scala | 96 +++++++++++- 12 files changed, 394 insertions(+), 203 deletions(-) delete mode 100644 nailgun_launcher/Dependency.java create mode 100644 nailgun_launcher/EarlyDependencies.java create mode 100644 nailgun_launcher/TrapSecurityManager.java create mode 100644 nailgun_launcher/TrappedExitCode.java diff --git a/cbt b/cbt index 372c145..5224744 100755 --- a/cbt +++ b/cbt @@ -141,7 +141,7 @@ stage1 () { done compiles=0 if [ $changed -eq 1 ]; then - rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files + #rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files echo "Compiling cbt/nailgun_launcher" 1>&2 javac -Xlint:deprecation -Xlint:unchecked -d $NAILGUN$TARGET `ls $NAILGUN*.java` compiles=$? diff --git a/nailgun_launcher/CBTUrlClassLoader.java b/nailgun_launcher/CBTUrlClassLoader.java index 72423a0..15440c7 100644 --- a/nailgun_launcher/CBTUrlClassLoader.java +++ b/nailgun_launcher/CBTUrlClassLoader.java @@ -13,6 +13,10 @@ class CbtURLClassLoader extends URLClassLoader{ + "\n)" ); } + public Class loadClass(String name) throws ClassNotFoundException{ + //System.out.println("loadClass("+name+") on \n"+this); + return super.loadClass(name); + } void assertExist(URL[] urls){ for(URL url: urls){ if(!new File(url.getPath()).exists()){ diff --git a/nailgun_launcher/Dependency.java b/nailgun_launcher/Dependency.java deleted file mode 100644 index 93f4785..0000000 --- a/nailgun_launcher/Dependency.java +++ /dev/null @@ -1,29 +0,0 @@ -package cbt; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; - -class EarlyDependency { - - final URL url; - final Path path; - final String hash; - - public EarlyDependency(String folder, String file, String hash) throws MalformedURLException { - this.path = Paths.get(NailgunLauncher.CBT_HOME + "/cache/maven/" + folder + "/" + file + ".jar"); - this.url = new URL("https://repo1.maven.org/maven2/" + folder + "/" + file + ".jar"); - this.hash = hash; - } - - // scala-lang dependency - public static EarlyDependency scala(String scalaModule, String hash) - throws MalformedURLException { - return new EarlyDependency( - "org/scala-lang/scala-" + scalaModule + "/" + NailgunLauncher.SCALA_VERSION, - "scala-" + scalaModule + "-" + NailgunLauncher.SCALA_VERSION, - hash - ); - } - -} diff --git a/nailgun_launcher/EarlyDependencies.java b/nailgun_launcher/EarlyDependencies.java new file mode 100644 index 0000000..1e129c7 --- /dev/null +++ b/nailgun_launcher/EarlyDependencies.java @@ -0,0 +1,99 @@ +// This file was auto-generated using `cbt admin cbtEarlyDependencies` +package cbt; +import java.io.*; +import java.nio.file.*; +import java.net.*; +import java.security.*; +import static cbt.NailgunLauncher.*; + +class EarlyDependencies{ + + /** ClassLoader for stage1 */ + ClassLoader stage1; + /** ClassLoader for zinc */ + ClassLoader zinc; + + String scalaReflect_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; + String scalaCompiler_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; + String scalaXml_1_0_5_File = MAVEN_CACHE + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"; + String scalaLibrary_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"; + String zinc_0_3_9_File = MAVEN_CACHE + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"; + String incrementalCompiler_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"; + String compilerInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"; + String scalaCompiler_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"; + String sbtInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"; + String scalaReflect_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"; + String scalaLibrary_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"; + + public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ + download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"), Paths.get(scalaReflect_2_11_8_File), "b74530deeba742ab4f3134de0c2da0edc49ca361"); + download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); + + // org.scala-lang:scala-library:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"), Paths.get(scalaLibrary_2_10_5_File), "57ac67a6cf6fd591e235c62f8893438e8d10431d"); + ClassLoader scalaLibrary_2_10_5_ = cachePut( + classLoader( scalaLibrary_2_10_5_File ), + scalaLibrary_2_10_5_File + ); + + // org.scala-lang:scala-reflect:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"), Paths.get(scalaReflect_2_10_5_File), "7392facb48876c67a89fcb086112b195f5f6bbc3"); + ClassLoader scalaReflect_2_10_5_ = cachePut( + classLoader( scalaReflect_2_10_5_File, scalaLibrary_2_10_5_ ), + scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:sbt-interface:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"), Paths.get(sbtInterface_0_13_9_File), "29848631415402c81b732e919be88f268df37250"); + ClassLoader sbtInterface_0_13_9_ = cachePut( + classLoader( sbtInterface_0_13_9_File, scalaReflect_2_10_5_ ), + sbtInterface_0_13_9_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // org.scala-lang:scala-compiler:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"), Paths.get(scalaCompiler_2_10_5_File), "f0f5bb444ca26a6e489af3dd35e24f7e2d2d118e"); + ClassLoader scalaCompiler_2_10_5_ = cachePut( + classLoader( scalaCompiler_2_10_5_File, sbtInterface_0_13_9_ ), + sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:compiler-interface:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"), Paths.get(compilerInterface_0_13_9_File), "2311addbed1182916ad00f83c57c0eeca1af382b"); + ClassLoader compilerInterface_0_13_9_ = cachePut( + classLoader( compilerInterface_0_13_9_File, scalaCompiler_2_10_5_ ), + compilerInterface_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:incremental-compiler:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"), Paths.get(incrementalCompiler_0_13_9_File), "fbbf1cadbed058aa226643e83543c35de43b13f0"); + ClassLoader incrementalCompiler_0_13_9_ = cachePut( + classLoader( incrementalCompiler_0_13_9_File, compilerInterface_0_13_9_ ), + compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.zinc:zinc:0.3.9 + download(new URL(MAVEN_URL + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"), Paths.get(zinc_0_3_9_File), "46a4556d1f36739879f4b2cc19a73d12b3036e9a"); + ClassLoader zinc_0_3_9_ = cachePut( + classLoader( zinc_0_3_9_File, incrementalCompiler_0_13_9_ ), + compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, zinc_0_3_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // org.scala-lang:scala-library:2.11.8 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); + ClassLoader scalaLibrary_2_11_8_ = cachePut( + classLoader( scalaLibrary_2_11_8_File ), + scalaLibrary_2_11_8_File + ); + + // org.scala-lang.modules:scala-xml_2.11:1.0.5 + download(new URL(MAVEN_URL + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); + ClassLoader scalaXml_1_0_5_ = cachePut( + classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), + scalaXml_1_0_5_File, scalaLibrary_2_11_8_File + ); + + stage1 = scalaXml_1_0_5_; + + zinc = zinc_0_3_9_; + } +} diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index 9121797..c213518 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -4,6 +4,7 @@ import java.lang.reflect.*; import java.net.*; import java.nio.*; import java.nio.file.*; +import static java.io.File.pathSeparator; import java.security.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -15,6 +16,16 @@ import javax.xml.bind.annotation.adapters.HexBinaryAdapter; * dependencies outside the JDK. */ public class NailgunLauncher{ + public static String SCALA_VERSION = "2.11.8"; + public static String SCALA_XML_VERSION = "1.0.5"; + public static String ZINC_VERSION = "0.3.9"; + + public static String CBT_HOME = System.getenv("CBT_HOME"); + public static String NAILGUN = System.getenv("NAILGUN"); + public static String TARGET = System.getenv("TARGET"); + public static String STAGE1 = CBT_HOME + "/stage1/"; + public static String MAVEN_CACHE = CBT_HOME + "/cache/maven"; + public static String MAVEN_URL = "https://repo1.maven.org/maven2"; /** * Persistent cache for caching classloaders for the JVM life time. Can be used as needed by user @@ -25,18 +36,8 @@ public class NailgunLauncher{ public static SecurityManager defaultSecurityManager = System.getSecurityManager(); - public static String CBT_HOME = System.getenv("CBT_HOME"); - public static String NAILGUN = System.getenv("NAILGUN"); - public static String STAGE1 = CBT_HOME + "/stage1/"; - public static String TARGET = System.getenv("TARGET"); - - public static String SCALA_VERSION = "2.11.8"; - - public static void _assert(Boolean condition, Object msg){ - if(!condition){ - throw new AssertionError("Assertion failed: "+msg); - } - } + public static long lastSuccessfullCompile = 0; + static ClassLoader stage1classLoader = null; public static void main(String[] args) throws ClassNotFoundException, NoSuchMethodException, @@ -45,125 +46,136 @@ public class NailgunLauncher{ MalformedURLException, IOException, NoSuchAlgorithmException { + long now = System.currentTimeMillis(); + //System.err.println("ClassLoader: "+stage1classLoader); + //System.err.println("lastSuccessfullCompile: "+lastSuccessfullCompile); + //System.err.println("now: "+now); + _assert(CBT_HOME != null, CBT_HOME); _assert(NAILGUN != null, NAILGUN); _assert(TARGET != null, TARGET); _assert(STAGE1 != null, STAGE1); - File f2 = new File(STAGE1); - _assert(f2.listFiles() != null, f2); - long lastCompiled = new File(STAGE1 + TARGET + "/cbt/Stage1.class").lastModified(); - for( File file: f2.listFiles() ){ - if( file.isFile() && file.toString().endsWith(".scala") - && file.lastModified() > lastCompiled ){ - - EarlyDependency[] dependencies = new EarlyDependency[]{ - EarlyDependency.scala("library", "DDD5A8BCED249BEDD86FB4578A39B9FB71480573"), - EarlyDependency.scala("compiler","FE1285C9F7B58954C5EF6D80B59063569C065E9A"), - EarlyDependency.scala("reflect", "B74530DEEBA742AB4F3134DE0C2DA0EDC49CA361"), - new EarlyDependency("org/scala-lang/modules/scala-xml_2.11/1.0.5", "scala-xml_2.11-1.0.5", "77ac9be4033768cf03cc04fbd1fc5e5711de2459") - }; - - ArrayList scalaClassPath = new ArrayList(); - - for (EarlyDependency d: dependencies) { - download( d.url, d.path, d.hash ); - scalaClassPath.add( d.path.toString() ); - } - - File stage1ClassFiles = new File(STAGE1 + TARGET + "/cbt/"); - if( stage1ClassFiles.exists() ){ - for( File f: stage1ClassFiles.listFiles() ){ - if( f.toString().endsWith(".class") ){ - f.delete(); - } - } - } + if(args[0].equals("check-alive")){ + System.exit(33); + return; + } - new File(STAGE1 + TARGET).mkdirs(); - - String s = File.pathSeparator; - ArrayList scalacArgsList = new ArrayList( - Arrays.asList( - new String[]{ - "-deprecation", "-feature", - "-cp", String.join( s, scalaClassPath.toArray(new String[scalaClassPath.size()])) + s + NAILGUN+TARGET, - "-d", STAGE1+TARGET - } - ) - ); - - for( File f: new File(STAGE1).listFiles() ){ - if( f.isFile() && f.toString().endsWith(".scala") ){ - scalacArgsList.add( f.toString() ); - } - } + List stage1SourceFiles = new ArrayList(); + for( File f: new File(STAGE1).listFiles() ){ + if( f.isFile() && f.toString().endsWith(".scala") ){ + stage1SourceFiles.add(f); + } + } - ArrayList urls = new ArrayList(); - for( String c: scalaClassPath ){ - urls.add(new URL("file:"+c)); - } - ClassLoader cl = new CbtURLClassLoader( (URL[]) urls.toArray(new URL[urls.size()]) ); - cl.loadClass("scala.tools.nsc.Main") - .getMethod("main", String[].class) - .invoke( null/* _cls.newInstance()*/, (Object) scalacArgsList.toArray(new String[scalacArgsList.size()])); + Boolean stage1SourcesChanged = false; + for( File file: stage1SourceFiles ){ + if( file.lastModified() > lastSuccessfullCompile ){ + stage1SourcesChanged = true; + //System.err.println("File change: "+file.lastModified()); break; } } - String library = CBT_HOME+"/cache/maven/org/scala-lang/scala-library/"+SCALA_VERSION+"/scala-library-"+SCALA_VERSION+".jar"; - if(!classLoaderCacheKeys.containsKey(library)){ - Object libraryKey = new Object(); - classLoaderCacheKeys.put(library,libraryKey); - ClassLoader libraryClassLoader = new CbtURLClassLoader( new URL[]{ new URL("file:"+library) } ); - classLoaderCacheValues.put(libraryKey, libraryClassLoader); - - String xml = CBT_HOME+"/cache/maven/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"; - Object xmlKey = new Object(); - classLoaderCacheKeys.put(xml,xmlKey); - ClassLoader xmlClassLoader = new CbtURLClassLoader( - new URL[]{ new URL("file:"+xml) }, - libraryClassLoader - ); - classLoaderCacheValues.put(xmlKey, xmlClassLoader); - - Object nailgunKey = new Object(); - classLoaderCacheKeys.put(NAILGUN+TARGET,nailgunKey); - ClassLoader nailgunClassLoader = new CbtURLClassLoader( - new URL[]{ new URL("file:"+NAILGUN+TARGET) }, - xmlClassLoader - ); - classLoaderCacheValues.put(nailgunKey, nailgunClassLoader); + if(stage1SourcesChanged || stage1classLoader == null){ + EarlyDependencies earlyDeps = new EarlyDependencies(); + int exitCode = zinc(earlyDeps, stage1SourceFiles); + if( exitCode == 0 ){ + lastSuccessfullCompile = now; + } else { + System.exit( exitCode ); + } + + ClassLoader nailgunClassLoader; + if( classLoaderCacheKeys.containsKey( NAILGUN+TARGET ) ){ + nailgunClassLoader = cacheGet( NAILGUN+TARGET ); + } else { + nailgunClassLoader = cachePut( classLoader(NAILGUN+TARGET, earlyDeps.stage1), NAILGUN+TARGET ); // FIXME: key is wrong here, should be full CP + } + + stage1classLoader = classLoader(STAGE1+TARGET, nailgunClassLoader); + } + + try{ + stage1classLoader + .loadClass("cbt.Stage1") + .getMethod("main", String[].class, ClassLoader.class) + .invoke( null, (Object) args, stage1classLoader); + }catch(Exception e){ + System.err.println(stage1classLoader); + throw e; } + } - if(args[0].equals("check-alive")){ - System.exit(33); - return; + public static void _assert(Boolean condition, Object msg){ + if(!condition){ + throw new AssertionError("Assertion failed: "+msg); } + } + + public static void runMain(String cls, String[] args, ClassLoader cl) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ + cl.loadClass(cls) + .getMethod("main", String[].class) + .invoke( null, (Object) args); + } - ClassLoader cl = new CbtURLClassLoader( - new URL[]{ new URL("file:"+STAGE1+TARGET) }, - classLoaderCacheValues.get( - classLoaderCacheKeys.get( NAILGUN+TARGET ) + static int zinc( EarlyDependencies earlyDeps, List sourceFiles ) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ + String cp = NAILGUN+TARGET + pathSeparator + earlyDeps.scalaXml_1_0_5_File + pathSeparator + earlyDeps.scalaLibrary_2_11_8_File; + List zincArgs = new ArrayList( + Arrays.asList( + new String[]{ + "-scala-compiler", earlyDeps.scalaCompiler_2_11_8_File, + "-scala-library", earlyDeps.scalaLibrary_2_11_8_File, + "-scala-extra", earlyDeps.scalaReflect_2_11_8_File, + "-sbt-interface", earlyDeps.sbtInterface_0_13_9_File, + "-compiler-interface", earlyDeps.compilerInterface_0_13_9_File, + "-cp", cp, + "-d", STAGE1+TARGET + } ) ); - try{ - cl.loadClass("cbt.Stage1") - .getMethod("main", String[].class, ClassLoader.class) - .invoke( null/* _cls.newInstance()*/, (Object) args, cl); - }catch(ClassNotFoundException e){ - System.err.println(cl); - throw e; - }catch(NoClassDefFoundError e){ - System.err.println(cl); - throw e; - }catch(InvocationTargetException e){ - System.err.println(cl); - throw e; + for( File f: sourceFiles ){ + zincArgs.add(f.toString()); + } + + try{ + System.setSecurityManager( new TrapSecurityManager() ); + PrintStream oldOut = System.out; + System.setOut(System.err); + runMain( "com.typesafe.zinc.Main", zincArgs.toArray(new String[zincArgs.size()]), earlyDeps.zinc ); + System.setOut(oldOut); + return 0;//throw new RuntimeException("zinc should have thrown an exit code"); + }catch( TrappedExitCode trapped ){ + return trapped.exitCode; + } finally { + System.setSecurityManager(NailgunLauncher.defaultSecurityManager); } } + static ClassLoader classLoader( String file ) throws MalformedURLException{ + return new CbtURLClassLoader( + new URL[]{ new URL("file:"+file) } + ); + } + static ClassLoader classLoader( String file, ClassLoader parent ) throws MalformedURLException{ + return new CbtURLClassLoader( + new URL[]{ new URL("file:"+file) }, parent + ); + } + static ClassLoader cacheGet( String key ){ + return classLoaderCacheValues.get( + classLoaderCacheKeys.get( key ) + ); + } + static ClassLoader cachePut( ClassLoader classLoader, String... jars ){ + String key = String.join( pathSeparator, jars ); + Object keyObject = new Object(); + classLoaderCacheKeys.put( key, keyObject ); + classLoaderCacheValues.put( keyObject, classLoader ); + return classLoader; + } + public static void download(URL urlString, Path target, String sha1) throws IOException, NoSuchAlgorithmException { final Path unverified = Paths.get(target+".unverified"); if(!Files.exists(target)) { @@ -189,5 +201,3 @@ public class NailgunLauncher{ return (new HexBinaryAdapter()).marshal(sha1.digest()); } } - - diff --git a/nailgun_launcher/TrapSecurityManager.java b/nailgun_launcher/TrapSecurityManager.java new file mode 100644 index 0000000..ed00582 --- /dev/null +++ b/nailgun_launcher/TrapSecurityManager.java @@ -0,0 +1,19 @@ +package cbt; +import java.security.*; +public class TrapSecurityManager extends SecurityManager{ + public void checkPermission( Permission permission ){ + /* + NOTE: is it actually ok, to just make these empty? + Calling .super leads to ClassNotFound exteption for a lambda. + Calling to the previous SecurityManager leads to a stack overflow + */ + } + public void checkPermission( Permission permission, Object context ){ + /* Does this methods need to be overidden? */ + } + @Override + public void checkExit( int status ){ + super.checkExit(status); + throw new TrappedExitCode(status); + } +} diff --git a/nailgun_launcher/TrappedExitCode.java b/nailgun_launcher/TrappedExitCode.java new file mode 100644 index 0000000..154db27 --- /dev/null +++ b/nailgun_launcher/TrappedExitCode.java @@ -0,0 +1,8 @@ +package cbt; +import java.security.*; +public class TrappedExitCode extends SecurityException{ + public int exitCode; + public TrappedExitCode(int exitCode){ + this.exitCode = exitCode; + } +} diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 69fc372..7278fcf 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -53,32 +53,29 @@ object Stage1{ val classLoaderCache = new ClassLoaderCache(logger) - val deps = ClassPath.flatten( - Seq( - JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - JavaDependency("org.scala-lang","scala-reflect",constants.scalaVersion), - ScalaDependency( - "org.scala-lang.modules", "scala-xml", "1.0.5", scalaVersion=constants.scalaMajorVersion - ), - JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") - ).map(_.classpath) + val deps = Dependencies( + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + JavaDependency("org.scala-lang","scala-reflect",constants.scalaVersion), + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") ) + val scalaXml = JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) + logger.stage1("before conditionally running zinc to recompile CBT") if( src.exists(newerThan(_, changeIndicator)) ) { logger.stage1("cbt.lib has changed. Recompiling.") - zinc( true, src, stage2Target, nailgunTarget +: stage1Target +: deps, classLoaderCache, Seq("-deprecation") )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) + zinc( true, src, stage2Target, nailgunTarget +: stage1Target +: Dependencies(deps, scalaXml).classpath, classLoaderCache, Seq("-deprecation") )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) } logger.stage1(s"[$now] calling CbtDependency.classLoader") val cp = stage2Target val cl = classLoaderCache.transient.get( - (stage2Target +: deps).string, + (stage2Target +: deps.classpath).string, cbt.URLClassLoader( ClassPath(Seq(stage2Target)), classLoaderCache.persistent.get( - deps.string, - cbt.URLClassLoader( deps, classLoader ) + deps.classpath.string, + cbt.URLClassLoader( deps.classpath, classLoader ) ) ) ) diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 0259cb0..c8af672 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -20,14 +20,14 @@ object ExitCode{ val Failure = ExitCode(1) } -class TrappedExitCode(private val exitCode: Int) extends Exception -object TrappedExitCode{ - def unapply(e: Throwable): Option[ExitCode] = +object CatchTrappedExitCode{ + def unapply(e: Throwable): Option[ExitCode] = { Option(e) flatMap { case i: InvocationTargetException => unapply(i.getTargetException) case e: TrappedExitCode => Some( ExitCode(e.exitCode) ) case _ => None } + } } case class Context( cwd: File, args: Seq[String], logger: Logger, classLoaderCache: ClassLoaderCache ) @@ -205,30 +205,12 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } } - private val trapSecurityManager = new SecurityManager { - override def checkPermission( permission: Permission ) = { - /* - NOTE: is it actually ok, to just make these empty? - Calling .super leads to ClassNotFound exteption for a lambda. - Calling to the previous SecurityManager leads to a stack overflow - */ - } - override def checkPermission( permission: Permission, context: Any ) = { - /* Does this methods need to be overidden? */ - } - override def checkExit( status: Int ) = { - super.checkExit(status) - logger.lib(s"checkExit($status)") - throw new TrappedExitCode(status) - } - } - def trapExitCode( code: => ExitCode ): ExitCode = { try{ - System.setSecurityManager( trapSecurityManager ) + System.setSecurityManager( new TrapSecurityManager ) code } catch { - case TrappedExitCode(exitCode) => + case CatchTrappedExitCode(exitCode) => exitCode } finally { System.setSecurityManager(NailgunLauncher.defaultSecurityManager) diff --git a/stage1/constants.scala b/stage1/constants.scala index 147e10c..4c39237 100644 --- a/stage1/constants.scala +++ b/stage1/constants.scala @@ -1,5 +1,7 @@ package cbt object constants{ + val scalaXmlVersion = NailgunLauncher.SCALA_XML_VERSION val scalaVersion = NailgunLauncher.SCALA_VERSION + val zincVersion = NailgunLauncher.ZINC_VERSION val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") } diff --git a/stage1/resolver.scala b/stage1/resolver.scala index e5cd027..b8e6544 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -154,16 +154,20 @@ abstract class Dependency{ new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents))) } + def linearize(deps: Seq[Dependency]): Seq[Dependency] = + if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) ) + private object transitiveDependenciesCache extends Cache[Seq[Dependency]] + /** return dependencies in order of linearized dependence. this is a bit tricky. */ def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ - val deps = (dependencies ++ dependencies.flatMap(_.transitiveDependencies)).distinct + val deps = linearize(dependencies) val hasInfo = deps.collect{ case d:ArtifactInfo => d } val noInfo = deps.filter{ case _:ArtifactInfo => false case _ => true } - noInfo ++ JavaDependency.removeOutdated( hasInfo ) - }.sortBy(_.targetClasspath.string) + noInfo ++ JavaDependency.updateOutdated( hasInfo ).reverse.distinct + } def show: String = this.getClass.getSimpleName // ========== debug ========== @@ -172,8 +176,8 @@ abstract class Dependency{ ( " " * indent ) ++ (if(updated) lib.red(show) else show) ++ dependencies.map( - _.dependencyTreeRecursion(indent + 1) - ).map( "\n" ++ _.toString ).mkString("") + "\n" ++ _.dependencyTreeRecursion(indent + 1) + ).mkString ) } @@ -202,6 +206,15 @@ case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implici def targetClasspath = exportedClasspath } +/** Allows to easily assemble a bunch of dependencies */ +case class Dependencies( _dependencies: Dependency* )(implicit val logger: Logger) extends Dependency{ + override def dependencies = _dependencies.to + def updated = dependencies.exists(_.updated) + def exportedClasspath = ClassPath(Seq()) + def exportedJars = Seq() + def targetClasspath = ClassPath(Seq()) +} + case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{ def updated = false // FIXME: think this through, might allow simplifications and/or optimizations override def canBeCached = false @@ -262,7 +275,7 @@ case class JavaDependency( private def jarFile: File = baseFile ++ ".jar" //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" private def pomUrl: URL = baseUrl ++ ".pom" - private def jarUrl: URL = baseUrl ++ ".jar" + private[cbt] def jarUrl: URL = baseUrl ++ ".jar" def exportedJars = Seq( jar ) def exportedClasspath = ClassPath( exportedJars ) @@ -271,31 +284,27 @@ case class JavaDependency( def jarSha1 = { val file = jarFile ++ ".sha1" - scala.util.Try{ - lib.download( jarUrl ++ ".sha1" , file, None ) - // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim - }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible) + lib.download( jarUrl ++ ".sha1" , file, None ) + // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom + Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim } def pomSha1 = { val file = pomFile++".sha1" - scala.util.Try{ - lib.download( pomUrl++".sha1" , file, None ) - // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim - }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible) + lib.download( pomUrl++".sha1" , file, None ) + // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom + Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim } private object jarCache extends Cache[File] def jar = jarCache{ - lib.download( jarUrl, jarFile, jarSha1 ) + lib.download( jarUrl, jarFile, Some(jarSha1) ) jarFile } def pomXml = XML.loadFile(pom.toString) def pom = { - lib.download( pomUrl, pomFile, pomSha1 ) + lib.download( pomUrl, pomFile, Some(pomSha1) ) pomFile } @@ -381,7 +390,7 @@ object JavaDependency{ case e: NumberFormatException => Right(str) } /* this obviously should be overridable somehow */ - def removeOutdated( + def updateOutdated( deps: Seq[ArtifactInfo], versionLessThan: (String, String) => Boolean = semanticVersionLessThan )(implicit logger: Logger): Seq[ArtifactInfo] = { @@ -391,11 +400,11 @@ object JavaDependency{ _.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) ) .last ) - deps.flatMap{ + deps.map{ d => - val l = latest.get((d.groupId,d.artifactId)) + val l = latest((d.groupId,d.artifactId)) if(d != l) logger.resolver("outdated: "++d.show) l - }.distinct + } } } diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala index f825cc1..47fcce3 100644 --- a/stage2/AdminTasks.scala +++ b/stage2/AdminTasks.scala @@ -1,6 +1,7 @@ package cbt import scala.collection.immutable.Seq -import java.io._ +import java.io.{Console=>_,_} +import java.nio.file._ class AdminTasks(lib: Lib, args: Array[String], cwd: File){ implicit val logger: Logger = lib.logger def resolve = { @@ -8,7 +9,7 @@ class AdminTasks(lib: Lib, args: Array[String], cwd: File){ args(1).split(",").toVector.map{ d => val v = d.split(":") - new JavaDependency(v(0),v(1),v(2))(lib.logger).classpath + new JavaDependency(v(0),v(1),v(2)).classpath } ) } @@ -16,7 +17,7 @@ class AdminTasks(lib: Lib, args: Array[String], cwd: File){ args(1).split(",").toVector.map{ d => val v = d.split(":") - new JavaDependency(v(0),v(1),v(2))(lib.logger).dependencyTree + new JavaDependency(v(0),v(1),v(2)).dependencyTree }.mkString("\n\n") } def amm = ammonite @@ -39,4 +40,93 @@ class AdminTasks(lib: Lib, args: Array[String], cwd: File){ ) } def scaffoldBasicBuild: Unit = lib.scaffoldBasicBuild( cwd ) + def cbtEarlyDependencies = { + val scalaVersion = args.lift(1).getOrElse(constants.scalaVersion) + val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") + val scalaXmlVersion = args.lift(2).getOrElse(constants.scalaXmlVersion) + val zincVersion = args.lift(3).getOrElse(constants.zincVersion) + /* + def tree(d: JavaDependency, indent: Int): String ={ + val dependencies = { + if( d.dependencies.nonEmpty ){ + d.dependencies.map{ + case d: JavaDependency => tree(d,indent + 1) + }.mkString(",\n" ++ ( " " * indent ),",\n" ++ ( " " * indent ), "") + } else "" + } + ( + s"""new EarlyDependency( "${d.groupId}", "${d.artifactId}", "${d.version}", "${d.jarSha1}"$dependencies)""" + ) + }*/ + val scalaDeps = Seq( + JavaDependency("org.scala-lang","scala-reflect",scalaVersion), + JavaDependency("org.scala-lang","scala-compiler",scalaVersion) + ) + + val scalaXml = Dependencies( + JavaDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion), + JavaDependency("org.scala-lang","scala-library",scalaVersion) + ) + + val zinc = JavaDependency("com.typesafe.zinc","zinc",zincVersion) + println(zinc.dependencyTree) + + def valName(dep: JavaDependency) = { + val words = dep.artifactId.split("_").head.split("-") + words(0) ++ words.drop(1).map(s => s(0).toString.toUpperCase ++ s.drop(1)).mkString ++ "_" ++ dep.version.replace(".","_") ++ "_" + } + + def vals(d: JavaDependency) = s""" """ + + def jarVal(dep: JavaDependency) = "_" + valName(dep) +"Jar" + def transitive(dep: Dependency) = (dep +: dep.transitiveDependencies.reverse).collect{case d: JavaDependency => d} + def codeEach(dep: Dependency) = { + transitive(dep).tails.map(_.reverse).toVector.reverse.drop(1).map{ + deps => + val d = deps.last + val parents = deps.dropRight(1) + val parentString = if(parents.isEmpty) "" else ( ", " ++ valName(parents.last) ) + val n = valName(d) + s""" + // ${d.groupId}:${d.artifactId}:${d.version} + download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}"); + ClassLoader $n = cachePut( + classLoader( ${n}File$parentString ), + ${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")} + );""" + } + } + val assignments = codeEach(zinc) ++ codeEach(scalaXml) + //{ case (name, dep) => s"$name =\n ${tree(dep, 4)};" }.mkString("\n\n ") + val code = s"""// This file was auto-generated using `cbt admin cbtEarlyDependencies` +package cbt; +import java.io.*; +import java.nio.file.*; +import java.net.*; +import java.security.*; +import static cbt.NailgunLauncher.*; + +class EarlyDependencies{ + + /** ClassLoader for stage1 */ + ClassLoader stage1; + /** ClassLoader for zinc */ + ClassLoader zinc; + +${(scalaDeps ++ transitive(scalaXml) ++ transitive(zinc)).map(d => s""" String ${valName(d)}File = MAVEN_CACHE + "${d.basePath}.jar";""").mkString("\n")} + + public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ +${scalaDeps.map(d => s""" download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} +${assignments.mkString("\n")} + + stage1 = scalaXml_${scalaXmlVersion.replace(".","_")}_; + + zinc = zinc_${zincVersion.replace(".","_")}_; + } +} +""" + val file = paths.nailgun ++ ("/" ++ "EarlyDependencies.java") + Files.write( file.toPath, code.getBytes ) + println( Console.GREEN ++ "Wrote " ++ file.string ++ Console.RESET ) + } } -- cgit v1.2.3