diff options
37 files changed, 1292 insertions, 714 deletions
diff --git a/bootstrap_scala/BootstrapScala.java b/bootstrap_scala/BootstrapScala.java deleted file mode 100644 index e2d7a5a..0000000 --- a/bootstrap_scala/BootstrapScala.java +++ /dev/null @@ -1,85 +0,0 @@ -import java.io.File; -import java.io.InputStream; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import java.util.Iterator; -import javax.xml.bind.annotation.adapters.HexBinaryAdapter; - -/** - * This file class allows bootstrapping out of Java into Scala. It downloads the Scala jars for the - * version number given as the first argument into the directory given as the second argument and - * returns a classpath String. - */ -public class BootstrapScala { - - public final static Dependency[] dependencies(String target, String scalaVersion) throws MalformedURLException { - return new Dependency[] { - Dependency.scala(target, scalaVersion, "library", "DDD5A8BCED249BEDD86FB4578A39B9FB71480573"), - Dependency.scala(target, scalaVersion, "compiler","FE1285C9F7B58954C5EF6D80B59063569C065E9A"), - Dependency.scala(target, scalaVersion, "reflect", "B74530DEEBA742AB4F3134DE0C2DA0EDC49CA361"), - new Dependency(target, "modules/scala-xml_2.11/1.0.5", "scala-xml_2.11-1.0.5", "77ac9be4033768cf03cc04fbd1fc5e5711de2459") - }; - } - - public static void main(String args[]) throws IOException, NoSuchAlgorithmException { - - if(args.length < 2){ - System.err.println("Usage: bootstrap_scala <scala version> <download directory>"); - System.exit(1); - } - - Dependency[] ds = dependencies( args[1], args[0] ); - new File(args[1]).mkdirs(); - for (Dependency d: ds) { - download( d.url, d.path, d.hash ); - } - - // Join dep. paths as a classpath - String classpath = ""; - Iterator<Dependency> depsIter = Arrays.asList(ds).iterator(); - while (depsIter.hasNext()) { - Dependency dep = depsIter.next(); - classpath += dep.path.toString(); - if (depsIter.hasNext()) { - classpath += File.pathSeparator; - } - } - - System.out.println(classpath); - - } - - public static void download(URL urlString, Path target, String sha1) throws IOException, NoSuchAlgorithmException { - final Path unverified = Paths.get(target+".unverified"); - if(!Files.exists(target)) { - new File(target.toString()).getParentFile().mkdirs(); - System.err.println("downloading " + urlString); - System.err.println("to " + target); - final InputStream stream = urlString.openStream(); - Files.copy(stream, unverified, StandardCopyOption.REPLACE_EXISTING); - stream.close(); - final String checksum = sha1(Files.readAllBytes(unverified)); - if(sha1 == null || sha1.toUpperCase().equals(checksum)) { - Files.move(unverified, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); - } else { - System.err.println(target + " checksum does not match.\nExpected: |" + sha1 + "|\nFound: |" + checksum + "|"); - System.exit(1); - } - } - } - - public static String sha1(byte[] bytes) throws NoSuchAlgorithmException { - final MessageDigest sha1 = MessageDigest.getInstance("SHA1"); - sha1.update(bytes, 0, bytes.length); - return (new HexBinaryAdapter()).marshal(sha1.digest()); - } - -} diff --git a/bootstrap_scala/Dependency.java b/bootstrap_scala/Dependency.java deleted file mode 100644 index 571047b..0000000 --- a/bootstrap_scala/Dependency.java +++ /dev/null @@ -1,29 +0,0 @@ -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; - -class Dependency { - - final URL url; - final Path path; - final String hash; - - public Dependency(String target, String folder, String file, String hash) throws MalformedURLException { - this.path = Paths.get(target + file + ".jar"); - this.url = new URL("https://repo1.maven.org/maven2/org/scala-lang/" + folder + "/" + file + ".jar"); - this.hash = hash; - } - - // scala-lang dependency - public static Dependency scala(String target, String scalaVersion, String scalaModule, String hash) - throws MalformedURLException { - return new Dependency( - target, - "scala-" + scalaModule + "/" + scalaVersion, - "scala-" + scalaModule + "-" + scalaVersion, - hash - ); - } - -} diff --git a/bootstrap_scala/bootstrap_scala b/bootstrap_scala/bootstrap_scala deleted file mode 100755 index b004c8d..0000000 --- a/bootstrap_scala/bootstrap_scala +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -_DIR=$(dirname $(readlink "$0") 2>/dev/null || dirname "$0" 2>/dev/null ) -DIR=$(dirname $($_DIR/../realpath/realpath.sh $0)) -JAVAC="javac -Xlint:deprecation" -TARGET=$DIR/target -CLASSES=$TARGET/classes/ -VERSION=$1 -CACHE=$DIR/cache/$VERSION/ - -COMPILER_JAR=scala-compiler-$VERSION.jar -LIBRARY_JAR=scala-library-$VERSION.jar -REFLECT_JAR=scala-reflect-$VERSION.jar -XML_JAR=scala-xml_2.11-1.0.5.jar # this is a bit fishy, because it doesn't take version into account - -mkdir -p $CLASSES - -if [ ! -f $CACHE$COMPILER_JAR ] || [ ! -f $CACHE$LIBRARY_JAR ] || [ ! -f $CACHE$REFLECT_JAR ]\ - || [ ! -f $CACHE$XML_JAR ] || [ $DIR/BootstrapScala.java -nt $CLASSES/BootstrapScala.class ] || [ $DIR/Dependency.java -nt $CLASSES/Dependency.class ] -then - echo "Compiling cbt/bootstrap_scala" 1>&2 - $JAVAC -d $CLASSES $DIR/BootstrapScala.java $DIR/Dependency.java - java -cp $CLASSES BootstrapScala $1 $CACHE -else - # for speedup - echo `for f in $CACHE*; do printf "$f "; done`|tr " " ":" -fi diff --git a/build/build.scala b/build/build.scala new file mode 100644 index 0000000..aa5d27a --- /dev/null +++ b/build/build.scala @@ -0,0 +1,18 @@ +import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: Context) extends BasicBuild(context){ + // FIXME: somehow consolidate this with cbt's own boot-strapping from source. + override def dependencies = super.dependencies ++ Seq( + JavaDependency("org.scala-lang","scala-library",constants.scalaVersion), + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), + JavaDependency("com.typesafe.zinc","zinc","0.3.9"), + ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5") + ) + override def sources = Seq( + "nailgun_launcher", "stage1", "stage2" + ).map(d => projectDirectory ++ ("/" + d)) +} @@ -78,12 +78,9 @@ log "Find out real path. Build realpath if needed." $* export CBT_HOME=$(dirname $($_DIR/realpath/realpath.sh $0)) -export SCALA_VERSION="2.11.8" export NAILGUN=$CBT_HOME/nailgun_launcher/ -export STAGE1=$CBT_HOME/stage1/ export TARGET=target/scala-2.11/classes/ mkdir -p $NAILGUN$TARGET -mkdir -p $STAGE1$TARGET nailgun_out=$NAILGUN/target/nailgun.stdout.log nailgun_err=$NAILGUN/target/nailgun.strerr.log @@ -135,18 +132,6 @@ if [ $use_nailgun -eq 0 ] && [ ! $server_up -eq 0 ]; then ng-server 127.0.0.1:$NAILGUN_PORT >> $nailgun_out 2>> $nailgun_err & fi -log "Downloading Scala jars if necessary..." $* -export SCALA_CLASSPATH=`$CBT_HOME/bootstrap_scala/bootstrap_scala $SCALA_VERSION` -if [ ! $? -eq 0 ]; then echo "Problem with bootstrap_scala" 1>&2; exit 1; fi - -SCALAC="java -Xmx256M -Xms32M\ - -Xbootclasspath/a:$SCALA_CLASSPATH\ - -Dscala.usejavacp=true\ - -Denv.emacs=\ - scala.tools.nsc.Main\ - -deprecation\ - -feature" - stage1 () { log "Checking for changes in cbt/nailgun_launcher" $* NAILGUN_INDICATOR=$NAILGUN$TARGET/cbt/NailgunLauncher.class @@ -156,9 +141,9 @@ stage1 () { done compiles=0 if [ $changed -eq 1 ]; then - rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files + #rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files echo "Compiling cbt/nailgun_launcher" 1>&2 - javac -Xlint:deprecation -d $NAILGUN$TARGET `ls $NAILGUN*.java` + javac -Xlint:deprecation -Xlint:unchecked -d $NAILGUN$TARGET `ls $NAILGUN*.java` compiles=$? if [ $compiles -ne 0 ]; then rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # triggers recompilation next time. @@ -172,45 +157,20 @@ stage1 () { fi fi - log "Checking for changes in cbt/stage1" $* - STAGE1_INDICATOR=$STAGE1$TARGET/cbt/Stage1.class - changed2=0 - for file in `ls $STAGE1*.scala`; do - if [ $file -nt $STAGE1_INDICATOR ]; then changed2=1; fi - done - compiles2=0 - - if [ $changed2 -eq 1 ]; then - rm $STAGE1$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files - echo "Compiling cbt/stage1" 1>&2 - $SCALAC -cp $NAILGUN$TARGET -d $STAGE1$TARGET `ls $STAGE1/*.scala` - compiles2=$? - if [ $compiles2 -ne 0 ]; then - rm $STAGE1$TARGET/cbt/*.class 2>/dev/null # triggers recompilation next time. - break - fi - fi - log "run CBT and loop if desired. This allows recompiling CBT itself as part of compile looping." $* - if [ "$1" = "admin" ] || [ "$2" = "admin" ]; then - mainClass=cbt.AdminStage1 - else - mainClass=cbt.Stage1 - fi - - CP=$STAGE1$TARGET:$SCALA_CLASSPATH + if [ $use_nailgun -eq 1 ] then log "Running JVM directly" $* # -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:5005 - java -cp $NAILGUN$TARGET cbt.NailgunLauncher $mainClass $CP "$CWD" $* + java -cp $NAILGUN$TARGET cbt.NailgunLauncher "$CWD" $* else log "Running via nailgun." $* for i in 0 1 2 3 4 5 6 7 8 9; do log "Adding classpath." $* $NG ng-cp $NAILGUN$TARGET >> $nailgun_out 2>> $nailgun_err log "Checking if nailgun is up yet." $* - $NG cbt.NailgunLauncher cbt.CheckAlive $CP "$CWD" $* >> $nailgun_out 2>> $nailgun_err + $NG cbt.NailgunLauncher check-alive >> $nailgun_out 2>> $nailgun_err alive=$? if [ $alive -eq 131 ] || [ $alive -eq 33 ]; then # the 33 is not working right now @@ -219,17 +179,17 @@ stage1 () { break else log "Nope. Sleeping for 0.5 seconds" $* - if [ "$i" -gt "1" ]; then - echo "Waiting for nailgun to start... (For problems try -Dlog=nailgun or check logs in cbt/nailgun_launcher/target/*.log)" 1>&2 - fi + #if [ "$i" -gt 1 ]; then + # echo "Waiting for nailgun to start... (In case of problems try -Dlog=nailgun or check logs in cbt/nailgun_launcher/target/*.log)" 1>&2 + #fi fi - sleep 0.5 + sleep 0.3 done - log "Running $mainClass via Nailgun." $* - $NG cbt.NailgunLauncher $mainClass $CP "$CWD" $* + log "Running CBT via Nailgun." $* + $NG cbt.NailgunLauncher "$CWD" $* fi exitCode=$? - log "Done running $mainClass." $* + log "Done running CBT." $* } while true; do @@ -240,7 +200,7 @@ while true; do echo "======= Restarting CBT =======" 1>&2 done -if [ $compiles -ne 0 ] || [ $compiles2 -ne 0 ]; then +if [ $compiles -ne 0 ]; then exitCode=1 fi diff --git a/nailgun_launcher/CBTUrlClassLoader.java b/nailgun_launcher/CBTUrlClassLoader.java new file mode 100644 index 0000000..88bf4a3 --- /dev/null +++ b/nailgun_launcher/CBTUrlClassLoader.java @@ -0,0 +1,35 @@ +package cbt; +import java.io.*; +import java.net.*; +import java.util.*; +class CbtURLClassLoader extends java.net.URLClassLoader{ + public String toString(){ + return ( + super.toString() + + "(\n " + + Arrays.toString(getURLs()) + + ",\n " + + String.join("\n ",getParent().toString().split("\n")) + + "\n)" + ); + } + public Class loadClass(String name) throws ClassNotFoundException{ + //System.out.println("loadClass("+name+") on \n"+this); + return super.loadClass(name); + } + void assertExist(URL[] urls){ + for(URL url: urls){ + if(!new File(url.getPath()).exists()){ + throw new AssertionError("File does not exist when trying to create CbtURLClassLoader: "+url); + } + } + } + public CbtURLClassLoader(URL[] urls, ClassLoader parent){ + super(urls, parent); + assertExist(urls); + } + public CbtURLClassLoader(URL[] urls){ + super(urls); + assertExist(urls); + } +}
\ No newline at end of file diff --git a/nailgun_launcher/EarlyDependencies.java b/nailgun_launcher/EarlyDependencies.java new file mode 100644 index 0000000..1e129c7 --- /dev/null +++ b/nailgun_launcher/EarlyDependencies.java @@ -0,0 +1,99 @@ +// This file was auto-generated using `cbt admin cbtEarlyDependencies` +package cbt; +import java.io.*; +import java.nio.file.*; +import java.net.*; +import java.security.*; +import static cbt.NailgunLauncher.*; + +class EarlyDependencies{ + + /** ClassLoader for stage1 */ + ClassLoader stage1; + /** ClassLoader for zinc */ + ClassLoader zinc; + + String scalaReflect_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; + String scalaCompiler_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; + String scalaXml_1_0_5_File = MAVEN_CACHE + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"; + String scalaLibrary_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"; + String zinc_0_3_9_File = MAVEN_CACHE + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"; + String incrementalCompiler_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"; + String compilerInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"; + String scalaCompiler_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"; + String sbtInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"; + String scalaReflect_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"; + String scalaLibrary_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"; + + public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ + download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"), Paths.get(scalaReflect_2_11_8_File), "b74530deeba742ab4f3134de0c2da0edc49ca361"); + download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); + + // org.scala-lang:scala-library:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"), Paths.get(scalaLibrary_2_10_5_File), "57ac67a6cf6fd591e235c62f8893438e8d10431d"); + ClassLoader scalaLibrary_2_10_5_ = cachePut( + classLoader( scalaLibrary_2_10_5_File ), + scalaLibrary_2_10_5_File + ); + + // org.scala-lang:scala-reflect:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"), Paths.get(scalaReflect_2_10_5_File), "7392facb48876c67a89fcb086112b195f5f6bbc3"); + ClassLoader scalaReflect_2_10_5_ = cachePut( + classLoader( scalaReflect_2_10_5_File, scalaLibrary_2_10_5_ ), + scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:sbt-interface:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"), Paths.get(sbtInterface_0_13_9_File), "29848631415402c81b732e919be88f268df37250"); + ClassLoader sbtInterface_0_13_9_ = cachePut( + classLoader( sbtInterface_0_13_9_File, scalaReflect_2_10_5_ ), + sbtInterface_0_13_9_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // org.scala-lang:scala-compiler:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"), Paths.get(scalaCompiler_2_10_5_File), "f0f5bb444ca26a6e489af3dd35e24f7e2d2d118e"); + ClassLoader scalaCompiler_2_10_5_ = cachePut( + classLoader( scalaCompiler_2_10_5_File, sbtInterface_0_13_9_ ), + sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:compiler-interface:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"), Paths.get(compilerInterface_0_13_9_File), "2311addbed1182916ad00f83c57c0eeca1af382b"); + ClassLoader compilerInterface_0_13_9_ = cachePut( + classLoader( compilerInterface_0_13_9_File, scalaCompiler_2_10_5_ ), + compilerInterface_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:incremental-compiler:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"), Paths.get(incrementalCompiler_0_13_9_File), "fbbf1cadbed058aa226643e83543c35de43b13f0"); + ClassLoader incrementalCompiler_0_13_9_ = cachePut( + classLoader( incrementalCompiler_0_13_9_File, compilerInterface_0_13_9_ ), + compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.zinc:zinc:0.3.9 + download(new URL(MAVEN_URL + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"), Paths.get(zinc_0_3_9_File), "46a4556d1f36739879f4b2cc19a73d12b3036e9a"); + ClassLoader zinc_0_3_9_ = cachePut( + classLoader( zinc_0_3_9_File, incrementalCompiler_0_13_9_ ), + compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, zinc_0_3_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // org.scala-lang:scala-library:2.11.8 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); + ClassLoader scalaLibrary_2_11_8_ = cachePut( + classLoader( scalaLibrary_2_11_8_File ), + scalaLibrary_2_11_8_File + ); + + // org.scala-lang.modules:scala-xml_2.11:1.0.5 + download(new URL(MAVEN_URL + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); + ClassLoader scalaXml_1_0_5_ = cachePut( + classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), + scalaXml_1_0_5_File, scalaLibrary_2_11_8_File + ); + + stage1 = scalaXml_1_0_5_; + + zinc = zinc_0_3_9_; + } +} diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index 11a8680..50a3c91 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -4,53 +4,211 @@ import java.lang.reflect.*; import java.net.*; import java.nio.*; import java.nio.file.*; +import static java.io.File.pathSeparator; +import java.security.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import javax.xml.bind.annotation.adapters.HexBinaryAdapter; /** * This launcher allows to start the JVM without loading anything else permanently into its * classpath except for the launcher itself. That's why it is written in Java without * dependencies outside the JDK. - * - * The main method loads the given class from the given class path, calls it's main - * methods passing in the additional arguments. */ public class NailgunLauncher{ + public static String SCALA_VERSION = "2.11.8"; + public static String SCALA_XML_VERSION = "1.0.5"; + public static String ZINC_VERSION = "0.3.9"; + + public static String CBT_HOME = System.getenv("CBT_HOME"); + public static String NAILGUN = System.getenv("NAILGUN"); + public static String TARGET = System.getenv("TARGET"); + public static String STAGE1 = CBT_HOME + "/stage1/"; + public static String MAVEN_CACHE = CBT_HOME + "/cache/maven"; + public static String MAVEN_URL = "https://repo1.maven.org/maven2"; /** * Persistent cache for caching classloaders for the JVM life time. Can be used as needed by user * code to improve startup time. */ - public static ConcurrentHashMap<String,ClassLoader> classLoaderCache = - new ConcurrentHashMap<String,ClassLoader>(); + public static ConcurrentHashMap<String, Object> classLoaderCacheKeys = new ConcurrentHashMap<String,Object>(); + public static ConcurrentHashMap<Object, ClassLoader> classLoaderCacheValues = new ConcurrentHashMap<Object,ClassLoader>(); public static SecurityManager defaultSecurityManager = System.getSecurityManager(); + public static long lastSuccessfullCompile = 0; + static ClassLoader stage1classLoader = null; + public static ClassLoader stage2classLoader = null; + public static void main(String[] args) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, - MalformedURLException { - if (args.length < 3) { - System.out.println("usage: <main class> <class path> <... args>"); - } else { - // TODO: cache this classloader, but invalidate on changes - String[] cp = args[1].split(File.pathSeparator); - - URL[] urls = new URL[cp.length]; - for(int i = 0; i < cp.length; i++){ - urls[i] = new URL("file:"+cp[i]); + MalformedURLException, + IOException, + NoSuchAlgorithmException { + long now = System.currentTimeMillis(); + //System.err.println("ClassLoader: "+stage1classLoader); + //System.err.println("lastSuccessfullCompile: "+lastSuccessfullCompile); + //System.err.println("now: "+now); + + _assert(CBT_HOME != null, CBT_HOME); + _assert(NAILGUN != null, NAILGUN); + _assert(TARGET != null, TARGET); + _assert(STAGE1 != null, STAGE1); + + if(args[0].equals("check-alive")){ + System.exit(33); + return; + } + + List<File> stage1SourceFiles = new ArrayList<File>(); + for( File f: new File(STAGE1).listFiles() ){ + if( f.isFile() && f.toString().endsWith(".scala") ){ + stage1SourceFiles.add(f); } + } + + Boolean stage1SourcesChanged = false; + for( File file: stage1SourceFiles ){ + if( file.lastModified() > lastSuccessfullCompile ){ + stage1SourcesChanged = true; + //System.err.println("File change: "+file.lastModified()); + break; + } + } - String[] newArgs = new String[args.length-2]; - for(int i = 0; i < args.length-2; i++){ - newArgs[i] = args[i+2]; + if(stage1SourcesChanged || stage1classLoader == null){ + EarlyDependencies earlyDeps = new EarlyDependencies(); + int exitCode = zinc(earlyDeps, stage1SourceFiles); + if( exitCode == 0 ){ + lastSuccessfullCompile = now; + } else { + System.exit( exitCode ); } - new URLClassLoader( urls ) - .loadClass(args[0]) + ClassLoader nailgunClassLoader; + if( classLoaderCacheKeys.containsKey( NAILGUN+TARGET ) ){ + nailgunClassLoader = cacheGet( NAILGUN+TARGET ); + } else { + nailgunClassLoader = cachePut( classLoader(NAILGUN+TARGET, earlyDeps.stage1), NAILGUN+TARGET ); // FIXME: key is wrong here, should be full CP + } + + stage1classLoader = classLoader(STAGE1+TARGET, nailgunClassLoader); + stage2classLoader = null; + } + + try{ + Integer exitCode = + (Integer) stage1classLoader + .loadClass("cbt.Stage1") + .getMethod("run", String[].class, ClassLoader.class, Boolean.class) + .invoke( null, (Object) args, stage1classLoader, stage1SourcesChanged); + System.exit(exitCode); + }catch(Exception e){ + System.err.println(stage1classLoader); + throw e; + } + } + + public static void _assert(Boolean condition, Object msg){ + if(!condition){ + throw new AssertionError("Assertion failed: "+msg); + } + } + + public static int runMain(String cls, String[] args, ClassLoader cl) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ + try{ + System.setSecurityManager( new TrapSecurityManager() ); + cl.loadClass(cls) .getMethod("main", String[].class) - .invoke( null/* _cls.newInstance()*/, (Object) newArgs ); + .invoke( null, (Object) args); + return 0; + }catch( InvocationTargetException exception ){ + Throwable cause = exception.getCause(); + if(cause instanceof TrappedExitCode){ + return ((TrappedExitCode) cause).exitCode; + } + throw exception; + } finally { + System.setSecurityManager(NailgunLauncher.defaultSecurityManager); + } + } + + static int zinc( EarlyDependencies earlyDeps, List<File> sourceFiles ) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ + String cp = NAILGUN+TARGET + pathSeparator + earlyDeps.scalaXml_1_0_5_File + pathSeparator + earlyDeps.scalaLibrary_2_11_8_File; + List<String> zincArgs = new ArrayList<String>( + Arrays.asList( + new String[]{ + "-scala-compiler", earlyDeps.scalaCompiler_2_11_8_File, + "-scala-library", earlyDeps.scalaLibrary_2_11_8_File, + "-scala-extra", earlyDeps.scalaReflect_2_11_8_File, + "-sbt-interface", earlyDeps.sbtInterface_0_13_9_File, + "-compiler-interface", earlyDeps.compilerInterface_0_13_9_File, + "-cp", cp, + "-d", STAGE1+TARGET + } + ) + ); + + for( File f: sourceFiles ){ + zincArgs.add(f.toString()); + } + + PrintStream oldOut = System.out; + try{ + System.setOut(System.err); + return runMain( "com.typesafe.zinc.Main", zincArgs.toArray(new String[zincArgs.size()]), earlyDeps.zinc ); + } finally { + System.setOut(oldOut); } } + + static ClassLoader classLoader( String file ) throws MalformedURLException{ + return new CbtURLClassLoader( + new URL[]{ new URL("file:"+file) } + ); + } + static ClassLoader classLoader( String file, ClassLoader parent ) throws MalformedURLException{ + return new CbtURLClassLoader( + new URL[]{ new URL("file:"+file) }, parent + ); + } + static ClassLoader cacheGet( String key ){ + return classLoaderCacheValues.get( + classLoaderCacheKeys.get( key ) + ); + } + static ClassLoader cachePut( ClassLoader classLoader, String... jars ){ + String key = String.join( pathSeparator, jars ); + Object keyObject = new Object(); + classLoaderCacheKeys.put( key, keyObject ); + classLoaderCacheValues.put( keyObject, classLoader ); + return classLoader; + } + + public static void download(URL urlString, Path target, String sha1) throws IOException, NoSuchAlgorithmException { + final Path unverified = Paths.get(target+".unverified"); + if(!Files.exists(target)) { + new File(target.toString()).getParentFile().mkdirs(); + System.err.println("downloading " + urlString); + System.err.println("to " + target); + final InputStream stream = urlString.openStream(); + Files.copy(stream, unverified, StandardCopyOption.REPLACE_EXISTING); + stream.close(); + final String checksum = sha1(Files.readAllBytes(unverified)); + if(sha1 == null || sha1.toUpperCase().equals(checksum)) { + Files.move(unverified, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } else { + System.err.println(target + " checksum does not match.\nExpected: |" + sha1 + "|\nFound: |" + checksum + "|"); + System.exit(1); + } + } + } + + public static String sha1(byte[] bytes) throws NoSuchAlgorithmException { + final MessageDigest sha1 = MessageDigest.getInstance("SHA1"); + sha1.update(bytes, 0, bytes.length); + return (new HexBinaryAdapter()).marshal(sha1.digest()); + } } diff --git a/nailgun_launcher/TrapSecurityManager.java b/nailgun_launcher/TrapSecurityManager.java new file mode 100644 index 0000000..ed00582 --- /dev/null +++ b/nailgun_launcher/TrapSecurityManager.java @@ -0,0 +1,19 @@ +package cbt; +import java.security.*; +public class TrapSecurityManager extends SecurityManager{ + public void checkPermission( Permission permission ){ + /* + NOTE: is it actually ok, to just make these empty? + Calling .super leads to ClassNotFound exteption for a lambda. + Calling to the previous SecurityManager leads to a stack overflow + */ + } + public void checkPermission( Permission permission, Object context ){ + /* Does this methods need to be overidden? */ + } + @Override + public void checkExit( int status ){ + super.checkExit(status); + throw new TrappedExitCode(status); + } +} diff --git a/nailgun_launcher/TrappedExitCode.java b/nailgun_launcher/TrappedExitCode.java new file mode 100644 index 0000000..154db27 --- /dev/null +++ b/nailgun_launcher/TrappedExitCode.java @@ -0,0 +1,8 @@ +package cbt; +import java.security.*; +public class TrappedExitCode extends SecurityException{ + public int exitCode; + public TrappedExitCode(int exitCode){ + this.exitCode = exitCode; + } +} diff --git a/stage1/CachingClassLoader.scala b/stage1/CachingClassLoader.scala new file mode 100644 index 0000000..e75f14c --- /dev/null +++ b/stage1/CachingClassLoader.scala @@ -0,0 +1,12 @@ +package cbt +import java.net._ +import java.util.concurrent.ConcurrentHashMap +import scala.util.Try + +trait CachingClassLoader extends ClassLoader{ + def logger: Logger + val cache = new KeyLockedLazyCache[String,Try[Class[_]]]( new ConcurrentHashMap, new ConcurrentHashMap, Some(logger) ) + override def loadClass(name: String, resolve: Boolean) = { + cache.get( name, Try(super.loadClass(name, resolve)) ).get + } +} diff --git a/stage1/ClassLoaderCache.scala b/stage1/ClassLoaderCache.scala index 18a0d0e..10d872d 100644 --- a/stage1/ClassLoaderCache.scala +++ b/stage1/ClassLoaderCache.scala @@ -1,25 +1,19 @@ package cbt import java.net._ +import java.util.concurrent.ConcurrentHashMap +import collection.JavaConversions._ -private[cbt] object ClassLoaderCache{ - private val cache = NailgunLauncher.classLoaderCache - def get( classpath: ClassPath )(implicit logger: Logger): ClassLoader - = cache.synchronized{ - val lib = new Stage1Lib(logger) - val key = classpath.strings.sorted.mkString(":") - if( cache.containsKey(key) ){ - logger.resolver("CACHE HIT: "++key) - cache.get(key) - } else { - logger.resolver("CACHE MISS: "++key) - val cl = new cbt.URLClassLoader( classpath, ClassLoader.getSystemClassLoader ) - cache.put( key, cl ) - cl - } - } - def remove( classpath: ClassPath ) = { - val key = classpath.strings.sorted.mkString(":") - cache.remove( key ) - } +class ClassLoaderCache(logger: Logger){ + val persistent = new KeyLockedLazyCache( + NailgunLauncher.classLoaderCacheKeys.asInstanceOf[ConcurrentHashMap[String,AnyRef]], + NailgunLauncher.classLoaderCacheValues.asInstanceOf[ConcurrentHashMap[AnyRef,ClassLoader]], + Some(logger) + ) + val transient = new KeyLockedLazyCache( + new ConcurrentHashMap[String,AnyRef], + new ConcurrentHashMap[AnyRef,ClassLoader], + Some(logger) + ) + override def toString = s"""ClassLoaderCache("""+ persistent.keys.keySet.toVector.map(_.toString).sorted.map(" "++_).mkString("\n","\n","\n") +""")""" } diff --git a/stage1/ClassPath.scala b/stage1/ClassPath.scala index 66a1b44..96963e4 100644 --- a/stage1/ClassPath.scala +++ b/stage1/ClassPath.scala @@ -25,6 +25,6 @@ case class ClassPath(files: Seq[File]){ def string = strings.mkString( File.pathSeparator ) def strings = files.map{ f => f.string ++ ( if(f.isDirectory) "/" else "" ) - } + }.sorted def toConsole = string } diff --git a/stage1/KeyLockedLazyCache.scala b/stage1/KeyLockedLazyCache.scala index c8b37ea..aca5f74 100644 --- a/stage1/KeyLockedLazyCache.scala +++ b/stage1/KeyLockedLazyCache.scala @@ -1,33 +1,54 @@ -/* package cbt + import java.util.concurrent.ConcurrentHashMap -import scala.concurrent.Future +private[cbt] class LockableKey /** A cache that lazily computes values if needed during lookup. Locking occurs on the key, so separate keys can be looked up simultaneously without a deadlock. */ -final private[cbt] class KeyLockedLazyCache[Key <: AnyRef,Value]{ - private val keys = new ConcurrentHashMap[Key,LockableKey]() - private val builds = new ConcurrentHashMap[LockableKey,Value]() - - private class LockableKey +final private[cbt] class KeyLockedLazyCache[Key <: AnyRef,Value <: AnyRef]( + val keys: ConcurrentHashMap[Key,AnyRef], + val values: ConcurrentHashMap[AnyRef,Value], + logger: Option[Logger] +){ def get( key: Key, value: => Value ): Value = { - val keyObject = keys.synchronized{ + val lockableKey = keys.synchronized{ if( ! (keys containsKey key) ){ - keys.put( key, new LockableKey ) + val lockableKey = new LockableKey + //logger.foreach(_.resolver("CACHE MISS: " ++ key.toString)) + keys.put( key, lockableKey ) + lockableKey + } else { + val lockableKey = keys get key + //logger.foreach(_.resolver("CACHE HIT: " ++ lockableKey.toString ++ " -> " ++ key.toString)) + lockableKey } - keys get key } + import collection.JavaConversions._ + //logger.resolver("CACHE: \n" ++ keys.mkString("\n")) // synchronizing on key only, so asking for a particular key does // not block the whole cache, but just that cache entry - key.synchronized{ - if( ! (builds containsKey keyObject) ){ - builds.put( keyObject, value ) + lockableKey.synchronized{ + if( ! (values containsKey lockableKey) ){ + values.put( lockableKey, value ) } - builds get keyObject + values get lockableKey + } + } + def update( key: Key, value: Value ): Value = { + val lockableKey = keys get key + lockableKey.synchronized{ + values.put( lockableKey, value ) + value } } + def remove( key: Key ) = keys.synchronized{ + assert(keys containsKey key) + val lockableKey = keys get key + keys.remove( key ) + assert(values containsKey lockableKey) + values.remove( lockableKey ) + } } -*/ diff --git a/stage1/MultiClassLoader.scala b/stage1/MultiClassLoader.scala index de9bd32..5a93a63 100644 --- a/stage1/MultiClassLoader.scala +++ b/stage1/MultiClassLoader.scala @@ -1,24 +1,31 @@ -/* package cbt import java.net._ import scala.util.Try - import scala.collection.immutable.Seq - -class MultiClassLoader(parents: Seq[ClassLoader]) extends ClassLoader { - override def loadClass(name: String) = { - //System.err.println("LOADING CLASS "++name); - val c = parents.toStream.map{ - parent => - Try{ +// do not make this a case class, required object identity equality +class MultiClassLoader(parents: Seq[ClassLoader])(implicit val logger: Logger) extends ClassLoader with CachingClassLoader{ + override def findClass(name: String) = { + parents.find( parent => + try{ parent.loadClass(name) - }.map(Option[Class[_]](_)).recover{ - case _:ClassNotFoundException => None - }.get - }.find(_.isDefined).flatten - c.getOrElse( ClassLoader.getSystemClassLoader.loadClass(name) ) + true + } catch { + case _:ClassNotFoundException => false + } + ).map( + _.loadClass(name) + ).getOrElse( throw new ClassNotFoundException(name) ) } - override def toString = "MultiClassLoader(" ++ parents.mkString(",") ++ ")" + override def toString = ( + scala.Console.BLUE + ++ super.toString + ++ scala.Console.RESET + ++ "(" + ++ ( + if(parents.nonEmpty)( + "\n" ++ parents.map(_.toString).mkString(",\n").split("\n").map(" "++_).mkString("\n") ++ "\n" + ) else "" + ) ++")" + ) } -*/ diff --git a/stage1/PoorMansProfiler.scala b/stage1/PoorMansProfiler.scala new file mode 100644 index 0000000..b7aa47d --- /dev/null +++ b/stage1/PoorMansProfiler.scala @@ -0,0 +1,23 @@ +/* +// temporary debugging tool +package cbt +import java.util.concurrent.ConcurrentHashMap +import collection.JavaConversions._ +object PoorMansProfiler{ + val entries = new ConcurrentHashMap[String, Long] + def profile[T](name: String)(code: => T): T = { + val before = System.currentTimeMillis + if(!(entries containsKey name)){ + entries.put( name, 0 ) + } + val res = code + entries.put( name, (entries get name) + (System.currentTimeMillis - before) ) + res + } + def summary: String = { + "Profiling Summary:\n" + entries.toSeq.sortBy(_._2).map{ + case (name, value) => name + ": " + (value / 1000.0) + }.mkString("\n") + } +} +*/
\ No newline at end of file diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 937d9b3..e8245c4 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -8,76 +8,103 @@ import scala.collection.JavaConverters._ import paths._ -object CheckAlive{ - def main(args: Array[String]): Unit = { - System.exit(33) - } -} - -class Init(args: Array[String]) { +final case class Stage1ArgsParser(_args: Seq[String]) { /** * Raw parameters including their `-D` flag. **/ - val propsRaw: Seq[String] = args.toVector.filter(_.startsWith("-D")) + val propsRaw: Seq[String] = _args.toVector.filter(_.startsWith("-D")) /** * All arguments that weren't `-D` property declarations. **/ - val argsV: Seq[String] = args.toVector diff propsRaw + val args: Seq[String] = _args.toVector diff propsRaw /** * Parsed properties, as a map of keys to values. **/ - lazy val props = propsRaw + val props = propsRaw .map(_.drop(2).split("=")).map({ case Array(key, value) => key -> value }).toMap ++ System.getProperties.asScala - val logger = new Logger(props.get("log")) -} + val enabledLoggers = props.get("log") -object Stage1 extends Stage1Base{ - def mainClass = ("cbt.Stage2") + val admin = _args contains "admin" } -object AdminStage1 extends Stage1Base{ - def mainClass = ("cbt.AdminStage2") + +abstract class Stage2Base{ + def run( context: Stage2Args ): Unit } -abstract class Stage1Base{ - def mainClass: String +case class Stage2Args( + cwd: File, + args: Seq[String], + cbtHasChanged: Boolean, + logger: Logger +) +object Stage1{ protected def newerThan( a: File, b: File ) ={ a.lastModified > b.lastModified } - def main(args: Array[String]): Unit = { - val init = new Init(args) - val lib = new Stage1Lib(init.logger) + def run(_args: Array[String], classLoader: ClassLoader, stage1SourcesChanged: java.lang.Boolean): Int = { + val args = Stage1ArgsParser(_args.toVector) + val logger = new Logger(args.enabledLoggers) + logger.stage1(s"Stage1 start") + + val lib = new Stage1Lib(logger) import lib._ - logger.stage1(s"[$now] Stage1 start") - logger.stage1("Stage1: after creating lib") + val sourceFiles = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) + val changeIndicator = stage2Target ++ "/cbt/Build.class" - val cwd = args(0) + val deps = Dependencies( + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") + ) - val src = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - val changeIndicator = stage2Target ++ "/cbt/Build.class" + val classLoaderCache = new ClassLoaderCache(logger) + + val stage2SourcesChanged = lib.needsUpdate(sourceFiles, stage2StatusFile) + logger.stage1("Compiling stage2 if necessary") + val scalaXml = JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) + compile( + stage2SourcesChanged, + sourceFiles, stage2Target, stage2StatusFile, + nailgunTarget +: stage1Target +: Dependencies(deps, scalaXml).classpath, + Seq("-deprecation"), classLoaderCache, + zincVersion = "0.3.9", scalaVersion = constants.scalaVersion + ) - logger.stage1("before conditionally running zinc to recompile CBT") - if( src.exists(newerThan(_, changeIndicator)) ) { - val stage1Classpath = CbtDependency()(logger).dependencyClasspath - logger.stage1("cbt.lib has changed. Recompiling with cp: " ++ stage1Classpath.string) - zinc( true, src, stage2Target, stage1Classpath, Seq("-deprecation") )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) - } logger.stage1(s"[$now] calling CbtDependency.classLoader") + if(NailgunLauncher.stage2classLoader == null){ + NailgunLauncher.stage2classLoader = CbtDependency().classLoader(classLoaderCache) + } logger.stage1(s"[$now] Run Stage2") - val ExitCode(exitCode) = /*trapExitCode*/{ // this - runMain( mainClass, cwd +: args.drop(1).toVector, CbtDependency()(logger).classLoader ) - } + val exitCode = ( + NailgunLauncher.stage2classLoader.loadClass( + if(args.admin) "cbt.AdminStage2" else "cbt.Stage2" + ) + .getMethod( "run", classOf[Stage2Args] ) + .invoke( + null, + Stage2Args( + new File( args.args(0) ), + args.args.drop(1).toVector, + // launcher changes cause entire nailgun restart, so no need for them here + cbtHasChanged = stage1SourcesChanged || stage2SourcesChanged, + logger + ) + ) match { + case code: ExitCode => code + case _ => ExitCode.Success + } + ).integer logger.stage1(s"[$now] Stage1 end") - System.exit(exitCode) + return exitCode; } } diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index b76e21b..105fe3e 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -6,6 +6,7 @@ import java.io._ import java.lang.reflect.InvocationTargetException import java.net._ import java.nio.file._ +import java.nio.file.attribute.FileTime import javax.tools._ import java.security._ import java.util._ @@ -14,23 +15,23 @@ import javax.xml.bind.annotation.adapters.HexBinaryAdapter import scala.collection.immutable.Seq // CLI interop -case class ExitCode(code: Int) +case class ExitCode(integer: Int) object ExitCode{ val Success = ExitCode(0) val Failure = ExitCode(1) } -class TrappedExitCode(private val exitCode: Int) extends Exception -object TrappedExitCode{ - def unapply(e: Throwable): Option[ExitCode] = +object CatchTrappedExitCode{ + def unapply(e: Throwable): Option[ExitCode] = { Option(e) flatMap { case i: InvocationTargetException => unapply(i.getTargetException) case e: TrappedExitCode => Some( ExitCode(e.exitCode) ) case _ => None } + } } -case class Context( cwd: File, args: Seq[String], logger: Logger ) +case class Context( cwd: File, args: Seq[String], logger: Logger, classLoaderCache: ClassLoaderCache ) class BaseLib{ def realpath(name: File) = new File(Paths.get(name.getAbsolutePath).normalize.toString) @@ -42,20 +43,6 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def scalaMajorVersion(scalaMinorVersion: String) = scalaMinorVersion.split("\\.").take(2).mkString(".") - // ========== reflection ========== - - /** Create instance of the given class via reflection */ - def create(cls: String)(args: Any*)(classLoader: ClassLoader): Any = { - logger.composition( logger.showInvocation("Stage1Lib.create", (classLoader,cls,args)) ) - import scala.reflect.runtime.universe._ - val m = runtimeMirror(classLoader) - val sym = m.classSymbol(classLoader.loadClass(cls)) - val cm = m.reflectClass( sym.asClass ) - val tpe = sym.toType - val ctorm = cm.reflectConstructor( tpe.decl(termNames.CONSTRUCTOR).asMethod ) - ctorm(args:_*) - } - // ========== file system / net ========== def array2hex(padTo: Int, array: Array[Byte]): String = { @@ -108,8 +95,9 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ trapExitCode{ classLoader .loadClass(cls) - .getMethod( "main", scala.reflect.classTag[Array[String]].runtimeClass ) + .getMethod( "main", classOf[Array[String]] ) .invoke( null, args.toArray.asInstanceOf[AnyRef] ) + ExitCode.Success } } @@ -124,71 +112,80 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } } - def zinc( + def needsUpdate( sourceFiles: Seq[File], statusFile: File ) = { + val lastCompile = statusFile.lastModified + sourceFiles.filter(_.lastModified > lastCompile).nonEmpty + } + + def compile( needsRecompile: Boolean, files: Seq[File], compileTarget: File, + statusFile: File, classpath: ClassPath, - extraArgs: Seq[String] = Seq() - )( zincVersion: String, scalaVersion: String ): Unit = { + scalacOptions: Seq[String] = Seq(), + classLoaderCache: ClassLoaderCache, + zincVersion: String, + scalaVersion: String + ): Option[File] = { val cp = classpath.string if(classpath.files.isEmpty) throw new Exception("Trying to compile with empty classpath. Source files: " ++ files.toString) - if(files.isEmpty) - throw new Exception("Trying to compile no files. ClassPath: " ++ cp) - - // only run zinc if files changed, for performance reasons - // FIXME: this is broken, need invalidate on changes in dependencies as well - if( needsRecompile ){ - val zinc = JavaDependency("com.typesafe.zinc","zinc", zincVersion) - val zincDeps = zinc.transitiveDependencies - - val sbtInterface = - zincDeps - .collect{ case d @ JavaDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none ) => d } - .headOption - .getOrElse( throw new Exception(s"cannot find sbt-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) - .jar - - val compilerInterface = - zincDeps - .collect{ case d @ JavaDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources ) => d } - .headOption - .getOrElse( throw new Exception(s"cannot find compiler-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) - .jar - - val scalaLibrary = JavaDependency("org.scala-lang","scala-library",scalaVersion).jar - val scalaReflect = JavaDependency("org.scala-lang","scala-reflect",scalaVersion).jar - val scalaCompiler = JavaDependency("org.scala-lang","scala-compiler",scalaVersion).jar - - val code = redirectOutToErr{ - lib.runMain( - "com.typesafe.zinc.Main", - Seq( - "-scala-compiler", scalaCompiler.toString, - "-scala-library", scalaLibrary.toString, - "-sbt-interface", sbtInterface.toString, - "-compiler-interface", compilerInterface.toString, - "-scala-extra", scalaReflect.toString, - "-cp", cp, - "-d", compileTarget.toString - ) ++ extraArgs.map("-S"++_) ++ files.map(_.toString), - zinc.classLoader - ) - } - - if(code != ExitCode.Success){ - // Ensure we trigger recompilation next time. This is currently required because we - // don't record the time of the last successful build elsewhere. But hopefully that will - // change soon. - val now = System.currentTimeMillis() - files.foreach(_.setLastModified(now)) - - // Tell the caller that things went wrong. - System.exit(code.code) + if( files.isEmpty ){ + None + }else{ + if( needsRecompile ){ + val zinc = JavaDependency("com.typesafe.zinc","zinc", zincVersion) + val zincDeps = zinc.transitiveDependencies + + val sbtInterface = + zincDeps + .collect{ case d @ JavaDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none ) => d } + .headOption + .getOrElse( throw new Exception(s"cannot find sbt-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) + .jar + + val compilerInterface = + zincDeps + .collect{ case d @ JavaDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources ) => d } + .headOption + .getOrElse( throw new Exception(s"cannot find compiler-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) + .jar + + val scalaLibrary = JavaDependency("org.scala-lang","scala-library",scalaVersion).jar + val scalaReflect = JavaDependency("org.scala-lang","scala-reflect",scalaVersion).jar + val scalaCompiler = JavaDependency("org.scala-lang","scala-compiler",scalaVersion).jar + + val start = System.currentTimeMillis + + val code = redirectOutToErr{ + lib.runMain( + "com.typesafe.zinc.Main", + Seq( + "-scala-compiler", scalaCompiler.toString, + "-scala-library", scalaLibrary.toString, + "-sbt-interface", sbtInterface.toString, + "-compiler-interface", compilerInterface.toString, + "-scala-extra", scalaReflect.toString, + "-cp", cp, + "-d", compileTarget.toString + ) ++ scalacOptions.map("-S"++_) ++ files.map(_.toString), + zinc.classLoader(classLoaderCache) + ) + } + + if(code == ExitCode.Success){ + // write version and when last compilation started so we can trigger + // recompile if cbt version changed or newer source files are seen + Files.write(statusFile.toPath, "".getBytes)//cbtVersion.getBytes) + Files.setLastModifiedTime(statusFile.toPath, FileTime.fromMillis(start) ) + } else { + System.exit(code.integer) // FIXME: let's find a better solution for error handling. Maybe a monad after all. + } } + Some( compileTarget ) } } def redirectOutToErr[T](code: => T): T = { @@ -201,31 +198,12 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } } - private val trapSecurityManager = new SecurityManager { - override def checkPermission( permission: Permission ) = { - /* - NOTE: is it actually ok, to just make these empty? - Calling .super leads to ClassNotFound exteption for a lambda. - Calling to the previous SecurityManager leads to a stack overflow - */ - } - override def checkPermission( permission: Permission, context: Any ) = { - /* Does this methods need to be overidden? */ - } - override def checkExit( status: Int ) = { - super.checkExit(status) - logger.lib(s"checkExit($status)") - throw new TrappedExitCode(status) - } - } - - def trapExitCode( code: => Unit ): ExitCode = { + def trapExitCode( code: => ExitCode ): ExitCode = { try{ - System.setSecurityManager( trapSecurityManager ) + System.setSecurityManager( new TrapSecurityManager ) code - ExitCode.Success } catch { - case TrappedExitCode(exitCode) => + case CatchTrappedExitCode(exitCode) => exitCode } finally { System.setSecurityManager(NailgunLauncher.defaultSecurityManager) diff --git a/stage1/URLClassLoader.scala b/stage1/URLClassLoader.scala index 870f186..9e96992 100644 --- a/stage1/URLClassLoader.scala +++ b/stage1/URLClassLoader.scala @@ -1,22 +1,47 @@ package cbt import java.net._ +import scala.util.Try -case class URLClassLoader(classPath: ClassPath, parent: ClassLoader) +class URLClassLoader( classPath: ClassPath, parent: ClassLoader )( implicit val logger: Logger ) extends java.net.URLClassLoader( - classPath.strings.map( - path => new URL("file:"++path) - ).toArray, + classPath.strings.map( p => new URL("file:" ++ p) ).toArray, parent - ){ + ) with CachingClassLoader{ + val id = Math.abs( new java.util.Random().nextInt ) override def toString = ( - scala.Console.BLUE ++ "cbt.URLClassLoader" ++ scala.Console.RESET - ++ "(\n " ++ getURLs.map(_.toString).sorted.mkString(",\n ") + scala.Console.BLUE + ++ getClass.getSimpleName ++ ":" ++ id.toString + ++ scala.Console.RESET + ++ "(\n" ++ ( - if(getParent() != ClassLoader.getSystemClassLoader()) - ",\n" ++ getParent().toString.split("\n").map(" "++_).mkString("\n") - else "" - ) + getURLs.map(_.toString).sorted.mkString(",\n") + ++ ( + if(getParent() != ClassLoader.getSystemClassLoader()) + ",\n" ++ getParent().toString + else "" + ) + ).split("\n").map(" "++_).mkString("\n") ++ "\n)" ) } + +/* +trait ClassLoaderLogging extends ClassLoader{ + def logger: Logger + val prefix = s"[${getClass.getSimpleName}] " + val postfix = " in \name" ++ this.toString + override def loadClass(name: String, resolve: Boolean): Class[_] = { + //logger.resolver(prefix ++ s"loadClass($name, $resolve)" ++ postfix ) + super.loadClass(name, resolve) + } + override def loadClass(name: String): Class[_] = { + //logger.resolver(prefix ++ s"loadClass($name)" ++ postfix ) + super.loadClass(name) + } + override def findClass(name: String): Class[_] = { + //logger.resolver(prefix ++ s"findClass($name)" ++ postfix ) + super.findClass(name) + } +} +*/ diff --git a/stage1/constants.scala b/stage1/constants.scala index a14754e..4c39237 100644 --- a/stage1/constants.scala +++ b/stage1/constants.scala @@ -1,5 +1,7 @@ package cbt object constants{ - val scalaVersion = Option(System.getenv("SCALA_VERSION")).get + val scalaXmlVersion = NailgunLauncher.SCALA_XML_VERSION + val scalaVersion = NailgunLauncher.SCALA_VERSION + val zincVersion = NailgunLauncher.ZINC_VERSION val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") } diff --git a/stage1/paths.scala b/stage1/paths.scala index d3856c8..f27e538 100644 --- a/stage1/paths.scala +++ b/stage1/paths.scala @@ -4,12 +4,14 @@ object paths{ val cbtHome: File = new File(Option(System.getenv("CBT_HOME")).get) val mavenCache: File = cbtHome ++ "/cache/maven" val userHome: File = new File(Option(System.getProperty("user.home")).get) - val stage1: File = new File(Option(System.getenv("STAGE1")).get) - val stage2: File = cbtHome ++ "/stage2" + val bootstrapScala: File = cbtHome ++ "/bootstrap_scala" val nailgun: File = new File(Option(System.getenv("NAILGUN")).get) + val stage1: File = new File(NailgunLauncher.STAGE1) + val stage2: File = cbtHome ++ "/stage2" private val target = Option(System.getenv("TARGET")).get.stripSuffix("/") val stage1Target: File = stage1 ++ ("/" ++ target) val stage2Target: File = stage2 ++ ("/" ++ target) + val stage2StatusFile: File = stage2Target ++ ".last-success" val nailgunTarget: File = nailgun ++ ("/" ++ target) val sonatypeLogin: File = cbtHome ++ "/sonatype.login" } diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 1dbadcc..2e8ef15 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -28,7 +28,7 @@ abstract class Dependency{ implicit def logger: Logger protected def lib = new Stage1Lib(logger) - def updated: Boolean + def needsUpdate: Boolean //def cacheClassLoader: Boolean = false private[cbt] def targetClasspath: ClassPath def exportedClasspath: ClassPath @@ -36,7 +36,6 @@ abstract class Dependency{ def jars: Seq[File] = exportedJars ++ dependencyJars def canBeCached = false - def cacheDependencyClassLoader = true //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] def exportClasspathConcurrently: ClassPath = { @@ -86,37 +85,64 @@ abstract class Dependency{ ) } - private object classLoaderCache extends Cache[URLClassLoader] - def classLoader: URLClassLoader = classLoaderCache{ - if( concurrencyEnabled ){ - // trigger concurrent building / downloading dependencies - exportClasspathConcurrently - } - val transitiveClassPath = transitiveDependencies.map{ - case d if d.canBeCached => Left(d) - case d => Right(d) - } - val buildClassPath = ClassPath.flatten( - transitiveClassPath.flatMap( - _.right.toOption.map(_.exportedClasspath) - ) - ) - val cachedClassPath = ClassPath.flatten( - transitiveClassPath.flatMap( - _.left.toOption - ).par.map(_.exportedClasspath).seq.sortBy(_.string) - ) - - if(cacheDependencyClassLoader){ - new URLClassLoader( - exportedClasspath ++ buildClassPath, - ClassLoaderCache.get( cachedClassPath ) + private def actual(current: Dependency, latest: Map[(String,String),Dependency]) = current match { + case d: ArtifactInfo => latest((d.groupId,d.artifactId)) + case d => d + } + private def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + if( dependencies.isEmpty ){ + ClassLoader.getSystemClassLoader + } else if( dependencies.size == 1 ){ + dependencies.head.classLoaderRecursion( latest, cache ) + } else if( dependencies.forall(_.canBeCached) ){ + assert(transitiveDependencies.forall(_.canBeCached)) + cache.persistent.get( + dependencyClasspath.string, + new MultiClassLoader( + dependencies.map( _.classLoaderRecursion(latest, cache) ) + ) ) } else { + val (cachable, nonCachable) = dependencies.partition(_.canBeCached) new URLClassLoader( - exportedClasspath ++ buildClassPath ++ cachedClassPath, ClassLoader.getSystemClassLoader + ClassPath.flatten( nonCachable.map(actual(_,latest)).map(_.exportedClasspath) ), + cache.persistent.get( + ClassPath.flatten( cachable.map(actual(_,latest)).map(_.exportedClasspath) ).string, + new MultiClassLoader( + cachable.map( _.classLoaderRecursion(latest, cache) ) + ) + ) + ) + new MultiClassLoader( + dependencies.map( _.classLoaderRecursion(latest, cache) ) + ) + } + } + protected def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + if( canBeCached ){ + val a = actual( this, latest ) + cache.persistent.get( + a.classpath.string, + new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) ) + } else { + new cbt.URLClassLoader( exportedClasspath, dependencyClassLoader(latest, cache) ) + } + } + private object classLoaderCache extends Cache[ClassLoader] + def classLoader( cache: ClassLoaderCache ): ClassLoader = classLoaderCache{ + if( concurrencyEnabled ){ + // trigger concurrent building / downloading dependencies + exportClasspathConcurrently } + classLoaderRecursion( + (this +: transitiveDependencies).collect{ + case d: ArtifactInfo => d + }.groupBy( + d => (d.groupId,d.artifactId) + ).mapValues(_.head), + cache + ) } def classpath : ClassPath = exportedClasspath ++ dependencyClasspath def dependencyJars : Seq[File] = transitiveDependencies.flatMap(_.jars) @@ -128,25 +154,30 @@ abstract class Dependency{ new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents))) } - def transitiveDependencies: Seq[Dependency] = { - val deps = dependencies.flatMap(_.resolveRecursive().linearize) + def linearize(deps: Seq[Dependency]): Seq[Dependency] = + if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) ) + + private object transitiveDependenciesCache extends Cache[Seq[Dependency]] + /** return dependencies in order of linearized dependence. this is a bit tricky. */ + def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ + val deps = linearize(dependencies) val hasInfo = deps.collect{ case d:ArtifactInfo => d } val noInfo = deps.filter{ case _:ArtifactInfo => false case _ => true } - noInfo ++ JavaDependency.removeOutdated( hasInfo ) - }.sortBy(_.targetClasspath.string) + noInfo ++ JavaDependency.updateOutdated( hasInfo ).reverse.distinct + } def show: String = this.getClass.getSimpleName // ========== debug ========== def dependencyTree: String = dependencyTreeRecursion() private def dependencyTreeRecursion(indent: Int = 0): String = ( ( " " * indent ) - ++ (if(updated) lib.red(show) else show) + ++ (if(needsUpdate) lib.red(show) else show) ++ dependencies.map( - _.dependencyTreeRecursion(indent + 1) - ).map( "\n" ++ _.toString ).mkString("") + "\n" ++ _.dependencyTreeRecursion(indent + 1) + ).mkString ) } @@ -156,7 +187,7 @@ class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends class ScalaReflectDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-reflect",version) case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd => - final val updated = false + override final val needsUpdate = false override def canBeCached = true def targetClasspath = ClassPath(Seq()) def exportedClasspath = ClassPath(Seq()) @@ -169,35 +200,56 @@ case class ScalaDependencies(version: String)(implicit val logger: Logger) exten } case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{ - def updated = false def exportedClasspath = ClassPath(Seq(path)) def exportedJars = Seq[File](path) + override def needsUpdate = false def targetClasspath = exportedClasspath } +/** Allows to easily assemble a bunch of dependencies */ +case class Dependencies( _dependencies: Dependency* )(implicit val logger: Logger) extends Dependency{ + override def dependencies = _dependencies.to + def needsUpdate = dependencies.exists(_.needsUpdate) + def exportedClasspath = ClassPath(Seq()) + def exportedJars = Seq() + def targetClasspath = ClassPath(Seq()) +} + case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{ - def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) - def exportedJars = Seq[File]() - def dependencies = ScalaDependencies(constants.scalaVersion).dependencies - def updated = false // FIXME: think this through, might allow simplifications and/or optimizations - def targetClasspath = exportedClasspath + def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations + override def canBeCached = false + /* + private object classLoaderRecursionCache extends Cache[ClassLoader] + override def classLoaderRecursion(latest: Map[(String,String),Dependency], cache: ClassLoaderCache) = classLoaderRecursionCache{ + println(System.currentTimeMillis) + val cl = getClass.getClassLoader + println(System.currentTimeMillis) + cl + ClassLoader.getSystemClassLoader + } + */ + override def targetClasspath = exportedClasspath + override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) + override def exportedJars = ???//Seq[File]() + override def dependencies = Seq( + JavaDependency("org.scala-lang","scala-library",constants.scalaVersion), + JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5") + ) + // FIXME: implement sanity check to prevent using incompatible scala-library and xml version on cp + override def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ) + = getClass.getClassLoader } case class CbtDependency()(implicit val logger: Logger) extends Dependency{ - def exportedClasspath = ClassPath( Seq( stage2Target ) ) - def exportedJars = Seq[File]() + def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations + override def canBeCached = false + override def targetClasspath = exportedClasspath + override def exportedClasspath = ClassPath( Seq( stage2Target ) ) + override def exportedJars = ??? override def dependencies = Seq( Stage1Dependency(), JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), - lib.ScalaDependency( - "com.lihaoyi","ammonite-ops","0.5.5", scalaVersion = constants.scalaMajorVersion - ), - lib.ScalaDependency( - "org.scala-lang.modules","scala-xml","1.0.5", scalaVersion = constants.scalaMajorVersion - ) + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") ) - def updated = false // FIXME: think this through, might allow simplifications and/or optimizations - def targetClasspath = exportedClasspath } case class Classifier(name: Option[String]) @@ -210,8 +262,14 @@ object Classifier{ case class JavaDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none )(implicit val logger: Logger) extends ArtifactInfo{ + assert(groupId != "", toString) + assert(artifactId != "", toString) + assert(version != "", toString) + assert(groupId != null, toString) + assert(artifactId != null, toString) + assert(version != null, toString) - def updated = false + override def needsUpdate = false override def canBeCached = true private val groupPath = groupId.split("\\.").mkString("/") @@ -226,7 +284,7 @@ case class JavaDependency( private def jarFile: File = baseFile ++ ".jar" //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" private def pomUrl: URL = baseUrl ++ ".pom" - private def jarUrl: URL = baseUrl ++ ".jar" + private[cbt] def jarUrl: URL = baseUrl ++ ".jar" def exportedJars = Seq( jar ) def exportedClasspath = ClassPath( exportedJars ) @@ -235,36 +293,33 @@ case class JavaDependency( def jarSha1 = { val file = jarFile ++ ".sha1" - scala.util.Try{ - lib.download( jarUrl ++ ".sha1" , file, None ) - // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim - }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible) + lib.download( jarUrl ++ ".sha1" , file, None ) + // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom + Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim } def pomSha1 = { val file = pomFile++".sha1" - scala.util.Try{ - lib.download( pomUrl++".sha1" , file, None ) - // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim - }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible) + lib.download( pomUrl++".sha1" , file, None ) + // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom + Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim } - def jar = { - lib.download( jarUrl, jarFile, jarSha1 ) + private object jarCache extends Cache[File] + def jar = jarCache{ + lib.download( jarUrl, jarFile, Some(jarSha1) ) jarFile } def pomXml = XML.loadFile(pom.toString) def pom = { - lib.download( pomUrl, pomFile, pomSha1 ) + lib.download( pomUrl, pomFile, Some(pomSha1) ) pomFile } // ========== pom traversal ========== - lazy val pomParents: Seq[JavaDependency] = { + lazy val transitivePom: Seq[JavaDependency] = { (pomXml \ "parent").collect{ case parent => JavaDependency( @@ -272,12 +327,12 @@ case class JavaDependency( (parent \ "artifactId").text, (parent \ "version").text )(logger) - } + }.flatMap(_.transitivePom) :+ this } lazy val properties: Map[String, String] = ( - pomParents.flatMap(_.properties) ++ { - val props = (pomXml \ "properties").flatMap(_.child).map{ + transitivePom.flatMap{ d => + val props = (d.pomXml \ "properties").flatMap(_.child).map{ tag => tag.label -> tag.text } logger.pom(s"Found properties in $pom: $props") @@ -285,17 +340,15 @@ case class JavaDependency( } ).toMap - lazy val dependencyVersions: Map[(String,String), String] = - pomParents.flatMap( + lazy val dependencyVersions: Map[String, (String,String)] = + transitivePom.flatMap( p => - p.dependencyVersions - ++ (p.pomXml \ "dependencyManagement" \ "dependencies" \ "dependency").map{ xml => val groupId = p.lookup(xml,_ \ "groupId").get val artifactId = p.lookup(xml,_ \ "artifactId").get val version = p.lookup(xml,_ \ "version").get - (groupId, artifactId) -> version + artifactId -> (groupId, version) } ).toMap @@ -303,14 +356,27 @@ case class JavaDependency( if(classifier == Classifier.sources) Seq() else (pomXml \ "dependencies" \ "dependency").collect{ case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" => - val groupId = lookup(xml,_ \ "groupId").get val artifactId = lookup(xml,_ \ "artifactId").get + val groupId = + lookup(xml,_ \ "groupId").getOrElse( + dependencyVersions + .get(artifactId).map(_._1) + .getOrElse( + throw new Exception(s"$artifactId not found in \n$dependencyVersions") + ) + ) + val version = + lookup(xml,_ \ "version").getOrElse( + dependencyVersions + .get(artifactId).map(_._2) + .getOrElse( + throw new Exception(s"$artifactId not found in \n$dependencyVersions") + ) + ) JavaDependency( - groupId, - artifactId, - lookup(xml,_ \ "version").getOrElse( dependencyVersions(groupId, artifactId) ), + groupId, artifactId, version, Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) - )(logger) + ) }.toVector } def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { @@ -319,7 +385,16 @@ case class JavaDependency( accessor(xml).headOption.flatMap{v => //println("found: "++v.text) v.text match { - case Substitution(path) => Option(properties(path)) + case Substitution(path) => Option( + properties.get(path).orElse( + transitivePom.reverse.flatMap{ d => + Some(path.split("\\.").toList).collect{ + case "project" :: path => + path.foldLeft(d.pomXml:NodeSeq){ case (xml,tag) => xml \ tag }.text + }.filter(_ != "") + }.headOption + ) + .getOrElse( throw new Exception(s"Can't find $path in \n$properties.\n\npomParents: $transitivePom\n\n pomXml:\n$pomXml" ))) //println("lookup "++path ++ ": "++(pomXml\path).text) case value => Option(value) } @@ -344,7 +419,7 @@ object JavaDependency{ case e: NumberFormatException => Right(str) } /* this obviously should be overridable somehow */ - def removeOutdated( + def updateOutdated( deps: Seq[ArtifactInfo], versionLessThan: (String, String) => Boolean = semanticVersionLessThan )(implicit logger: Logger): Seq[ArtifactInfo] = { @@ -354,11 +429,11 @@ object JavaDependency{ _.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) ) .last ) - deps.flatMap{ + deps.map{ d => - val l = latest.get((d.groupId,d.artifactId)) + val l = latest((d.groupId,d.artifactId)) if(d != l) logger.resolver("outdated: "++d.show) l - }.distinct + } } } diff --git a/stage2/AdminStage2.scala b/stage2/AdminStage2.scala index d923b22..883b5ed 100644 --- a/stage2/AdminStage2.scala +++ b/stage2/AdminStage2.scala @@ -1,13 +1,12 @@ package cbt import java.io._ -object AdminStage2{ - def main(_args: Array[String]) = { - val args = _args.drop(1).dropWhile(Seq("admin","direct") contains _) - val init = new Init(args) - val lib = new Lib(init.logger) - val adminTasks = new AdminTasks(lib, args, new File(_args(0))) +object AdminStage2 extends Stage2Base{ + def run( _args: Stage2Args ): Unit = { + val args = _args.args.dropWhile(Seq("admin","direct") contains _) + val lib = new Lib(_args.logger) + val adminTasks = new AdminTasks(lib, args, _args.cwd) new lib.ReflectObject(adminTasks){ - def usage: String = "Available methods: " ++ lib.taskNames(subclassType).mkString(" ") + def usage: String = "Available methods: " ++ lib.taskNames(adminTasks.getClass).mkString(" ") }.callNullary(args.lift(0)) } } diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala index e7fc78b..069b712 100644 --- a/stage2/AdminTasks.scala +++ b/stage2/AdminTasks.scala @@ -1,35 +1,132 @@ package cbt import scala.collection.immutable.Seq -import java.io._ -class AdminTasks(lib: Lib, args: Array[String], cwd: File){ +import java.io.{Console=>_,_} +import java.nio.file._ +class AdminTasks(lib: Lib, args: Seq[String], cwd: File){ implicit val logger: Logger = lib.logger def resolve = { ClassPath.flatten( args(1).split(",").toVector.map{ d => val v = d.split(":") - new JavaDependency(v(0),v(1),v(2))(lib.logger).classpath + new JavaDependency(v(0),v(1),v(2)).classpath } ) } + def dependencyTree = { + args(1).split(",").toVector.map{ + d => + val v = d.split(":") + new JavaDependency(v(0),v(1),v(2)).dependencyTree + }.mkString("\n\n") + } def amm = ammonite def ammonite = { val version = args.lift(1).getOrElse(constants.scalaVersion) val scalac = new ScalaCompilerDependency( version ) val d = JavaDependency( - "com.lihaoyi","ammonite-repl_2.11.7",args.lift(1).getOrElse("0.5.6") + "com.lihaoyi","ammonite-repl_2.11.7",args.lift(1).getOrElse("0.5.7") ) // FIXME: this does not work quite yet, throws NoSuchFileException: /ammonite/repl/frontend/ReplBridge$.class lib.runMain( - "ammonite.repl.Main", Seq(), d.classLoader + "ammonite.repl.Main", Seq(), d.classLoader(new ClassLoaderCache(logger)) ) } def scala = { val version = args.lift(1).getOrElse(constants.scalaVersion) val scalac = new ScalaCompilerDependency( version ) lib.runMain( - "scala.tools.nsc.MainGenericRunner", Seq("-cp", scalac.classpath.string), scalac.classLoader + "scala.tools.nsc.MainGenericRunner", Seq("-cp", scalac.classpath.string), scalac.classLoader(new ClassLoaderCache(logger)) ) } def scaffoldBasicBuild: Unit = lib.scaffoldBasicBuild( cwd ) + def cbtEarlyDependencies = { + val scalaVersion = args.lift(1).getOrElse(constants.scalaVersion) + val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") + val scalaXmlVersion = args.lift(2).getOrElse(constants.scalaXmlVersion) + val zincVersion = args.lift(3).getOrElse(constants.zincVersion) + /* + def tree(d: JavaDependency, indent: Int): String ={ + val dependencies = { + if( d.dependencies.nonEmpty ){ + d.dependencies.map{ + case d: JavaDependency => tree(d,indent + 1) + }.mkString(",\n" ++ ( " " * indent ),",\n" ++ ( " " * indent ), "") + } else "" + } + ( + s"""new EarlyDependency( "${d.groupId}", "${d.artifactId}", "${d.version}", "${d.jarSha1}"$dependencies)""" + ) + }*/ + val scalaDeps = Seq( + JavaDependency("org.scala-lang","scala-reflect",scalaVersion), + JavaDependency("org.scala-lang","scala-compiler",scalaVersion) + ) + + val scalaXml = Dependencies( + JavaDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion), + JavaDependency("org.scala-lang","scala-library",scalaVersion) + ) + + val zinc = JavaDependency("com.typesafe.zinc","zinc",zincVersion) + println(zinc.dependencyTree) + + def valName(dep: JavaDependency) = { + val words = dep.artifactId.split("_").head.split("-") + words(0) ++ words.drop(1).map(s => s(0).toString.toUpperCase ++ s.drop(1)).mkString ++ "_" ++ dep.version.replace(".","_") ++ "_" + } + + def vals(d: JavaDependency) = s""" """ + + def jarVal(dep: JavaDependency) = "_" + valName(dep) +"Jar" + def transitive(dep: Dependency) = (dep +: dep.transitiveDependencies.reverse).collect{case d: JavaDependency => d} + def codeEach(dep: Dependency) = { + transitive(dep).tails.map(_.reverse).toVector.reverse.drop(1).map{ + deps => + val d = deps.last + val parents = deps.dropRight(1) + val parentString = if(parents.isEmpty) "" else ( ", " ++ valName(parents.last) ) + val n = valName(d) + s""" + // ${d.groupId}:${d.artifactId}:${d.version} + download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}"); + ClassLoader $n = cachePut( + classLoader( ${n}File$parentString ), + ${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")} + );""" + } + } + val assignments = codeEach(zinc) ++ codeEach(scalaXml) + //{ case (name, dep) => s"$name =\n ${tree(dep, 4)};" }.mkString("\n\n ") + val code = s"""// This file was auto-generated using `cbt admin cbtEarlyDependencies` +package cbt; +import java.io.*; +import java.nio.file.*; +import java.net.*; +import java.security.*; +import static cbt.NailgunLauncher.*; + +class EarlyDependencies{ + + /** ClassLoader for stage1 */ + ClassLoader stage1; + /** ClassLoader for zinc */ + ClassLoader zinc; + +${(scalaDeps ++ transitive(scalaXml) ++ transitive(zinc)).map(d => s""" String ${valName(d)}File = MAVEN_CACHE + "${d.basePath}.jar";""").mkString("\n")} + + public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ +${scalaDeps.map(d => s""" download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} +${assignments.mkString("\n")} + + stage1 = scalaXml_${scalaXmlVersion.replace(".","_")}_; + + zinc = zinc_${zincVersion.replace(".","_")}_; + } +} +""" + val file = paths.nailgun ++ ("/" ++ "EarlyDependencies.java") + Files.write( file.toPath, code.getBytes ) + println( Console.GREEN ++ "Wrote " ++ file.string ++ Console.RESET ) + } } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 2f90197..9ed8c26 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -2,7 +2,6 @@ package cbt import cbt.paths._ import java.io._ -import java.lang.reflect.InvocationTargetException import java.net._ import java.nio.file.{Path =>_,_} import java.nio.file.Files.readAllBytes @@ -10,15 +9,13 @@ import java.security.MessageDigest import java.util.jar._ import scala.collection.immutable.Seq -import scala.reflect.runtime.{universe => ru} import scala.util._ -import ammonite.ops.{cwd => _,_} - class BasicBuild( context: Context ) extends Build( context ) class Build(val context: Context) extends Dependency with TriggerLoop{ // library available to builds implicit final val logger: Logger = context.logger + implicit final val classLoaderCache: ClassLoaderCache = context.classLoaderCache override final protected val lib: Lib = new Lib(logger) // ========== general stuff ========== @@ -26,7 +23,7 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ def enableConcurrency = false final def projectDirectory: File = lib.realpath(context.cwd) assert( projectDirectory.exists, "projectDirectory does not exist: " ++ projectDirectory.string ) - final def usage: Unit = new lib.ReflectBuild(this).usage + final def usage: String = lib.usage(this.getClass, context) // ========== meta data ========== @@ -51,6 +48,12 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ def apiTarget: File = scalaTarget ++ "/api" /** directory where the class files should be put (in package directories) */ def compileTarget: File = scalaTarget ++ "/classes" + /** + File which cbt uses to determine if it needs to trigger an incremental re-compile. + Last modified date is the time when the last successful compilation started. + Contents is the cbt version git hash. + */ + def compileStatusFile: File = compileTarget ++ ".last-success" /** Source directories and files. Defaults to .scala and .java files in src/ and top-level. */ def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(sourceFileFilter) @@ -113,7 +116,7 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ override def dependencyClasspath : ClassPath = ClassPath(localJars) ++ super.dependencyClasspath override def dependencyJars : Seq[File] = localJars ++ super.dependencyJars - def exportedClasspath : ClassPath = ClassPath(Seq(compile)) + def exportedClasspath : ClassPath = ClassPath(compile.toSeq:_*) def targetClasspath = ClassPath(Seq(compileTarget)) def exportedJars: Seq[File] = Seq() // ========== compile, run, test ========== @@ -121,38 +124,25 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ /** scalac options used for zinc and scaladoc */ def scalacOptions: Seq[String] = Seq( "-feature", "-deprecation", "-unchecked" ) - val updated: Boolean = { - val existingClassFiles = lib.listFilesRecursive(compileTarget) - val sourcesChanged = existingClassFiles.nonEmpty && { - val oldestClassFile = existingClassFiles.sortBy(_.lastModified).head - val oldestClassFileAge = oldestClassFile.lastModified - val changedSourceFiles = sourceFiles.filter(_.lastModified > oldestClassFileAge) - if(changedSourceFiles.nonEmpty){ - /* - println(changedSourceFiles) - println(changedSourceFiles.map(_.lastModified)) - println(changedSourceFiles.map(_.lastModified > oldestClassFileAge)) - println(oldestClassFile) - println(oldestClassFileAge) - println("-"*80) - */ - } - changedSourceFiles.nonEmpty - } - sourcesChanged || transitiveDependencies.map(_.updated).fold(false)(_ || _) + private object needsUpdateCache extends Cache[Boolean] + def needsUpdate: Boolean = { + needsUpdateCache( + lib.needsUpdate( sourceFiles, compileStatusFile ) + || transitiveDependencies.exists(_.needsUpdate) + ) } - private object compileCache extends Cache[File] - def compile: File = compileCache{ + private object compileCache extends Cache[Option[File]] + def compile: Option[File] = compileCache{ lib.compile( - updated, - sourceFiles, compileTarget, dependencyClasspath, scalacOptions, - zincVersion = zincVersion, scalaVersion = scalaVersion + needsUpdate, + sourceFiles, compileTarget, compileStatusFile, dependencyClasspath, scalacOptions, + context.classLoaderCache, zincVersion = zincVersion, scalaVersion = scalaVersion ) } def runClass: String = "Main" - def run: ExitCode = lib.runMainIfFound( runClass, context.args, classLoader ) + def run: ExitCode = lib.runMainIfFound( runClass, context.args, classLoader(context.classLoaderCache) ) def test: ExitCode = lib.test(context) diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index 5e0f5d3..9746d8c 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -8,10 +8,14 @@ class BuildBuild(context: Context) extends Build(context){ val managedBuild = { val managedContext = context.copy( cwd = managedBuildDirectory ) val cl = new cbt.URLClassLoader( - classpath, + exportedClasspath, classOf[BuildBuild].getClassLoader // FIXME: this looks wrong. Should be ClassLoader.getSystemClassLoader but that crashes ) - lib.create( lib.buildClassName )( managedContext )( cl ).asInstanceOf[Build] + cl + .loadClass(lib.buildClassName) + .getConstructor(classOf[Context]) + .newInstance(managedContext) + .asInstanceOf[Build] } override def triggerLoopFiles = super.triggerLoopFiles ++ managedBuild.triggerLoopFiles override def finalBuild = managedBuild.finalBuild diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 84a0100..e3a01c7 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -25,7 +25,7 @@ case class BuildDependency(context: Context) extends TriggerLoop{ def exportedJars = Seq() def dependencies = Seq(build) def triggerLoopFiles = root.triggerLoopFiles - final val updated = build.updated + override final val needsUpdate = build.needsUpdate def targetClasspath = ClassPath(Seq()) } /* diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index c3e38b6..59de98a 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -7,7 +7,7 @@ import org.eclipse.jgit.lib.Ref case class GitDependency( url: String, ref: String // example: git://github.com/cvogt/cbt.git#<some-hash> -)(implicit val logger: Logger) extends Dependency{ +)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache ) extends Dependency{ override def lib = new Lib(logger) // TODO: add support for authentication via ssh and/or https @@ -37,7 +37,7 @@ case class GitDependency( } val managedBuild = lib.loadDynamic( - Context( cwd = checkoutDirectory, args = Seq(), logger ) + Context( cwd = checkoutDirectory, args = Seq(), logger, classLoaderCache ) ) Seq( managedBuild ) } @@ -45,5 +45,5 @@ case class GitDependency( def exportedClasspath = ClassPath(Seq()) def exportedJars = Seq() private[cbt] def targetClasspath = exportedClasspath - def updated: Boolean = false + def needsUpdate: Boolean = false } diff --git a/stage2/Lib.scala b/stage2/Lib.scala index 60e7dd4..dd4a12f 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -8,13 +8,11 @@ import java.nio.file.{Path =>_,_} import java.nio.file.Files.readAllBytes import java.security.MessageDigest import java.util.jar._ +import java.lang.reflect.Method import scala.collection.immutable.Seq -import scala.reflect.runtime.{universe => ru} import scala.util._ -import ammonite.ops.{cwd => _,_} - // pom model case class Developer(id: String, name: String, timezone: String, url: URL) case class License(name: String, url: URL) @@ -55,28 +53,18 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } } - def compile( - updated: Boolean, - sourceFiles: Seq[File], compileTarget: File, dependenyClasspath: ClassPath, - compileArgs: Seq[String], zincVersion: String, scalaVersion: String - ): File = { - if(sourceFiles.nonEmpty) - lib.zinc( - updated, sourceFiles, compileTarget, dependenyClasspath, compileArgs - )( zincVersion = zincVersion, scalaVersion = scalaVersion ) - compileTarget - } - - def srcJar(sources: Seq[File], artifactId: String, version: String, jarTarget: File): File = { - val file = jarTarget ++ ("/"++artifactId++"-"++version++"-sources.jar") - lib.jarFile(file, sources) - file + def srcJar(sourceFiles: Seq[File], artifactId: String, version: String, jarTarget: File): Option[File] = { + lib.jarFile( + jarTarget ++ ("/"++artifactId++"-"++version++"-sources.jar"), + sourceFiles + ) } - def jar(artifactId: String, version: String, compileTarget: File, jarTarget: File): File = { - val file = jarTarget ++ ("/"++artifactId++"-"++version++".jar") - lib.jarFile(file, Seq(compileTarget)) - file + def jar(artifactId: String, version: String, compileTarget: File, jarTarget: File): Option[File] = { + lib.jarFile( + jarTarget ++ ("/"++artifactId++"-"++version++".jar"), + Seq(compileTarget) + ) } def docJar( @@ -87,29 +75,31 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ jarTarget: File, artifactId: String, version: String, - compileArgs: Seq[String] - ): File = { - mkdir(Path(apiTarget)) - if(sourceFiles.nonEmpty){ + compileArgs: Seq[String], + classLoaderCache: ClassLoaderCache + ): Option[File] = { + if(sourceFiles.isEmpty){ + None + } else { + apiTarget.mkdirs val args = Seq( // FIXME: can we use compiler dependency here? "-cp", dependencyClasspath.string, // FIXME: does this break for builds that don't have scalac dependencies? "-d", apiTarget.toString ) ++ compileArgs ++ sourceFiles.map(_.toString) logger.lib("creating docs for source files "+args.mkString(", ")) - trapExitCode{ - redirectOutToErr{ - runMain( - "scala.tools.nsc.ScalaDoc", - args, - ScalaDependencies(scalaVersion)(logger).classLoader - ) - } + redirectOutToErr{ + runMain( + "scala.tools.nsc.ScalaDoc", + args, + ScalaDependencies(scalaVersion)(logger).classLoader(classLoaderCache) + ) } + lib.jarFile( + jarTarget ++ ("/"++artifactId++"-"++version++"-javadoc.jar"), + Vector(apiTarget) + ) } - val docJar = jarTarget ++ ("/"++artifactId++"-"++version++"-javadoc.jar") - lib.jarFile(docJar, Vector(apiTarget)) - docJar } def test( context: Context ): ExitCode = { @@ -128,60 +118,66 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } // task reflection helpers - import ru._ - private lazy val anyRefMembers: Set[String] = ru.typeOf[AnyRef].members.toSet.map(taskName) - def taskNames(tpe: Type): Seq[String] = tpe.members.toVector.flatMap(lib.toTask).map(taskName).sorted - private def taskName(method: Symbol): String = method.name.decodedName.toString - def toTask(symbol: Symbol): Option[MethodSymbol] = { - Option(symbol) - .filter(_.isPublic) - .filter(_.isMethod) - .map(_.asMethod) - .filter(_.paramLists.flatten.size == 0) - .filterNot(taskName(_) contains "$") - .filterNot(t => anyRefMembers contains taskName(t)) - } + def tasks(cls:Class[_]): Map[String, Method] = + Stream + .iterate(cls.asInstanceOf[Class[Any]])(_.getSuperclass) + .takeWhile(_ != null) + .toVector + .dropRight(1) // drop Object + .reverse + .flatMap( + c => + c + .getDeclaredMethods + .filterNot( _.getName contains "$" ) + .filter{ m => + java.lang.reflect.Modifier.isPublic(m.getModifiers) + } + .filter( _.getParameterCount == 0 ) + .map(m => NameTransformer.decode(m.getName) -> m) + ).toMap - class ReflectBuild(val build: Build) extends ReflectObject(build){ - def usage: String = { - val baseTasks = lib.taskNames(ru.typeOf[Build]) - val thisTasks = lib.taskNames(subclassType) diff baseTasks + def taskNames(cls: Class[_]): Seq[String] = tasks(cls).keys.toVector.sorted + + def usage(buildClass: Class[_], context: Context): String = { + val baseTasks = lib.taskNames(classOf[Build]) + val thisTasks = lib.taskNames(buildClass) diff baseTasks + ( ( - ( - if( thisTasks.nonEmpty ){ - s"""Methods provided by Build ${build.context.cwd} + if( thisTasks.nonEmpty ){ + s"""Methods provided by Build ${context} ${thisTasks.mkString(" ")} """ - } else "" - ) ++ s"""Methods provided by CBT (but possibly overwritten) + } else "" + ) ++ s"""Methods provided by CBT (but possibly overwritten) ${baseTasks.mkString(" ")}""" ) ++ "\n" - } } + class ReflectBuild[T:scala.reflect.ClassTag](build: Build) extends ReflectObject(build){ + def usage = lib.usage(build.getClass, build.context) + } abstract class ReflectObject[T:scala.reflect.ClassTag](obj: T){ - lazy val mirror = ru.runtimeMirror(obj.getClass.getClassLoader) - lazy val subclassType = mirror.classSymbol(obj.getClass).toType def usage: String def callNullary( taskName: Option[String] ): Unit = { - taskName - .map{ n => subclassType.member(ru.TermName(n).encodedName) } - .filter(_ != ru.NoSymbol) - .flatMap(toTask _) - .map{ methodSymbol => - val result = mirror.reflect(obj).reflectMethod(methodSymbol)() - + val ts = tasks(obj.getClass) + taskName.map( NameTransformer.encode ).flatMap(ts.get).map{ method => + val result: Option[Any] = Option(method.invoke(obj)) // null in case of Unit + result.flatMap{ + case v: Option[_] => v + case other => Some(other) + }.map{ + value => // Try to render console representation. Probably not the best way to do this. - scala.util.Try( result.getClass.getDeclaredMethod("toConsole") ) match { - case scala.util.Success(m) => - println(m.invoke(result)) + scala.util.Try( value.getClass.getDeclaredMethod("toConsole") ) match { + case scala.util.Success(toConsole) => + println(toConsole.invoke(value)) - case scala.util.Failure(e) if e.getMessage contains "toConsole" => - result match { - case () => "" + case scala.util.Failure(e) if Option(e.getMessage).getOrElse("") contains "toConsole" => + value match { case ExitCode(code) => System.exit(code) case other => println( other.toString ) // no method .toConsole, using to String } @@ -189,16 +185,17 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ case scala.util.Failure(e) => throw e } - }.getOrElse{ - taskName.foreach{ n => - System.err.println(s"Method not found: $n") - System.err.println("") - } - System.err.println(usage) - taskName.foreach{ _ => - ExitCode.Failure - } + }.getOrElse("") + }.getOrElse{ + taskName.foreach{ n => + System.err.println(s"Method not found: $n") + System.err.println("") } + System.err.println(usage) + taskName.foreach{ _ => + ExitCode.Failure + } + } } } @@ -207,35 +204,41 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ def dirname(path: File): File = new File(realpath(path).string.stripSuffix("/").split("/").dropRight(1).mkString("/")) def nameAndContents(file: File) = basename(file) -> readAllBytes(Paths.get(file.toString)) - def jarFile( jarFile: File, files: Seq[File] ): Unit = { - logger.lib("Start packaging "++jarFile.string) - val manifest = new Manifest - manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0") - val jar = new JarOutputStream(new FileOutputStream(jarFile.toString), manifest) - - val names = for { - base <- files.filter(_.exists).map(realpath) - file <- listFilesRecursive(base) if file.isFile - } yield { - val name = if(base.isDirectory){ - file.toString stripPrefix base.toString - } else file.toString - val entry = new JarEntry( name ) - entry.setTime(file.lastModified) - jar.putNextEntry(entry) - jar.write( readAllBytes( Paths.get(file.toString) ) ) - jar.closeEntry - name - } + def jarFile( jarFile: File, files: Seq[File] ): Option[File] = { + if( files.isEmpty ){ + None + } else { + logger.lib("Start packaging "++jarFile.string) + val manifest = new Manifest + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0") + val jar = new JarOutputStream(new FileOutputStream(jarFile.toString), manifest) + + val names = for { + base <- files.filter(_.exists).map(realpath) + file <- listFilesRecursive(base) if file.isFile + } yield { + val name = if(base.isDirectory){ + file.toString stripPrefix base.toString + } else file.toString + val entry = new JarEntry( name ) + entry.setTime(file.lastModified) + jar.putNextEntry(entry) + jar.write( readAllBytes( Paths.get(file.toString) ) ) + jar.closeEntry + name + } - val duplicateFiles = (names diff names.distinct).distinct - assert( - duplicateFiles.isEmpty, - s"Conflicting file names when trying to create $jarFile: "++duplicateFiles.mkString(", ") - ) + val duplicateFiles = (names diff names.distinct).distinct + assert( + duplicateFiles.isEmpty, + s"Conflicting file names when trying to create $jarFile: "++duplicateFiles.mkString(", ") + ) - jar.close - logger.lib("Done packaging " ++ jarFile.toString) + jar.close + logger.lib("Done packaging " ++ jarFile.toString) + + Some(jarFile) + } } lazy val passphrase = @@ -321,8 +324,9 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ </dependencies> </project> val path = jarTarget.toString ++ ( "/" ++ artifactId ++ "-" ++ version ++ ".pom" ) - write.over(Path(path), "<?xml version='1.0' encoding='UTF-8'?>\n" ++ xml.toString) - new File(path) + val file = new File(path) + Files.write(file.toPath, ("<?xml version='1.0' encoding='UTF-8'?>\n" ++ xml.toString).getBytes) + file } def concurrently[T,R]( concurrencyEnabled: Boolean )( items: Seq[T] )( projection: T => R ): Seq[R] = { @@ -364,7 +368,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ val httpCon = url.openConnection.asInstanceOf[HttpURLConnection] httpCon.setDoOutput(true) httpCon.setRequestMethod("PUT") - val userPassword = read(Path(sonatypeLogin)).trim + val userPassword = new String(readAllBytes(sonatypeLogin.toPath)).trim val encoding = new sun.misc.BASE64Encoder().encode(userPassword.getBytes) httpCon.setRequestProperty("Authorization", "Basic " ++ encoding) httpCon.setRequestProperty("Content-Type", "application/binary") diff --git a/stage2/NameTransformer.scala b/stage2/NameTransformer.scala new file mode 100644 index 0000000..33489ca --- /dev/null +++ b/stage2/NameTransformer.scala @@ -0,0 +1,161 @@ +// Adapted from https://github.com/scala/scala/blob/5cb3d4ec14488ce2fc5a1cc8ebdd12845859c57d/src/library/scala/reflect/NameTransformer.scala +/* __ *\ +** ________ ___ / / ___ Scala API ** +** / __/ __// _ | / / / _ | (c) 2003-2013, LAMP/EPFL ** +** __\ \/ /__/ __ |/ /__/ __ | http://scala-lang.org/ ** +** /____/\___/_/ |_/____/_/ | | ** +** |/ ** +\* */ + +package cbt + +/** Provides functions to encode and decode Scala symbolic names. + * Also provides some constants. + */ +object NameTransformer { + // XXX Short term: providing a way to alter these without having to recompile + // the compiler before recompiling the compiler. + val MODULE_SUFFIX_STRING = sys.props.getOrElse("SCALA_MODULE_SUFFIX_STRING", "$") + val NAME_JOIN_STRING = sys.props.getOrElse("SCALA_NAME_JOIN_STRING", "$") + val MODULE_INSTANCE_NAME = "MODULE$" + val LOCAL_SUFFIX_STRING = " " + val SETTER_SUFFIX_STRING = "_$eq" + val TRAIT_SETTER_SEPARATOR_STRING = "$_setter_$" + + private val nops = 128 + private val ncodes = 26 * 26 + + private class OpCodes(val op: Char, val code: String, val next: OpCodes) + + private val op2code = new Array[String](nops) + private val code2op = new Array[OpCodes](ncodes) + private def enterOp(op: Char, code: String) = { + op2code(op.toInt) = code + val c = (code.charAt(1) - 'a') * 26 + code.charAt(2) - 'a' + code2op(c.toInt) = new OpCodes(op, code, code2op(c)) + } + + /* Note: decoding assumes opcodes are only ever lowercase. */ + enterOp('~', "$tilde") + enterOp('=', "$eq") + enterOp('<', "$less") + enterOp('>', "$greater") + enterOp('!', "$bang") + enterOp('#', "$hash") + enterOp('%', "$percent") + enterOp('^', "$up") + enterOp('&', "$amp") + enterOp('|', "$bar") + enterOp('*', "$times") + enterOp('/', "$div") + enterOp('+', "$plus") + enterOp('-', "$minus") + enterOp(':', "$colon") + enterOp('\\', "$bslash") + enterOp('?', "$qmark") + enterOp('@', "$at") + + /** Replace operator symbols by corresponding `\$opname`. + * + * @param name the string to encode + * @return the string with all recognized opchars replaced with their encoding + */ + def encode(name: String): String = { + var buf: StringBuilder = null + val len = name.length() + var i = 0 + while (i < len) { + val c = name charAt i + if (c < nops && (op2code(c.toInt) ne null)) { + if (buf eq null) { + buf = new StringBuilder() + buf.append(name.substring(0, i)) + } + buf.append(op2code(c.toInt)) + /* Handle glyphs that are not valid Java/JVM identifiers */ + } + else if (!Character.isJavaIdentifierPart(c)) { + if (buf eq null) { + buf = new StringBuilder() + buf.append(name.substring(0, i)) + } + buf.append("$u%04X".format(c.toInt)) + } + else if (buf ne null) { + buf.append(c) + } + i += 1 + } + if (buf eq null) name else buf.toString() + } + + /** Replace `\$opname` by corresponding operator symbol. + * + * @param name0 the string to decode + * @return the string with all recognized operator symbol encodings replaced with their name + */ + def decode(name0: String): String = { + //System.out.println("decode: " + name);//DEBUG + val name = if (name0.endsWith("<init>")) name0.stripSuffix("<init>") + "this" + else name0 + var buf: StringBuilder = null + val len = name.length() + var i = 0 + while (i < len) { + var ops: OpCodes = null + var unicode = false + val c = name charAt i + if (c == '$' && i + 2 < len) { + val ch1 = name.charAt(i+1) + if ('a' <= ch1 && ch1 <= 'z') { + val ch2 = name.charAt(i+2) + if ('a' <= ch2 && ch2 <= 'z') { + ops = code2op((ch1 - 'a') * 26 + ch2 - 'a') + while ((ops ne null) && !name.startsWith(ops.code, i)) ops = ops.next + if (ops ne null) { + if (buf eq null) { + buf = new StringBuilder() + buf.append(name.substring(0, i)) + } + buf.append(ops.op) + i += ops.code.length() + } + /* Handle the decoding of Unicode glyphs that are + * not valid Java/JVM identifiers */ + } else if ((len - i) >= 6 && // Check that there are enough characters left + ch1 == 'u' && + ((Character.isDigit(ch2)) || + ('A' <= ch2 && ch2 <= 'F'))) { + /* Skip past "$u", next four should be hexadecimal */ + val hex = name.substring(i+2, i+6) + try { + val str = Integer.parseInt(hex, 16).toChar + if (buf eq null) { + buf = new StringBuilder() + buf.append(name.substring(0, i)) + } + buf.append(str) + /* 2 for "$u", 4 for hexadecimal number */ + i += 6 + unicode = true + } catch { + case _:NumberFormatException => + /* `hex` did not decode to a hexadecimal number, so + * do nothing. */ + } + } + } + } + /* If we didn't see an opcode or encoded Unicode glyph, and the + buffer is non-empty, write the current character and advance + one */ + if ((ops eq null) && !unicode) { + if (buf ne null) + buf.append(c) + i += 1 + } + } + //System.out.println("= " + (if (buf == null) name else buf.toString()));//DEBUG + if (buf eq null) name else buf.toString() + } +} diff --git a/stage2/PackageBuild.scala b/stage2/PackageBuild.scala index 2866b7c..79e54a7 100644 --- a/stage2/PackageBuild.scala +++ b/stage2/PackageBuild.scala @@ -4,23 +4,23 @@ import scala.collection.immutable.Seq abstract class PackageBuild(context: Context) extends BasicBuild(context) with ArtifactInfo{ def `package`: Seq[File] = lib.concurrently( enableConcurrency )( Seq(() => jar, () => docJar, () => srcJar) - )( _() ) + )( _() ).flatten - private object cacheJarBasicBuild extends Cache[File] - def jar: File = cacheJarBasicBuild{ - lib.jar( artifactId, version, compile, jarTarget ) + private object cacheJarBasicBuild extends Cache[Option[File]] + def jar: Option[File] = cacheJarBasicBuild{ + compile.flatMap( lib.jar( artifactId, version, _, jarTarget ) ) } - private object cacheSrcJarBasicBuild extends Cache[File] - def srcJar: File = cacheSrcJarBasicBuild{ + private object cacheSrcJarBasicBuild extends Cache[Option[File]] + def srcJar: Option[File] = cacheSrcJarBasicBuild{ lib.srcJar( sourceFiles, artifactId, version, scalaTarget ) } - private object cacheDocBasicBuild extends Cache[File] - def docJar: File = cacheDocBasicBuild{ - lib.docJar( scalaVersion, sourceFiles, dependencyClasspath, apiTarget, jarTarget, artifactId, version, scalacOptions ) + private object cacheDocBasicBuild extends Cache[Option[File]] + def docJar: Option[File] = cacheDocBasicBuild{ + lib.docJar( scalaVersion, sourceFiles, dependencyClasspath, apiTarget, jarTarget, artifactId, version, scalacOptions, context.classLoaderCache ) } - override def jars = jar +: dependencyJars - override def exportedJars: Seq[File] = Seq(jar) + override def jars = jar.toVector ++ dependencyJars + override def exportedJars: Seq[File] = jar.toVector } diff --git a/stage2/Scaffold.scala b/stage2/Scaffold.scala index e181ebf..3dcb9ae 100644 --- a/stage2/Scaffold.scala +++ b/stage2/Scaffold.scala @@ -1,13 +1,14 @@ package cbt import java.io._ +import java.nio.file._ import java.net._ -import ammonite.ops.{cwd => _,_} - trait Scaffold{ def logger: Logger private def createFile( projectDirectory: File, fileName: String, code: String ){ - write( Path( projectDirectory.string ++ "/" ++ fileName ), code ) + val outputFile = projectDirectory ++ ("/" ++ fileName) + outputFile.getParentFile.mkdirs + Files.write( ( outputFile ).toPath, code.getBytes, StandardOpenOption.CREATE_NEW ) import scala.Console._ println( GREEN ++ "Created " ++ fileName ++ RESET ) } diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index 4145e55..e893a06 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -8,26 +8,24 @@ import scala.collection.immutable.Seq import cbt.paths._ +object Stage2 extends Stage2Base{ + def run( args: Stage2Args ): Unit = { + import args.logger -object Stage2{ - def main(args: Array[String]): Unit = { - val init = new Init(args) - import init._ + val lib = new Lib(args.logger) - val lib = new Lib(init.logger) - - init.logger.stage2(s"[$now] Stage2 start") - val loop = argsV.lift(1) == Some("loop") - val direct = argsV.lift(1) == Some("direct") + logger.stage2(s"[$now] Stage2 start") + val loop = args.args.lift(0) == Some("loop") + val direct = args.args.lift(0) == Some("direct") val taskIndex = if (loop || direct) { - 2 - } else { 1 + } else { + 0 } - val task = argsV.lift( taskIndex ) + val task = args.args.lift( taskIndex ) - val context = Context( new File(argsV(0)), argsV.drop( taskIndex + 1 ), logger ) + val context = Context( args.cwd, args.args.drop( taskIndex ), logger, /*args.cbtHasChanged,*/ new ClassLoaderCache(logger) ) val first = lib.loadRoot( context ) val build = first.finalBuild @@ -47,14 +45,15 @@ object Stage2{ scala.util.control.Breaks.break case file if triggerFiles.exists(file.toString startsWith _.toString) => - val reflectBuild = new lib.ReflectBuild( lib.loadDynamic(context) ) - logger.loop(s"Re-running $task for " ++ reflectBuild.build.projectDirectory.toString) + val build = lib.loadDynamic(context) + val reflectBuild = new lib.ReflectBuild( build ) + logger.loop(s"Re-running $task for " ++ build.projectDirectory.toString) reflectBuild.callNullary(task) } } else { new lib.ReflectBuild(build).callNullary(task) } - init.logger.stage2(s"[$now] Stage2 end") + logger.stage2(s"[$now] Stage2 end") } } diff --git a/stage2/mixins.scala b/stage2/mixins.scala index 2b38cdf..c3a57da 100644 --- a/stage2/mixins.scala +++ b/stage2/mixins.scala @@ -20,16 +20,13 @@ trait ScalaTest extends Build with Test{ "org.scalatest" %% "scalatest" % scalaTestVersion ) ++ super.dependencies - // workaround probable ScalaTest bug throwing away the outer classloader. Not caching doesn't nest them. - override def cacheDependencyClassLoader = false - override def run: ExitCode = { val discoveryPath = compile.toString++"/" context.logger.lib("discoveryPath: " ++ discoveryPath) lib.runMain( "org.scalatest.tools.Runner", Seq("-R", discoveryPath, "-oF") ++ context.args.drop(1), - classLoader + classLoader(context.classLoaderCache) ) } } diff --git a/test/simple/Main.scala b/test/simple/Main.scala index 43542ff..1c423ca 100644 --- a/test/simple/Main.scala +++ b/test/simple/Main.scala @@ -1,4 +1,6 @@ import ai.x.diff +import org.eclipse.jgit.lib.Ref +import com.spotify.missinglink.ArtifactLoader object Main extends App{ println(diff.DiffShow.diff("a","b")) } diff --git a/test/simple/build/build.scala b/test/simple/build/build.scala index f3efc19..d3887b3 100644 --- a/test/simple/build/build.scala +++ b/test/simple/build/build.scala @@ -6,7 +6,9 @@ class Build(context: cbt.Context) extends BasicBuild(context){ ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), JavaDependency("joda-time", "joda-time", "2.9.2"), GitDependency("https://github.com/xdotai/diff.git", "2e275642041006ff39efde22da7742c2e9a0f63f"), - // the below tests pom inheritance with dependencyManagement and variable substitution - JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") + // the below tests pom inheritance with dependencyManagement and variable substitution for pom properties + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), + // the below tests pom inheritance with variable substitution for pom xml tag contents + JavaDependency("com.spotify", "missinglink-core", "0.1.1") ) ++ super.dependencies } diff --git a/test/test.scala b/test/test.scala index 47bd28b..7261287 100644 --- a/test/test.scala +++ b/test/test.scala @@ -4,9 +4,9 @@ import scala.collection.immutable.Seq // micro framework object Main{ - def main(args: Array[String]): Unit = { - val init = new Init(args) - implicit val logger: Logger = init.logger + def main(_args: Array[String]): Unit = { + val args = new Stage1ArgsParser(_args.toVector) + implicit val logger: Logger = new Logger(args.enabledLoggers) var successes = 0 var failures = 0 @@ -24,9 +24,9 @@ object Main{ }.get } - def runCbt(path: String, args: Seq[String])(implicit logger: Logger): Result = { + def runCbt(path: String, _args: Seq[String])(implicit logger: Logger): Result = { import java.io._ - val allArgs: Seq[String] = ((cbtHome.string ++ "/cbt") +: "direct" +: (args ++ init.propsRaw)) + val allArgs: Seq[String] = ((cbtHome.string ++ "/cbt") +: "direct" +: (_args ++ args.propsRaw)) logger.test(allArgs.toString) val pb = new ProcessBuilder( allArgs :_* ) pb.directory(cbtHome ++ ("/test/" ++ path)) @@ -41,7 +41,7 @@ object Main{ } case class Result(exit0: Boolean, out: String, err: String) def assertSuccess(res: Result, msg: => String)(implicit logger: Logger) = { - assert(res.exit0, msg + res.toString) + assert(res.exit0, msg ++ res.toString) } // tests @@ -49,19 +49,19 @@ object Main{ val usageString = "Methods provided by CBT" val res = runCbt(path, Seq()) logger.test(res.toString) - val debugToken = "usage " + path +" " + val debugToken = "usage " ++ path ++ " " assertSuccess(res,debugToken) - assert(res.out == "", debugToken+ res.toString) - assert(res.err contains usageString, debugToken+res.toString) + assert(res.out == "", debugToken ++ res.toString) + assert(res.err contains usageString, debugToken ++ res.toString) } def compile(path: String)(implicit logger: Logger) = { val res = runCbt(path, Seq("compile")) - val debugToken = "compile " + path +" " + val debugToken = "compile " ++ path ++ " " assertSuccess(res,debugToken) // assert(res.err == "", res.err) // FIXME: enable this } - logger.test( "Running tests " ++ args.toList.toString ) + logger.test( "Running tests " ++ _args.toList.toString ) usage("nothing") compile("nothing") @@ -71,7 +71,7 @@ object Main{ compile("simple") { - val noContext = Context(cbtHome ++ "/test/nothing", Seq(), logger) + val noContext = Context(cbtHome ++ "/test/nothing", Seq(), logger, new ClassLoaderCache(logger)) val b = new Build(noContext){ override def dependencies = Seq( JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), |