diff options
author | Jan Christopher Vogt <oss.nsp@cvogt.org> | 2016-04-02 16:06:40 -0400 |
---|---|---|
committer | Jan Christopher Vogt <oss.nsp@cvogt.org> | 2016-04-02 16:06:40 -0400 |
commit | 63b54f79c10854e38b2a4a43ee39f508458e280f (patch) | |
tree | 6a5791efedc2d297cfac1ad8bbaac0b090105149 | |
parent | 16b02cf34078113c833225297b686752aa26b407 (diff) | |
parent | efe68c7e710aa8c54144715408b7faca36f52c27 (diff) | |
download | cbt-63b54f79c10854e38b2a4a43ee39f508458e280f.tar.gz cbt-63b54f79c10854e38b2a4a43ee39f508458e280f.tar.bz2 cbt-63b54f79c10854e38b2a4a43ee39f508458e280f.zip |
Rewrite CBT's classloading and dependency classloaders, fetch zinc early and various smaller changes
Rewrite CBT's classloading and dependency classloaders, fetch zinc early and various smaller changes
37 files changed, 1292 insertions, 714 deletions
diff --git a/bootstrap_scala/BootstrapScala.java b/bootstrap_scala/BootstrapScala.java deleted file mode 100644 index e2d7a5a..0000000 --- a/bootstrap_scala/BootstrapScala.java +++ /dev/null @@ -1,85 +0,0 @@ -import java.io.File; -import java.io.InputStream; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import java.util.Iterator; -import javax.xml.bind.annotation.adapters.HexBinaryAdapter; - -/** - * This file class allows bootstrapping out of Java into Scala. It downloads the Scala jars for the - * version number given as the first argument into the directory given as the second argument and - * returns a classpath String. - */ -public class BootstrapScala { - - public final static Dependency[] dependencies(String target, String scalaVersion) throws MalformedURLException { - return new Dependency[] { - Dependency.scala(target, scalaVersion, "library", "DDD5A8BCED249BEDD86FB4578A39B9FB71480573"), - Dependency.scala(target, scalaVersion, "compiler","FE1285C9F7B58954C5EF6D80B59063569C065E9A"), - Dependency.scala(target, scalaVersion, "reflect", "B74530DEEBA742AB4F3134DE0C2DA0EDC49CA361"), - new Dependency(target, "modules/scala-xml_2.11/1.0.5", "scala-xml_2.11-1.0.5", "77ac9be4033768cf03cc04fbd1fc5e5711de2459") - }; - } - - public static void main(String args[]) throws IOException, NoSuchAlgorithmException { - - if(args.length < 2){ - System.err.println("Usage: bootstrap_scala <scala version> <download directory>"); - System.exit(1); - } - - Dependency[] ds = dependencies( args[1], args[0] ); - new File(args[1]).mkdirs(); - for (Dependency d: ds) { - download( d.url, d.path, d.hash ); - } - - // Join dep. paths as a classpath - String classpath = ""; - Iterator<Dependency> depsIter = Arrays.asList(ds).iterator(); - while (depsIter.hasNext()) { - Dependency dep = depsIter.next(); - classpath += dep.path.toString(); - if (depsIter.hasNext()) { - classpath += File.pathSeparator; - } - } - - System.out.println(classpath); - - } - - public static void download(URL urlString, Path target, String sha1) throws IOException, NoSuchAlgorithmException { - final Path unverified = Paths.get(target+".unverified"); - if(!Files.exists(target)) { - new File(target.toString()).getParentFile().mkdirs(); - System.err.println("downloading " + urlString); - System.err.println("to " + target); - final InputStream stream = urlString.openStream(); - Files.copy(stream, unverified, StandardCopyOption.REPLACE_EXISTING); - stream.close(); - final String checksum = sha1(Files.readAllBytes(unverified)); - if(sha1 == null || sha1.toUpperCase().equals(checksum)) { - Files.move(unverified, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); - } else { - System.err.println(target + " checksum does not match.\nExpected: |" + sha1 + "|\nFound: |" + checksum + "|"); - System.exit(1); - } - } - } - - public static String sha1(byte[] bytes) throws NoSuchAlgorithmException { - final MessageDigest sha1 = MessageDigest.getInstance("SHA1"); - sha1.update(bytes, 0, bytes.length); - return (new HexBinaryAdapter()).marshal(sha1.digest()); - } - -} diff --git a/bootstrap_scala/Dependency.java b/bootstrap_scala/Dependency.java deleted file mode 100644 index 571047b..0000000 --- a/bootstrap_scala/Dependency.java +++ /dev/null @@ -1,29 +0,0 @@ -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; - -class Dependency { - - final URL url; - final Path path; - final String hash; - - public Dependency(String target, String folder, String file, String hash) throws MalformedURLException { - this.path = Paths.get(target + file + ".jar"); - this.url = new URL("https://repo1.maven.org/maven2/org/scala-lang/" + folder + "/" + file + ".jar"); - this.hash = hash; - } - - // scala-lang dependency - public static Dependency scala(String target, String scalaVersion, String scalaModule, String hash) - throws MalformedURLException { - return new Dependency( - target, - "scala-" + scalaModule + "/" + scalaVersion, - "scala-" + scalaModule + "-" + scalaVersion, - hash - ); - } - -} diff --git a/bootstrap_scala/bootstrap_scala b/bootstrap_scala/bootstrap_scala deleted file mode 100755 index b004c8d..0000000 --- a/bootstrap_scala/bootstrap_scala +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -_DIR=$(dirname $(readlink "$0") 2>/dev/null || dirname "$0" 2>/dev/null ) -DIR=$(dirname $($_DIR/../realpath/realpath.sh $0)) -JAVAC="javac -Xlint:deprecation" -TARGET=$DIR/target -CLASSES=$TARGET/classes/ -VERSION=$1 -CACHE=$DIR/cache/$VERSION/ - -COMPILER_JAR=scala-compiler-$VERSION.jar -LIBRARY_JAR=scala-library-$VERSION.jar -REFLECT_JAR=scala-reflect-$VERSION.jar -XML_JAR=scala-xml_2.11-1.0.5.jar # this is a bit fishy, because it doesn't take version into account - -mkdir -p $CLASSES - -if [ ! -f $CACHE$COMPILER_JAR ] || [ ! -f $CACHE$LIBRARY_JAR ] || [ ! -f $CACHE$REFLECT_JAR ]\ - || [ ! -f $CACHE$XML_JAR ] || [ $DIR/BootstrapScala.java -nt $CLASSES/BootstrapScala.class ] || [ $DIR/Dependency.java -nt $CLASSES/Dependency.class ] -then - echo "Compiling cbt/bootstrap_scala" 1>&2 - $JAVAC -d $CLASSES $DIR/BootstrapScala.java $DIR/Dependency.java - java -cp $CLASSES BootstrapScala $1 $CACHE -else - # for speedup - echo `for f in $CACHE*; do printf "$f "; done`|tr " " ":" -fi diff --git a/build/build.scala b/build/build.scala new file mode 100644 index 0000000..aa5d27a --- /dev/null +++ b/build/build.scala @@ -0,0 +1,18 @@ +import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: Context) extends BasicBuild(context){ + // FIXME: somehow consolidate this with cbt's own boot-strapping from source. + override def dependencies = super.dependencies ++ Seq( + JavaDependency("org.scala-lang","scala-library",constants.scalaVersion), + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), + JavaDependency("com.typesafe.zinc","zinc","0.3.9"), + ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5") + ) + override def sources = Seq( + "nailgun_launcher", "stage1", "stage2" + ).map(d => projectDirectory ++ ("/" + d)) +} @@ -78,12 +78,9 @@ log "Find out real path. Build realpath if needed." $* export CBT_HOME=$(dirname $($_DIR/realpath/realpath.sh $0)) -export SCALA_VERSION="2.11.8" export NAILGUN=$CBT_HOME/nailgun_launcher/ -export STAGE1=$CBT_HOME/stage1/ export TARGET=target/scala-2.11/classes/ mkdir -p $NAILGUN$TARGET -mkdir -p $STAGE1$TARGET nailgun_out=$NAILGUN/target/nailgun.stdout.log nailgun_err=$NAILGUN/target/nailgun.strerr.log @@ -135,18 +132,6 @@ if [ $use_nailgun -eq 0 ] && [ ! $server_up -eq 0 ]; then ng-server 127.0.0.1:$NAILGUN_PORT >> $nailgun_out 2>> $nailgun_err & fi -log "Downloading Scala jars if necessary..." $* -export SCALA_CLASSPATH=`$CBT_HOME/bootstrap_scala/bootstrap_scala $SCALA_VERSION` -if [ ! $? -eq 0 ]; then echo "Problem with bootstrap_scala" 1>&2; exit 1; fi - -SCALAC="java -Xmx256M -Xms32M\ - -Xbootclasspath/a:$SCALA_CLASSPATH\ - -Dscala.usejavacp=true\ - -Denv.emacs=\ - scala.tools.nsc.Main\ - -deprecation\ - -feature" - stage1 () { log "Checking for changes in cbt/nailgun_launcher" $* NAILGUN_INDICATOR=$NAILGUN$TARGET/cbt/NailgunLauncher.class @@ -156,9 +141,9 @@ stage1 () { done compiles=0 if [ $changed -eq 1 ]; then - rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files + #rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files echo "Compiling cbt/nailgun_launcher" 1>&2 - javac -Xlint:deprecation -d $NAILGUN$TARGET `ls $NAILGUN*.java` + javac -Xlint:deprecation -Xlint:unchecked -d $NAILGUN$TARGET `ls $NAILGUN*.java` compiles=$? if [ $compiles -ne 0 ]; then rm $NAILGUN$TARGET/cbt/*.class 2>/dev/null # triggers recompilation next time. @@ -172,45 +157,20 @@ stage1 () { fi fi - log "Checking for changes in cbt/stage1" $* - STAGE1_INDICATOR=$STAGE1$TARGET/cbt/Stage1.class - changed2=0 - for file in `ls $STAGE1*.scala`; do - if [ $file -nt $STAGE1_INDICATOR ]; then changed2=1; fi - done - compiles2=0 - - if [ $changed2 -eq 1 ]; then - rm $STAGE1$TARGET/cbt/*.class 2>/dev/null # defensive delete of potentially broken class files - echo "Compiling cbt/stage1" 1>&2 - $SCALAC -cp $NAILGUN$TARGET -d $STAGE1$TARGET `ls $STAGE1/*.scala` - compiles2=$? - if [ $compiles2 -ne 0 ]; then - rm $STAGE1$TARGET/cbt/*.class 2>/dev/null # triggers recompilation next time. - break - fi - fi - log "run CBT and loop if desired. This allows recompiling CBT itself as part of compile looping." $* - if [ "$1" = "admin" ] || [ "$2" = "admin" ]; then - mainClass=cbt.AdminStage1 - else - mainClass=cbt.Stage1 - fi - - CP=$STAGE1$TARGET:$SCALA_CLASSPATH + if [ $use_nailgun -eq 1 ] then log "Running JVM directly" $* # -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:5005 - java -cp $NAILGUN$TARGET cbt.NailgunLauncher $mainClass $CP "$CWD" $* + java -cp $NAILGUN$TARGET cbt.NailgunLauncher "$CWD" $* else log "Running via nailgun." $* for i in 0 1 2 3 4 5 6 7 8 9; do log "Adding classpath." $* $NG ng-cp $NAILGUN$TARGET >> $nailgun_out 2>> $nailgun_err log "Checking if nailgun is up yet." $* - $NG cbt.NailgunLauncher cbt.CheckAlive $CP "$CWD" $* >> $nailgun_out 2>> $nailgun_err + $NG cbt.NailgunLauncher check-alive >> $nailgun_out 2>> $nailgun_err alive=$? if [ $alive -eq 131 ] || [ $alive -eq 33 ]; then # the 33 is not working right now @@ -219,17 +179,17 @@ stage1 () { break else log "Nope. Sleeping for 0.5 seconds" $* - if [ "$i" -gt "1" ]; then - echo "Waiting for nailgun to start... (For problems try -Dlog=nailgun or check logs in cbt/nailgun_launcher/target/*.log)" 1>&2 - fi + #if [ "$i" -gt 1 ]; then + # echo "Waiting for nailgun to start... (In case of problems try -Dlog=nailgun or check logs in cbt/nailgun_launcher/target/*.log)" 1>&2 + #fi fi - sleep 0.5 + sleep 0.3 done - log "Running $mainClass via Nailgun." $* - $NG cbt.NailgunLauncher $mainClass $CP "$CWD" $* + log "Running CBT via Nailgun." $* + $NG cbt.NailgunLauncher "$CWD" $* fi exitCode=$? - log "Done running $mainClass." $* + log "Done running CBT." $* } while true; do @@ -240,7 +200,7 @@ while true; do echo "======= Restarting CBT =======" 1>&2 done -if [ $compiles -ne 0 ] || [ $compiles2 -ne 0 ]; then +if [ $compiles -ne 0 ]; then exitCode=1 fi diff --git a/nailgun_launcher/CBTUrlClassLoader.java b/nailgun_launcher/CBTUrlClassLoader.java new file mode 100644 index 0000000..88bf4a3 --- /dev/null +++ b/nailgun_launcher/CBTUrlClassLoader.java @@ -0,0 +1,35 @@ +package cbt; +import java.io.*; +import java.net.*; +import java.util.*; +class CbtURLClassLoader extends java.net.URLClassLoader{ + public String toString(){ + return ( + super.toString() + + "(\n " + + Arrays.toString(getURLs()) + + ",\n " + + String.join("\n ",getParent().toString().split("\n")) + + "\n)" + ); + } + public Class loadClass(String name) throws ClassNotFoundException{ + //System.out.println("loadClass("+name+") on \n"+this); + return super.loadClass(name); + } + void assertExist(URL[] urls){ + for(URL url: urls){ + if(!new File(url.getPath()).exists()){ + throw new AssertionError("File does not exist when trying to create CbtURLClassLoader: "+url); + } + } + } + public CbtURLClassLoader(URL[] urls, ClassLoader parent){ + super(urls, parent); + assertExist(urls); + } + public CbtURLClassLoader(URL[] urls){ + super(urls); + assertExist(urls); + } +}
\ No newline at end of file diff --git a/nailgun_launcher/EarlyDependencies.java b/nailgun_launcher/EarlyDependencies.java new file mode 100644 index 0000000..1e129c7 --- /dev/null +++ b/nailgun_launcher/EarlyDependencies.java @@ -0,0 +1,99 @@ +// This file was auto-generated using `cbt admin cbtEarlyDependencies` +package cbt; +import java.io.*; +import java.nio.file.*; +import java.net.*; +import java.security.*; +import static cbt.NailgunLauncher.*; + +class EarlyDependencies{ + + /** ClassLoader for stage1 */ + ClassLoader stage1; + /** ClassLoader for zinc */ + ClassLoader zinc; + + String scalaReflect_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; + String scalaCompiler_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; + String scalaXml_1_0_5_File = MAVEN_CACHE + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"; + String scalaLibrary_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"; + String zinc_0_3_9_File = MAVEN_CACHE + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"; + String incrementalCompiler_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"; + String compilerInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"; + String scalaCompiler_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"; + String sbtInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"; + String scalaReflect_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"; + String scalaLibrary_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"; + + public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ + download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"), Paths.get(scalaReflect_2_11_8_File), "b74530deeba742ab4f3134de0c2da0edc49ca361"); + download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); + + // org.scala-lang:scala-library:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"), Paths.get(scalaLibrary_2_10_5_File), "57ac67a6cf6fd591e235c62f8893438e8d10431d"); + ClassLoader scalaLibrary_2_10_5_ = cachePut( + classLoader( scalaLibrary_2_10_5_File ), + scalaLibrary_2_10_5_File + ); + + // org.scala-lang:scala-reflect:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"), Paths.get(scalaReflect_2_10_5_File), "7392facb48876c67a89fcb086112b195f5f6bbc3"); + ClassLoader scalaReflect_2_10_5_ = cachePut( + classLoader( scalaReflect_2_10_5_File, scalaLibrary_2_10_5_ ), + scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:sbt-interface:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"), Paths.get(sbtInterface_0_13_9_File), "29848631415402c81b732e919be88f268df37250"); + ClassLoader sbtInterface_0_13_9_ = cachePut( + classLoader( sbtInterface_0_13_9_File, scalaReflect_2_10_5_ ), + sbtInterface_0_13_9_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // org.scala-lang:scala-compiler:2.10.5 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"), Paths.get(scalaCompiler_2_10_5_File), "f0f5bb444ca26a6e489af3dd35e24f7e2d2d118e"); + ClassLoader scalaCompiler_2_10_5_ = cachePut( + classLoader( scalaCompiler_2_10_5_File, sbtInterface_0_13_9_ ), + sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:compiler-interface:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"), Paths.get(compilerInterface_0_13_9_File), "2311addbed1182916ad00f83c57c0eeca1af382b"); + ClassLoader compilerInterface_0_13_9_ = cachePut( + classLoader( compilerInterface_0_13_9_File, scalaCompiler_2_10_5_ ), + compilerInterface_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.sbt:incremental-compiler:0.13.9 + download(new URL(MAVEN_URL + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"), Paths.get(incrementalCompiler_0_13_9_File), "fbbf1cadbed058aa226643e83543c35de43b13f0"); + ClassLoader incrementalCompiler_0_13_9_ = cachePut( + classLoader( incrementalCompiler_0_13_9_File, compilerInterface_0_13_9_ ), + compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // com.typesafe.zinc:zinc:0.3.9 + download(new URL(MAVEN_URL + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"), Paths.get(zinc_0_3_9_File), "46a4556d1f36739879f4b2cc19a73d12b3036e9a"); + ClassLoader zinc_0_3_9_ = cachePut( + classLoader( zinc_0_3_9_File, incrementalCompiler_0_13_9_ ), + compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, zinc_0_3_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File + ); + + // org.scala-lang:scala-library:2.11.8 + download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); + ClassLoader scalaLibrary_2_11_8_ = cachePut( + classLoader( scalaLibrary_2_11_8_File ), + scalaLibrary_2_11_8_File + ); + + // org.scala-lang.modules:scala-xml_2.11:1.0.5 + download(new URL(MAVEN_URL + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); + ClassLoader scalaXml_1_0_5_ = cachePut( + classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), + scalaXml_1_0_5_File, scalaLibrary_2_11_8_File + ); + + stage1 = scalaXml_1_0_5_; + + zinc = zinc_0_3_9_; + } +} diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index 11a8680..50a3c91 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -4,53 +4,211 @@ import java.lang.reflect.*; import java.net.*; import java.nio.*; import java.nio.file.*; +import static java.io.File.pathSeparator; +import java.security.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import javax.xml.bind.annotation.adapters.HexBinaryAdapter; /** * This launcher allows to start the JVM without loading anything else permanently into its * classpath except for the launcher itself. That's why it is written in Java without * dependencies outside the JDK. - * - * The main method loads the given class from the given class path, calls it's main - * methods passing in the additional arguments. */ public class NailgunLauncher{ + public static String SCALA_VERSION = "2.11.8"; + public static String SCALA_XML_VERSION = "1.0.5"; + public static String ZINC_VERSION = "0.3.9"; + + public static String CBT_HOME = System.getenv("CBT_HOME"); + public static String NAILGUN = System.getenv("NAILGUN"); + public static String TARGET = System.getenv("TARGET"); + public static String STAGE1 = CBT_HOME + "/stage1/"; + public static String MAVEN_CACHE = CBT_HOME + "/cache/maven"; + public static String MAVEN_URL = "https://repo1.maven.org/maven2"; /** * Persistent cache for caching classloaders for the JVM life time. Can be used as needed by user * code to improve startup time. */ - public static ConcurrentHashMap<String,ClassLoader> classLoaderCache = - new ConcurrentHashMap<String,ClassLoader>(); + public static ConcurrentHashMap<String, Object> classLoaderCacheKeys = new ConcurrentHashMap<String,Object>(); + public static ConcurrentHashMap<Object, ClassLoader> classLoaderCacheValues = new ConcurrentHashMap<Object,ClassLoader>(); public static SecurityManager defaultSecurityManager = System.getSecurityManager(); + public static long lastSuccessfullCompile = 0; + static ClassLoader stage1classLoader = null; + public static ClassLoader stage2classLoader = null; + public static void main(String[] args) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException, - MalformedURLException { - if (args.length < 3) { - System.out.println("usage: <main class> <class path> <... args>"); - } else { - // TODO: cache this classloader, but invalidate on changes - String[] cp = args[1].split(File.pathSeparator); - - URL[] urls = new URL[cp.length]; - for(int i = 0; i < cp.length; i++){ - urls[i] = new URL("file:"+cp[i]); + MalformedURLException, + IOException, + NoSuchAlgorithmException { + long now = System.currentTimeMillis(); + //System.err.println("ClassLoader: "+stage1classLoader); + //System.err.println("lastSuccessfullCompile: "+lastSuccessfullCompile); + //System.err.println("now: "+now); + + _assert(CBT_HOME != null, CBT_HOME); + _assert(NAILGUN != null, NAILGUN); + _assert(TARGET != null, TARGET); + _assert(STAGE1 != null, STAGE1); + + if(args[0].equals("check-alive")){ + System.exit(33); + return; + } + + List<File> stage1SourceFiles = new ArrayList<File>(); + for( File f: new File(STAGE1).listFiles() ){ + if( f.isFile() && f.toString().endsWith(".scala") ){ + stage1SourceFiles.add(f); } + } + + Boolean stage1SourcesChanged = false; + for( File file: stage1SourceFiles ){ + if( file.lastModified() > lastSuccessfullCompile ){ + stage1SourcesChanged = true; + //System.err.println("File change: "+file.lastModified()); + break; + } + } - String[] newArgs = new String[args.length-2]; - for(int i = 0; i < args.length-2; i++){ - newArgs[i] = args[i+2]; + if(stage1SourcesChanged || stage1classLoader == null){ + EarlyDependencies earlyDeps = new EarlyDependencies(); + int exitCode = zinc(earlyDeps, stage1SourceFiles); + if( exitCode == 0 ){ + lastSuccessfullCompile = now; + } else { + System.exit( exitCode ); } - new URLClassLoader( urls ) - .loadClass(args[0]) + ClassLoader nailgunClassLoader; + if( classLoaderCacheKeys.containsKey( NAILGUN+TARGET ) ){ + nailgunClassLoader = cacheGet( NAILGUN+TARGET ); + } else { + nailgunClassLoader = cachePut( classLoader(NAILGUN+TARGET, earlyDeps.stage1), NAILGUN+TARGET ); // FIXME: key is wrong here, should be full CP + } + + stage1classLoader = classLoader(STAGE1+TARGET, nailgunClassLoader); + stage2classLoader = null; + } + + try{ + Integer exitCode = + (Integer) stage1classLoader + .loadClass("cbt.Stage1") + .getMethod("run", String[].class, ClassLoader.class, Boolean.class) + .invoke( null, (Object) args, stage1classLoader, stage1SourcesChanged); + System.exit(exitCode); + }catch(Exception e){ + System.err.println(stage1classLoader); + throw e; + } + } + + public static void _assert(Boolean condition, Object msg){ + if(!condition){ + throw new AssertionError("Assertion failed: "+msg); + } + } + + public static int runMain(String cls, String[] args, ClassLoader cl) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ + try{ + System.setSecurityManager( new TrapSecurityManager() ); + cl.loadClass(cls) .getMethod("main", String[].class) - .invoke( null/* _cls.newInstance()*/, (Object) newArgs ); + .invoke( null, (Object) args); + return 0; + }catch( InvocationTargetException exception ){ + Throwable cause = exception.getCause(); + if(cause instanceof TrappedExitCode){ + return ((TrappedExitCode) cause).exitCode; + } + throw exception; + } finally { + System.setSecurityManager(NailgunLauncher.defaultSecurityManager); + } + } + + static int zinc( EarlyDependencies earlyDeps, List<File> sourceFiles ) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ + String cp = NAILGUN+TARGET + pathSeparator + earlyDeps.scalaXml_1_0_5_File + pathSeparator + earlyDeps.scalaLibrary_2_11_8_File; + List<String> zincArgs = new ArrayList<String>( + Arrays.asList( + new String[]{ + "-scala-compiler", earlyDeps.scalaCompiler_2_11_8_File, + "-scala-library", earlyDeps.scalaLibrary_2_11_8_File, + "-scala-extra", earlyDeps.scalaReflect_2_11_8_File, + "-sbt-interface", earlyDeps.sbtInterface_0_13_9_File, + "-compiler-interface", earlyDeps.compilerInterface_0_13_9_File, + "-cp", cp, + "-d", STAGE1+TARGET + } + ) + ); + + for( File f: sourceFiles ){ + zincArgs.add(f.toString()); + } + + PrintStream oldOut = System.out; + try{ + System.setOut(System.err); + return runMain( "com.typesafe.zinc.Main", zincArgs.toArray(new String[zincArgs.size()]), earlyDeps.zinc ); + } finally { + System.setOut(oldOut); } } + + static ClassLoader classLoader( String file ) throws MalformedURLException{ + return new CbtURLClassLoader( + new URL[]{ new URL("file:"+file) } + ); + } + static ClassLoader classLoader( String file, ClassLoader parent ) throws MalformedURLException{ + return new CbtURLClassLoader( + new URL[]{ new URL("file:"+file) }, parent + ); + } + static ClassLoader cacheGet( String key ){ + return classLoaderCacheValues.get( + classLoaderCacheKeys.get( key ) + ); + } + static ClassLoader cachePut( ClassLoader classLoader, String... jars ){ + String key = String.join( pathSeparator, jars ); + Object keyObject = new Object(); + classLoaderCacheKeys.put( key, keyObject ); + classLoaderCacheValues.put( keyObject, classLoader ); + return classLoader; + } + + public static void download(URL urlString, Path target, String sha1) throws IOException, NoSuchAlgorithmException { + final Path unverified = Paths.get(target+".unverified"); + if(!Files.exists(target)) { + new File(target.toString()).getParentFile().mkdirs(); + System.err.println("downloading " + urlString); + System.err.println("to " + target); + final InputStream stream = urlString.openStream(); + Files.copy(stream, unverified, StandardCopyOption.REPLACE_EXISTING); + stream.close(); + final String checksum = sha1(Files.readAllBytes(unverified)); + if(sha1 == null || sha1.toUpperCase().equals(checksum)) { + Files.move(unverified, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } else { + System.err.println(target + " checksum does not match.\nExpected: |" + sha1 + "|\nFound: |" + checksum + "|"); + System.exit(1); + } + } + } + + public static String sha1(byte[] bytes) throws NoSuchAlgorithmException { + final MessageDigest sha1 = MessageDigest.getInstance("SHA1"); + sha1.update(bytes, 0, bytes.length); + return (new HexBinaryAdapter()).marshal(sha1.digest()); + } } diff --git a/nailgun_launcher/TrapSecurityManager.java b/nailgun_launcher/TrapSecurityManager.java new file mode 100644 index 0000000..ed00582 --- /dev/null +++ b/nailgun_launcher/TrapSecurityManager.java @@ -0,0 +1,19 @@ +package cbt; +import java.security.*; +public class TrapSecurityManager extends SecurityManager{ + public void checkPermission( Permission permission ){ + /* + NOTE: is it actually ok, to just make these empty? + Calling .super leads to ClassNotFound exteption for a lambda. + Calling to the previous SecurityManager leads to a stack overflow + */ + } + public void checkPermission( Permission permission, Object context ){ + /* Does this methods need to be overidden? */ + } + @Override + public void checkExit( int status ){ + super.checkExit(status); + throw new TrappedExitCode(status); + } +} diff --git a/nailgun_launcher/TrappedExitCode.java b/nailgun_launcher/TrappedExitCode.java new file mode 100644 index 0000000..154db27 --- /dev/null +++ b/nailgun_launcher/TrappedExitCode.java @@ -0,0 +1,8 @@ +package cbt; +import java.security.*; +public class TrappedExitCode extends SecurityException{ + public int exitCode; + public TrappedExitCode(int exitCode){ + this.exitCode = exitCode; + } +} diff --git a/stage1/CachingClassLoader.scala b/stage1/CachingClassLoader.scala new file mode 100644 index 0000000..e75f14c --- /dev/null +++ b/stage1/CachingClassLoader.scala @@ -0,0 +1,12 @@ +package cbt +import java.net._ +import java.util.concurrent.ConcurrentHashMap +import scala.util.Try + +trait CachingClassLoader extends ClassLoader{ + def logger: Logger + val cache = new KeyLockedLazyCache[String,Try[Class[_]]]( new ConcurrentHashMap, new ConcurrentHashMap, Some(logger) ) + override def loadClass(name: String, resolve: Boolean) = { + cache.get( name, Try(super.loadClass(name, resolve)) ).get + } +} diff --git a/stage1/ClassLoaderCache.scala b/stage1/ClassLoaderCache.scala index 18a0d0e..10d872d 100644 --- a/stage1/ClassLoaderCache.scala +++ b/stage1/ClassLoaderCache.scala @@ -1,25 +1,19 @@ package cbt import java.net._ +import java.util.concurrent.ConcurrentHashMap +import collection.JavaConversions._ -private[cbt] object ClassLoaderCache{ - private val cache = NailgunLauncher.classLoaderCache - def get( classpath: ClassPath )(implicit logger: Logger): ClassLoader - = cache.synchronized{ - val lib = new Stage1Lib(logger) - val key = classpath.strings.sorted.mkString(":") - if( cache.containsKey(key) ){ - logger.resolver("CACHE HIT: "++key) - cache.get(key) - } else { - logger.resolver("CACHE MISS: "++key) - val cl = new cbt.URLClassLoader( classpath, ClassLoader.getSystemClassLoader ) - cache.put( key, cl ) - cl - } - } - def remove( classpath: ClassPath ) = { - val key = classpath.strings.sorted.mkString(":") - cache.remove( key ) - } +class ClassLoaderCache(logger: Logger){ + val persistent = new KeyLockedLazyCache( + NailgunLauncher.classLoaderCacheKeys.asInstanceOf[ConcurrentHashMap[String,AnyRef]], + NailgunLauncher.classLoaderCacheValues.asInstanceOf[ConcurrentHashMap[AnyRef,ClassLoader]], + Some(logger) + ) + val transient = new KeyLockedLazyCache( + new ConcurrentHashMap[String,AnyRef], + new ConcurrentHashMap[AnyRef,ClassLoader], + Some(logger) + ) + override def toString = s"""ClassLoaderCache("""+ persistent.keys.keySet.toVector.map(_.toString).sorted.map(" "++_).mkString("\n","\n","\n") +""")""" } diff --git a/stage1/ClassPath.scala b/stage1/ClassPath.scala index 66a1b44..96963e4 100644 --- a/stage1/ClassPath.scala +++ b/stage1/ClassPath.scala @@ -25,6 +25,6 @@ case class ClassPath(files: Seq[File]){ def string = strings.mkString( File.pathSeparator ) def strings = files.map{ f => f.string ++ ( if(f.isDirectory) "/" else "" ) - } + }.sorted def toConsole = string } diff --git a/stage1/KeyLockedLazyCache.scala b/stage1/KeyLockedLazyCache.scala index c8b37ea..aca5f74 100644 --- a/stage1/KeyLockedLazyCache.scala +++ b/stage1/KeyLockedLazyCache.scala @@ -1,33 +1,54 @@ -/* package cbt + import java.util.concurrent.ConcurrentHashMap -import scala.concurrent.Future +private[cbt] class LockableKey /** A cache that lazily computes values if needed during lookup. Locking occurs on the key, so separate keys can be looked up simultaneously without a deadlock. */ -final private[cbt] class KeyLockedLazyCache[Key <: AnyRef,Value]{ - private val keys = new ConcurrentHashMap[Key,LockableKey]() - private val builds = new ConcurrentHashMap[LockableKey,Value]() - - private class LockableKey +final private[cbt] class KeyLockedLazyCache[Key <: AnyRef,Value <: AnyRef]( + val keys: ConcurrentHashMap[Key,AnyRef], + val values: ConcurrentHashMap[AnyRef,Value], + logger: Option[Logger] +){ def get( key: Key, value: => Value ): Value = { - val keyObject = keys.synchronized{ + val lockableKey = keys.synchronized{ if( ! (keys containsKey key) ){ - keys.put( key, new LockableKey ) + val lockableKey = new LockableKey + //logger.foreach(_.resolver("CACHE MISS: " ++ key.toString)) + keys.put( key, lockableKey ) + lockableKey + } else { + val lockableKey = keys get key + //logger.foreach(_.resolver("CACHE HIT: " ++ lockableKey.toString ++ " -> " ++ key.toString)) + lockableKey } - keys get key } + import collection.JavaConversions._ + //logger.resolver("CACHE: \n" ++ keys.mkString("\n")) // synchronizing on key only, so asking for a particular key does // not block the whole cache, but just that cache entry - key.synchronized{ - if( ! (builds containsKey keyObject) ){ - builds.put( keyObject, value ) + lockableKey.synchronized{ + if( ! (values containsKey lockableKey) ){ + values.put( lockableKey, value ) } - builds get keyObject + values get lockableKey + } + } + def update( key: Key, value: Value ): Value = { + val lockableKey = keys get key + lockableKey.synchronized{ + values.put( lockableKey, value ) + value } } + def remove( key: Key ) = keys.synchronized{ + assert(keys containsKey key) + val lockableKey = keys get key + keys.remove( key ) + assert(values containsKey lockableKey) + values.remove( lockableKey ) + } } -*/ diff --git a/stage1/MultiClassLoader.scala b/stage1/MultiClassLoader.scala index de9bd32..5a93a63 100644 --- a/stage1/MultiClassLoader.scala +++ b/stage1/MultiClassLoader.scala @@ -1,24 +1,31 @@ -/* package cbt import java.net._ import scala.util.Try - import scala.collection.immutable.Seq - -class MultiClassLoader(parents: Seq[ClassLoader]) extends ClassLoader { - override def loadClass(name: String) = { - //System.err.println("LOADING CLASS "++name); - val c = parents.toStream.map{ - parent => - Try{ +// do not make this a case class, required object identity equality +class MultiClassLoader(parents: Seq[ClassLoader])(implicit val logger: Logger) extends ClassLoader with CachingClassLoader{ + override def findClass(name: String) = { + parents.find( parent => + try{ parent.loadClass(name) - }.map(Option[Class[_]](_)).recover{ - case _:ClassNotFoundException => None - }.get - }.find(_.isDefined).flatten - c.getOrElse( ClassLoader.getSystemClassLoader.loadClass(name) ) + true + } catch { + case _:ClassNotFoundException => false + } + ).map( + _.loadClass(name) + ).getOrElse( throw new ClassNotFoundException(name) ) } - override def toString = "MultiClassLoader(" ++ parents.mkString(",") ++ ")" + override def toString = ( + scala.Console.BLUE + ++ super.toString + ++ scala.Console.RESET + ++ "(" + ++ ( + if(parents.nonEmpty)( + "\n" ++ parents.map(_.toString).mkString(",\n").split("\n").map(" "++_).mkString("\n") ++ "\n" + ) else "" + ) ++")" + ) } -*/ diff --git a/stage1/PoorMansProfiler.scala b/stage1/PoorMansProfiler.scala new file mode 100644 index 0000000..b7aa47d --- /dev/null +++ b/stage1/PoorMansProfiler.scala @@ -0,0 +1,23 @@ +/* +// temporary debugging tool +package cbt +import java.util.concurrent.ConcurrentHashMap +import collection.JavaConversions._ +object PoorMansProfiler{ + val entries = new ConcurrentHashMap[String, Long] + def profile[T](name: String)(code: => T): T = { + val before = System.currentTimeMillis + if(!(entries containsKey name)){ + entries.put( name, 0 ) + } + val res = code + entries.put( name, (entries get name) + (System.currentTimeMillis - before) ) + res + } + def summary: String = { + "Profiling Summary:\n" + entries.toSeq.sortBy(_._2).map{ + case (name, value) => name + ": " + (value / 1000.0) + }.mkString("\n") + } +} +*/
\ No newline at end of file diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 937d9b3..e8245c4 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -8,76 +8,103 @@ import scala.collection.JavaConverters._ import paths._ -object CheckAlive{ - def main(args: Array[String]): Unit = { - System.exit(33) - } -} - -class Init(args: Array[String]) { +final case class Stage1ArgsParser(_args: Seq[String]) { /** * Raw parameters including their `-D` flag. **/ - val propsRaw: Seq[String] = args.toVector.filter(_.startsWith("-D")) + val propsRaw: Seq[String] = _args.toVector.filter(_.startsWith("-D")) /** * All arguments that weren't `-D` property declarations. **/ - val argsV: Seq[String] = args.toVector diff propsRaw + val args: Seq[String] = _args.toVector diff propsRaw /** * Parsed properties, as a map of keys to values. **/ - lazy val props = propsRaw + val props = propsRaw .map(_.drop(2).split("=")).map({ case Array(key, value) => key -> value }).toMap ++ System.getProperties.asScala - val logger = new Logger(props.get("log")) -} + val enabledLoggers = props.get("log") -object Stage1 extends Stage1Base{ - def mainClass = ("cbt.Stage2") + val admin = _args contains "admin" } -object AdminStage1 extends Stage1Base{ - def mainClass = ("cbt.AdminStage2") + +abstract class Stage2Base{ + def run( context: Stage2Args ): Unit } -abstract class Stage1Base{ - def mainClass: String +case class Stage2Args( + cwd: File, + args: Seq[String], + cbtHasChanged: Boolean, + logger: Logger +) +object Stage1{ protected def newerThan( a: File, b: File ) ={ a.lastModified > b.lastModified } - def main(args: Array[String]): Unit = { - val init = new Init(args) - val lib = new Stage1Lib(init.logger) + def run(_args: Array[String], classLoader: ClassLoader, stage1SourcesChanged: java.lang.Boolean): Int = { + val args = Stage1ArgsParser(_args.toVector) + val logger = new Logger(args.enabledLoggers) + logger.stage1(s"Stage1 start") + + val lib = new Stage1Lib(logger) import lib._ - logger.stage1(s"[$now] Stage1 start") - logger.stage1("Stage1: after creating lib") + val sourceFiles = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) + val changeIndicator = stage2Target ++ "/cbt/Build.class" - val cwd = args(0) + val deps = Dependencies( + JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") + ) - val src = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - val changeIndicator = stage2Target ++ "/cbt/Build.class" + val classLoaderCache = new ClassLoaderCache(logger) + + val stage2SourcesChanged = lib.needsUpdate(sourceFiles, stage2StatusFile) + logger.stage1("Compiling stage2 if necessary") + val scalaXml = JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) + compile( + stage2SourcesChanged, + sourceFiles, stage2Target, stage2StatusFile, + nailgunTarget +: stage1Target +: Dependencies(deps, scalaXml).classpath, + Seq("-deprecation"), classLoaderCache, + zincVersion = "0.3.9", scalaVersion = constants.scalaVersion + ) - logger.stage1("before conditionally running zinc to recompile CBT") - if( src.exists(newerThan(_, changeIndicator)) ) { - val stage1Classpath = CbtDependency()(logger).dependencyClasspath - logger.stage1("cbt.lib has changed. Recompiling with cp: " ++ stage1Classpath.string) - zinc( true, src, stage2Target, stage1Classpath, Seq("-deprecation") )( zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) - } logger.stage1(s"[$now] calling CbtDependency.classLoader") + if(NailgunLauncher.stage2classLoader == null){ + NailgunLauncher.stage2classLoader = CbtDependency().classLoader(classLoaderCache) + } logger.stage1(s"[$now] Run Stage2") - val ExitCode(exitCode) = /*trapExitCode*/{ // this - runMain( mainClass, cwd +: args.drop(1).toVector, CbtDependency()(logger).classLoader ) - } + val exitCode = ( + NailgunLauncher.stage2classLoader.loadClass( + if(args.admin) "cbt.AdminStage2" else "cbt.Stage2" + ) + .getMethod( "run", classOf[Stage2Args] ) + .invoke( + null, + Stage2Args( + new File( args.args(0) ), + args.args.drop(1).toVector, + // launcher changes cause entire nailgun restart, so no need for them here + cbtHasChanged = stage1SourcesChanged || stage2SourcesChanged, + logger + ) + ) match { + case code: ExitCode => code + case _ => ExitCode.Success + } + ).integer logger.stage1(s"[$now] Stage1 end") - System.exit(exitCode) + return exitCode; } } diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index b76e21b..105fe3e 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -6,6 +6,7 @@ import java.io._ import java.lang.reflect.InvocationTargetException import java.net._ import java.nio.file._ +import java.nio.file.attribute.FileTime import javax.tools._ import java.security._ import java.util._ @@ -14,23 +15,23 @@ import javax.xml.bind.annotation.adapters.HexBinaryAdapter import scala.collection.immutable.Seq // CLI interop -case class ExitCode(code: Int) +case class ExitCode(integer: Int) object ExitCode{ val Success = ExitCode(0) val Failure = ExitCode(1) } -class TrappedExitCode(private val exitCode: Int) extends Exception -object TrappedExitCode{ - def unapply(e: Throwable): Option[ExitCode] = +object CatchTrappedExitCode{ + def unapply(e: Throwable): Option[ExitCode] = { Option(e) flatMap { case i: InvocationTargetException => unapply(i.getTargetException) case e: TrappedExitCode => Some( ExitCode(e.exitCode) ) case _ => None } + } } -case class Context( cwd: File, args: Seq[String], logger: Logger ) +case class Context( cwd: File, args: Seq[String], logger: Logger, classLoaderCache: ClassLoaderCache ) class BaseLib{ def realpath(name: File) = new File(Paths.get(name.getAbsolutePath).normalize.toString) @@ -42,20 +43,6 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def scalaMajorVersion(scalaMinorVersion: String) = scalaMinorVersion.split("\\.").take(2).mkString(".") - // ========== reflection ========== - - /** Create instance of the given class via reflection */ - def create(cls: String)(args: Any*)(classLoader: ClassLoader): Any = { - logger.composition( logger.showInvocation("Stage1Lib.create", (classLoader,cls,args)) ) - import scala.reflect.runtime.universe._ - val m = runtimeMirror(classLoader) - val sym = m.classSymbol(classLoader.loadClass(cls)) - val cm = m.reflectClass( sym.asClass ) - val tpe = sym.toType - val ctorm = cm.reflectConstructor( tpe.decl(termNames.CONSTRUCTOR).asMethod ) - ctorm(args:_*) - } - // ========== file system / net ========== def array2hex(padTo: Int, array: Array[Byte]): String = { @@ -108,8 +95,9 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ trapExitCode{ classLoader .loadClass(cls) - .getMethod( "main", scala.reflect.classTag[Array[String]].runtimeClass ) + .getMethod( "main", classOf[Array[String]] ) .invoke( null, args.toArray.asInstanceOf[AnyRef] ) + ExitCode.Success } } @@ -124,71 +112,80 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } } - def zinc( + def needsUpdate( sourceFiles: Seq[File], statusFile: File ) = { + val lastCompile = statusFile.lastModified + sourceFiles.filter(_.lastModified > lastCompile).nonEmpty + } + + def compile( needsRecompile: Boolean, files: Seq[File], compileTarget: File, + statusFile: File, classpath: ClassPath, - extraArgs: Seq[String] = Seq() - )( zincVersion: String, scalaVersion: String ): Unit = { + scalacOptions: Seq[String] = Seq(), + classLoaderCache: ClassLoaderCache, + zincVersion: String, + scalaVersion: String + ): Option[File] = { val cp = classpath.string if(classpath.files.isEmpty) throw new Exception("Trying to compile with empty classpath. Source files: " ++ files.toString) - if(files.isEmpty) - throw new Exception("Trying to compile no files. ClassPath: " ++ cp) - - // only run zinc if files changed, for performance reasons - // FIXME: this is broken, need invalidate on changes in dependencies as well - if( needsRecompile ){ - val zinc = JavaDependency("com.typesafe.zinc","zinc", zincVersion) - val zincDeps = zinc.transitiveDependencies - - val sbtInterface = - zincDeps - .collect{ case d @ JavaDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none ) => d } - .headOption - .getOrElse( throw new Exception(s"cannot find sbt-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) - .jar - - val compilerInterface = - zincDeps - .collect{ case d @ JavaDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources ) => d } - .headOption - .getOrElse( throw new Exception(s"cannot find compiler-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) - .jar - - val scalaLibrary = JavaDependency("org.scala-lang","scala-library",scalaVersion).jar - val scalaReflect = JavaDependency("org.scala-lang","scala-reflect",scalaVersion).jar - val scalaCompiler = JavaDependency("org.scala-lang","scala-compiler",scalaVersion).jar - - val code = redirectOutToErr{ - lib.runMain( - "com.typesafe.zinc.Main", - Seq( - "-scala-compiler", scalaCompiler.toString, - "-scala-library", scalaLibrary.toString, - "-sbt-interface", sbtInterface.toString, - "-compiler-interface", compilerInterface.toString, - "-scala-extra", scalaReflect.toString, - "-cp", cp, - "-d", compileTarget.toString - ) ++ extraArgs.map("-S"++_) ++ files.map(_.toString), - zinc.classLoader - ) - } - - if(code != ExitCode.Success){ - // Ensure we trigger recompilation next time. This is currently required because we - // don't record the time of the last successful build elsewhere. But hopefully that will - // change soon. - val now = System.currentTimeMillis() - files.foreach(_.setLastModified(now)) - - // Tell the caller that things went wrong. - System.exit(code.code) + if( files.isEmpty ){ + None + }else{ + if( needsRecompile ){ + val zinc = JavaDependency("com.typesafe.zinc","zinc", zincVersion) + val zincDeps = zinc.transitiveDependencies + + val sbtInterface = + zincDeps + .collect{ case d @ JavaDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none ) => d } + .headOption + .getOrElse( throw new Exception(s"cannot find sbt-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) + .jar + + val compilerInterface = + zincDeps + .collect{ case d @ JavaDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources ) => d } + .headOption + .getOrElse( throw new Exception(s"cannot find compiler-interface in zinc $zincVersion dependencies: "++zincDeps.toString) ) + .jar + + val scalaLibrary = JavaDependency("org.scala-lang","scala-library",scalaVersion).jar + val scalaReflect = JavaDependency("org.scala-lang","scala-reflect",scalaVersion).jar + val scalaCompiler = JavaDependency("org.scala-lang","scala-compiler",scalaVersion).jar + + val start = System.currentTimeMillis + + val code = redirectOutToErr{ + lib.runMain( + "com.typesafe.zinc.Main", + Seq( + "-scala-compiler", scalaCompiler.toString, + "-scala-library", scalaLibrary.toString, + "-sbt-interface", sbtInterface.toString, + "-compiler-interface", compilerInterface.toString, + "-scala-extra", scalaReflect.toString, + "-cp", cp, + "-d", compileTarget.toString + ) ++ scalacOptions.map("-S"++_) ++ files.map(_.toString), + zinc.classLoader(classLoaderCache) + ) + } + + if(code == ExitCode.Success){ + // write version and when last compilation started so we can trigger + // recompile if cbt version changed or newer source files are seen + Files.write(statusFile.toPath, "".getBytes)//cbtVersion.getBytes) + Files.setLastModifiedTime(statusFile.toPath, FileTime.fromMillis(start) ) + } else { + System.exit(code.integer) // FIXME: let's find a better solution for error handling. Maybe a monad after all. + } } + Some( compileTarget ) } } def redirectOutToErr[T](code: => T): T = { @@ -201,31 +198,12 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } } - private val trapSecurityManager = new SecurityManager { - override def checkPermission( permission: Permission ) = { - /* - NOTE: is it actually ok, to just make these empty? - Calling .super leads to ClassNotFound exteption for a lambda. - Calling to the previous SecurityManager leads to a stack overflow - */ - } - override def checkPermission( permission: Permission, context: Any ) = { - /* Does this methods need to be overidden? */ - } - override def checkExit( status: Int ) = { - super.checkExit(status) - logger.lib(s"checkExit($status)") - throw new TrappedExitCode(status) - } - } - - def trapExitCode( code: => Unit ): ExitCode = { + def trapExitCode( code: => ExitCode ): ExitCode = { try{ - System.setSecurityManager( trapSecurityManager ) + System.setSecurityManager( new TrapSecurityManager ) code - ExitCode.Success } catch { - case TrappedExitCode(exitCode) => + case CatchTrappedExitCode(exitCode) => exitCode } finally { System.setSecurityManager(NailgunLauncher.defaultSecurityManager) diff --git a/stage1/URLClassLoader.scala b/stage1/URLClassLoader.scala index 870f186..9e96992 100644 --- a/stage1/URLClassLoader.scala +++ b/stage1/URLClassLoader.scala @@ -1,22 +1,47 @@ package cbt import java.net._ +import scala.util.Try -case class URLClassLoader(classPath: ClassPath, parent: ClassLoader) +class URLClassLoader( classPath: ClassPath, parent: ClassLoader )( implicit val logger: Logger ) extends java.net.URLClassLoader( - classPath.strings.map( - path => new URL("file:"++path) - ).toArray, + classPath.strings.map( p => new URL("file:" ++ p) ).toArray, parent - ){ + ) with CachingClassLoader{ + val id = Math.abs( new java.util.Random().nextInt ) override def toString = ( - scala.Console.BLUE ++ "cbt.URLClassLoader" ++ scala.Console.RESET - ++ "(\n " ++ getURLs.map(_.toString).sorted.mkString(",\n ") + scala.Console.BLUE + ++ getClass.getSimpleName ++ ":" ++ id.toString + ++ scala.Console.RESET + ++ "(\n" ++ ( - if(getParent() != ClassLoader.getSystemClassLoader()) - ",\n" ++ getParent().toString.split("\n").map(" "++_).mkString("\n") - else "" - ) + getURLs.map(_.toString).sorted.mkString(",\n") + ++ ( + if(getParent() != ClassLoader.getSystemClassLoader()) + ",\n" ++ getParent().toString + else "" + ) + ).split("\n").map(" "++_).mkString("\n") ++ "\n)" ) } + +/* +trait ClassLoaderLogging extends ClassLoader{ + def logger: Logger + val prefix = s"[${getClass.getSimpleName}] " + val postfix = " in \name" ++ this.toString + override def loadClass(name: String, resolve: Boolean): Class[_] = { + //logger.resolver(prefix ++ s"loadClass($name, $resolve)" ++ postfix ) + super.loadClass(name, resolve) + } + override def loadClass(name: String): Class[_] = { + //logger.resolver(prefix ++ s"loadClass($name)" ++ postfix ) + super.loadClass(name) + } + override def findClass(name: String): Class[_] = { + //logger.resolver(prefix ++ s"findClass($name)" ++ postfix ) + super.findClass(name) + } +} +*/ diff --git a/stage1/constants.scala b/stage1/constants.scala index a14754e..4c39237 100644 --- a/stage1/constants.scala +++ b/stage1/constants.scala @@ -1,5 +1,7 @@ package cbt object constants{ - val scalaVersion = Option(System.getenv("SCALA_VERSION")).get + val scalaXmlVersion = NailgunLauncher.SCALA_XML_VERSION + val scalaVersion = NailgunLauncher.SCALA_VERSION + val zincVersion = NailgunLauncher.ZINC_VERSION val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") } diff --git a/stage1/paths.scala b/stage1/paths.scala index d3856c8..f27e538 100644 --- a/stage1/paths.scala +++ b/stage1/paths.scala @@ -4,12 +4,14 @@ object paths{ val cbtHome: File = new File(Option(System.getenv("CBT_HOME")).get) val mavenCache: File = cbtHome ++ "/cache/maven" val userHome: File = new File(Option(System.getProperty("user.home")).get) - val stage1: File = new File(Option(System.getenv("STAGE1")).get) - val stage2: File = cbtHome ++ "/stage2" + val bootstrapScala: File = cbtHome ++ "/bootstrap_scala" val nailgun: File = new File(Option(System.getenv("NAILGUN")).get) + val stage1: File = new File(NailgunLauncher.STAGE1) + val stage2: File = cbtHome ++ "/stage2" private val target = Option(System.getenv("TARGET")).get.stripSuffix("/") val stage1Target: File = stage1 ++ ("/" ++ target) val stage2Target: File = stage2 ++ ("/" ++ target) + val stage2StatusFile: File = stage2Target ++ ".last-success" val nailgunTarget: File = nailgun ++ ("/" ++ target) val sonatypeLogin: File = cbtHome ++ "/sonatype.login" } diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 1dbadcc..2e8ef15 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -28,7 +28,7 @@ abstract class Dependency{ implicit def logger: Logger protected def lib = new Stage1Lib(logger) - def updated: Boolean + def needsUpdate: Boolean //def cacheClassLoader: Boolean = false private[cbt] def targetClasspath: ClassPath def exportedClasspath: ClassPath @@ -36,7 +36,6 @@ abstract class Dependency{ def jars: Seq[File] = exportedJars ++ dependencyJars def canBeCached = false - def cacheDependencyClassLoader = true //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] def exportClasspathConcurrently: ClassPath = { @@ -86,37 +85,64 @@ abstract class Dependency{ ) } - private object classLoaderCache extends Cache[URLClassLoader] - def classLoader: URLClassLoader = classLoaderCache{ - if( concurrencyEnabled ){ - // trigger concurrent building / downloading dependencies - exportClasspathConcurrently - } - val transitiveClassPath = transitiveDependencies.map{ - case d if d.canBeCached => Left(d) - case d => Right(d) - } - val buildClassPath = ClassPath.flatten( - transitiveClassPath.flatMap( - _.right.toOption.map(_.exportedClasspath) - ) - ) - val cachedClassPath = ClassPath.flatten( - transitiveClassPath.flatMap( - _.left.toOption - ).par.map(_.exportedClasspath).seq.sortBy(_.string) - ) - - if(cacheDependencyClassLoader){ - new URLClassLoader( - exportedClasspath ++ buildClassPath, - ClassLoaderCache.get( cachedClassPath ) + private def actual(current: Dependency, latest: Map[(String,String),Dependency]) = current match { + case d: ArtifactInfo => latest((d.groupId,d.artifactId)) + case d => d + } + private def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + if( dependencies.isEmpty ){ + ClassLoader.getSystemClassLoader + } else if( dependencies.size == 1 ){ + dependencies.head.classLoaderRecursion( latest, cache ) + } else if( dependencies.forall(_.canBeCached) ){ + assert(transitiveDependencies.forall(_.canBeCached)) + cache.persistent.get( + dependencyClasspath.string, + new MultiClassLoader( + dependencies.map( _.classLoaderRecursion(latest, cache) ) + ) ) } else { + val (cachable, nonCachable) = dependencies.partition(_.canBeCached) new URLClassLoader( - exportedClasspath ++ buildClassPath ++ cachedClassPath, ClassLoader.getSystemClassLoader + ClassPath.flatten( nonCachable.map(actual(_,latest)).map(_.exportedClasspath) ), + cache.persistent.get( + ClassPath.flatten( cachable.map(actual(_,latest)).map(_.exportedClasspath) ).string, + new MultiClassLoader( + cachable.map( _.classLoaderRecursion(latest, cache) ) + ) + ) + ) + new MultiClassLoader( + dependencies.map( _.classLoaderRecursion(latest, cache) ) + ) + } + } + protected def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + if( canBeCached ){ + val a = actual( this, latest ) + cache.persistent.get( + a.classpath.string, + new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) ) + } else { + new cbt.URLClassLoader( exportedClasspath, dependencyClassLoader(latest, cache) ) + } + } + private object classLoaderCache extends Cache[ClassLoader] + def classLoader( cache: ClassLoaderCache ): ClassLoader = classLoaderCache{ + if( concurrencyEnabled ){ + // trigger concurrent building / downloading dependencies + exportClasspathConcurrently } + classLoaderRecursion( + (this +: transitiveDependencies).collect{ + case d: ArtifactInfo => d + }.groupBy( + d => (d.groupId,d.artifactId) + ).mapValues(_.head), + cache + ) } def classpath : ClassPath = exportedClasspath ++ dependencyClasspath def dependencyJars : Seq[File] = transitiveDependencies.flatMap(_.jars) @@ -128,25 +154,30 @@ abstract class Dependency{ new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents))) } - def transitiveDependencies: Seq[Dependency] = { - val deps = dependencies.flatMap(_.resolveRecursive().linearize) + def linearize(deps: Seq[Dependency]): Seq[Dependency] = + if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) ) + + private object transitiveDependenciesCache extends Cache[Seq[Dependency]] + /** return dependencies in order of linearized dependence. this is a bit tricky. */ + def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ + val deps = linearize(dependencies) val hasInfo = deps.collect{ case d:ArtifactInfo => d } val noInfo = deps.filter{ case _:ArtifactInfo => false case _ => true } - noInfo ++ JavaDependency.removeOutdated( hasInfo ) - }.sortBy(_.targetClasspath.string) + noInfo ++ JavaDependency.updateOutdated( hasInfo ).reverse.distinct + } def show: String = this.getClass.getSimpleName // ========== debug ========== def dependencyTree: String = dependencyTreeRecursion() private def dependencyTreeRecursion(indent: Int = 0): String = ( ( " " * indent ) - ++ (if(updated) lib.red(show) else show) + ++ (if(needsUpdate) lib.red(show) else show) ++ dependencies.map( - _.dependencyTreeRecursion(indent + 1) - ).map( "\n" ++ _.toString ).mkString("") + "\n" ++ _.dependencyTreeRecursion(indent + 1) + ).mkString ) } @@ -156,7 +187,7 @@ class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends class ScalaReflectDependency (version: String)(implicit logger: Logger) extends JavaDependency("org.scala-lang","scala-reflect",version) case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd => - final val updated = false + override final val needsUpdate = false override def canBeCached = true def targetClasspath = ClassPath(Seq()) def exportedClasspath = ClassPath(Seq()) @@ -169,35 +200,56 @@ case class ScalaDependencies(version: String)(implicit val logger: Logger) exten } case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{ - def updated = false def exportedClasspath = ClassPath(Seq(path)) def exportedJars = Seq[File](path) + override def needsUpdate = false def targetClasspath = exportedClasspath } +/** Allows to easily assemble a bunch of dependencies */ +case class Dependencies( _dependencies: Dependency* )(implicit val logger: Logger) extends Dependency{ + override def dependencies = _dependencies.to + def needsUpdate = dependencies.exists(_.needsUpdate) + def exportedClasspath = ClassPath(Seq()) + def exportedJars = Seq() + def targetClasspath = ClassPath(Seq()) +} + case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{ - def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) - def exportedJars = Seq[File]() - def dependencies = ScalaDependencies(constants.scalaVersion).dependencies - def updated = false // FIXME: think this through, might allow simplifications and/or optimizations - def targetClasspath = exportedClasspath + def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations + override def canBeCached = false + /* + private object classLoaderRecursionCache extends Cache[ClassLoader] + override def classLoaderRecursion(latest: Map[(String,String),Dependency], cache: ClassLoaderCache) = classLoaderRecursionCache{ + println(System.currentTimeMillis) + val cl = getClass.getClassLoader + println(System.currentTimeMillis) + cl + ClassLoader.getSystemClassLoader + } + */ + override def targetClasspath = exportedClasspath + override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) + override def exportedJars = ???//Seq[File]() + override def dependencies = Seq( + JavaDependency("org.scala-lang","scala-library",constants.scalaVersion), + JavaDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5") + ) + // FIXME: implement sanity check to prevent using incompatible scala-library and xml version on cp + override def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ) + = getClass.getClassLoader } case class CbtDependency()(implicit val logger: Logger) extends Dependency{ - def exportedClasspath = ClassPath( Seq( stage2Target ) ) - def exportedJars = Seq[File]() + def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations + override def canBeCached = false + override def targetClasspath = exportedClasspath + override def exportedClasspath = ClassPath( Seq( stage2Target ) ) + override def exportedJars = ??? override def dependencies = Seq( Stage1Dependency(), JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), - lib.ScalaDependency( - "com.lihaoyi","ammonite-ops","0.5.5", scalaVersion = constants.scalaMajorVersion - ), - lib.ScalaDependency( - "org.scala-lang.modules","scala-xml","1.0.5", scalaVersion = constants.scalaMajorVersion - ) + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") ) - def updated = false // FIXME: think this through, might allow simplifications and/or optimizations - def targetClasspath = exportedClasspath } case class Classifier(name: Option[String]) @@ -210,8 +262,14 @@ object Classifier{ case class JavaDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none )(implicit val logger: Logger) extends ArtifactInfo{ + assert(groupId != "", toString) + assert(artifactId != "", toString) + assert(version != "", toString) + assert(groupId != null, toString) + assert(artifactId != null, toString) + assert(version != null, toString) - def updated = false + override def needsUpdate = false override def canBeCached = true private val groupPath = groupId.split("\\.").mkString("/") @@ -226,7 +284,7 @@ case class JavaDependency( private def jarFile: File = baseFile ++ ".jar" //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" private def pomUrl: URL = baseUrl ++ ".pom" - private def jarUrl: URL = baseUrl ++ ".jar" + private[cbt] def jarUrl: URL = baseUrl ++ ".jar" def exportedJars = Seq( jar ) def exportedClasspath = ClassPath( exportedJars ) @@ -235,36 +293,33 @@ case class JavaDependency( def jarSha1 = { val file = jarFile ++ ".sha1" - scala.util.Try{ - lib.download( jarUrl ++ ".sha1" , file, None ) - // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim - }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible) + lib.download( jarUrl ++ ".sha1" , file, None ) + // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom + Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim } def pomSha1 = { val file = pomFile++".sha1" - scala.util.Try{ - lib.download( pomUrl++".sha1" , file, None ) - // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom - Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim - }.toOption // FIXME: .toOption is a temporary solution to ignore if libs don't have one (not sure that's even possible) + lib.download( pomUrl++".sha1" , file, None ) + // split(" ") here so checksum file contents in this format work: df7f15de037a1ee4d57d2ed779739089f560338c jna-3.2.2.pom + Files.readAllLines(Paths.get(file.string)).mkString("\n").split(" ").head.trim } - def jar = { - lib.download( jarUrl, jarFile, jarSha1 ) + private object jarCache extends Cache[File] + def jar = jarCache{ + lib.download( jarUrl, jarFile, Some(jarSha1) ) jarFile } def pomXml = XML.loadFile(pom.toString) def pom = { - lib.download( pomUrl, pomFile, pomSha1 ) + lib.download( pomUrl, pomFile, Some(pomSha1) ) pomFile } // ========== pom traversal ========== - lazy val pomParents: Seq[JavaDependency] = { + lazy val transitivePom: Seq[JavaDependency] = { (pomXml \ "parent").collect{ case parent => JavaDependency( @@ -272,12 +327,12 @@ case class JavaDependency( (parent \ "artifactId").text, (parent \ "version").text )(logger) - } + }.flatMap(_.transitivePom) :+ this } lazy val properties: Map[String, String] = ( - pomParents.flatMap(_.properties) ++ { - val props = (pomXml \ "properties").flatMap(_.child).map{ + transitivePom.flatMap{ d => + val props = (d.pomXml \ "properties").flatMap(_.child).map{ tag => tag.label -> tag.text } logger.pom(s"Found properties in $pom: $props") @@ -285,17 +340,15 @@ case class JavaDependency( } ).toMap - lazy val dependencyVersions: Map[(String,String), String] = - pomParents.flatMap( + lazy val dependencyVersions: Map[String, (String,String)] = + transitivePom.flatMap( p => - p.dependencyVersions - ++ (p.pomXml \ "dependencyManagement" \ "dependencies" \ "dependency").map{ xml => val groupId = p.lookup(xml,_ \ "groupId").get val artifactId = p.lookup(xml,_ \ "artifactId").get val version = p.lookup(xml,_ \ "version").get - (groupId, artifactId) -> version + artifactId -> (groupId, version) } ).toMap @@ -303,14 +356,27 @@ case class JavaDependency( if(classifier == Classifier.sources) Seq() else (pomXml \ "dependencies" \ "dependency").collect{ case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" => - val groupId = lookup(xml,_ \ "groupId").get val artifactId = lookup(xml,_ \ "artifactId").get + val groupId = + lookup(xml,_ \ "groupId").getOrElse( + dependencyVersions + .get(artifactId).map(_._1) + .getOrElse( + throw new Exception(s"$artifactId not found in \n$dependencyVersions") + ) + ) + val version = + lookup(xml,_ \ "version").getOrElse( + dependencyVersions + .get(artifactId).map(_._2) + .getOrElse( + throw new Exception(s"$artifactId not found in \n$dependencyVersions") + ) + ) JavaDependency( - groupId, - artifactId, - lookup(xml,_ \ "version").getOrElse( dependencyVersions(groupId, artifactId) ), + groupId, artifactId, version, Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) - )(logger) + ) }.toVector } def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { @@ -319,7 +385,16 @@ case class JavaDependency( accessor(xml).headOption.flatMap{v => //println("found: "++v.text) v.text match { - case Substitution(path) => Option(properties(path)) + case Substitution(path) => Option( + properties.get(path).orElse( + transitivePom.reverse.flatMap{ d => + Some(path.split("\\.").toList).collect{ + case "project" :: path => + path.foldLeft(d.pomXml:NodeSeq){ case (xml,tag) => xml \ tag }.text + }.filter(_ != "") + }.headOption + ) + .getOrElse( throw new Exception(s"Can't find $path in \n$properties.\n\npomParents: $transitivePom\n\n pomXml:\n$pomXml" ))) //println("lookup "++path ++ ": "++(pomXml\path).text) case value => Option(value) } @@ -344,7 +419,7 @@ object JavaDependency{ case e: NumberFormatException => Right(str) } /* this obviously should be overridable somehow */ - def removeOutdated( + def updateOutdated( deps: Seq[ArtifactInfo], versionLessThan: (String, String) => Boolean = semanticVersionLessThan )(implicit logger: Logger): Seq[ArtifactInfo] = { @@ -354,11 +429,11 @@ object JavaDependency{ _.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) ) .last ) - deps.flatMap{ + deps.map{ d => - val l = latest.get((d.groupId,d.artifactId)) + val l = latest((d.groupId,d.artifactId)) if(d != l) logger.resolver("outdated: "++d.show) l - }.distinct + } } } diff --git a/stage2/AdminStage2.scala b/stage2/AdminStage2.scala index d923b22..883b5ed 100644 --- a/stage2/AdminStage2.scala +++ b/stage2/AdminStage2.scala @@ -1,13 +1,12 @@ package cbt import java.io._ -object AdminStage2{ - def main(_args: Array[String]) = { - val args = _args.drop(1).dropWhile(Seq("admin","direct") contains _) - val init = new Init(args) - val lib = new Lib(init.logger) - val adminTasks = new AdminTasks(lib, args, new File(_args(0))) +object AdminStage2 extends Stage2Base{ + def run( _args: Stage2Args ): Unit = { + val args = _args.args.dropWhile(Seq("admin","direct") contains _) + val lib = new Lib(_args.logger) + val adminTasks = new AdminTasks(lib, args, _args.cwd) new lib.ReflectObject(adminTasks){ - def usage: String = "Available methods: " ++ lib.taskNames(subclassType).mkString(" ") + def usage: String = "Available methods: " ++ lib.taskNames(adminTasks.getClass).mkString(" ") }.callNullary(args.lift(0)) } } diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala index e7fc78b..069b712 100644 --- a/stage2/AdminTasks.scala +++ b/stage2/AdminTasks.scala @@ -1,35 +1,132 @@ package cbt import scala.collection.immutable.Seq -import java.io._ -class AdminTasks(lib: Lib, args: Array[String], cwd: File){ +import java.io.{Console=>_,_} +import java.nio.file._ +class AdminTasks(lib: Lib, args: Seq[String], cwd: File){ implicit val logger: Logger = lib.logger def resolve = { ClassPath.flatten( args(1).split(",").toVector.map{ d => val v = d.split(":") - new JavaDependency(v(0),v(1),v(2))(lib.logger).classpath + new JavaDependency(v(0),v(1),v(2)).classpath } ) } + def dependencyTree = { + args(1).split(",").toVector.map{ + d => + val v = d.split(":") + new JavaDependency(v(0),v(1),v(2)).dependencyTree + }.mkString("\n\n") + } def amm = ammonite def ammonite = { val version = args.lift(1).getOrElse(constants.scalaVersion) val scalac = new ScalaCompilerDependency( version ) val d = JavaDependency( - "com.lihaoyi","ammonite-repl_2.11.7",args.lift(1).getOrElse("0.5.6") + "com.lihaoyi","ammonite-repl_2.11.7",args.lift(1).getOrElse("0.5.7") ) // FIXME: this does not work quite yet, throws NoSuchFileException: /ammonite/repl/frontend/ReplBridge$.class lib.runMain( - "ammonite.repl.Main", Seq(), d.classLoader + "ammonite.repl.Main", Seq(), d.classLoader(new ClassLoaderCache(logger)) ) } def scala = { val version = args.lift(1).getOrElse(constants.scalaVersion) val scalac = new ScalaCompilerDependency( version ) lib.runMain( - "scala.tools.nsc.MainGenericRunner", Seq("-cp", scalac.classpath.string), scalac.classLoader + "scala.tools.nsc.MainGenericRunner", Seq("-cp", scalac.classpath.string), scalac.classLoader(new ClassLoaderCache(logger)) ) } def scaffoldBasicBuild: Unit = lib.scaffoldBasicBuild( cwd ) + def cbtEarlyDependencies = { + val scalaVersion = args.lift(1).getOrElse(constants.scalaVersion) + val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") + val scalaXmlVersion = args.lift(2).getOrElse(constants.scalaXmlVersion) + val zincVersion = args.lift(3).getOrElse(constants.zincVersion) + /* + def tree(d: JavaDependency, indent: Int): String ={ + val dependencies = { + if( d.dependencies.nonEmpty ){ + d.dependencies.map{ + case d: JavaDependency => tree(d,indent + 1) + }.mkString(",\n" ++ ( " " * indent ),",\n" ++ ( " " * indent ), "") + } else "" + } + ( + s"""new EarlyDependency( "${d.groupId}", "${d.artifactId}", "${d.version}", "${d.jarSha1}"$dependencies)""" + ) + }*/ + val scalaDeps = Seq( + JavaDependency("org.scala-lang","scala-reflect",scalaVersion), + JavaDependency("org.scala-lang","scala-compiler",scalaVersion) + ) + + val scalaXml = Dependencies( + JavaDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion), + JavaDependency("org.scala-lang","scala-library",scalaVersion) + ) + + val zinc = JavaDependency("com.typesafe.zinc","zinc",zincVersion) + println(zinc.dependencyTree) + + def valName(dep: JavaDependency) = { + val words = dep.artifactId.split("_").head.split("-") + words(0) ++ words.drop(1).map(s => s(0).toString.toUpperCase ++ s.drop(1)).mkString ++ "_" ++ dep.version.replace(".","_") ++ "_" + } + + def vals(d: JavaDependency) = s""" """ + + def jarVal(dep: JavaDependency) = "_" + valName(dep) +"Jar" + def transitive(dep: Dependency) = (dep +: dep.transitiveDependencies.reverse).collect{case d: JavaDependency => d} + def codeEach(dep: Dependency) = { + transitive(dep).tails.map(_.reverse).toVector.reverse.drop(1).map{ + deps => + val d = deps.last + val parents = deps.dropRight(1) + val parentString = if(parents.isEmpty) "" else ( ", " ++ valName(parents.last) ) + val n = valName(d) + s""" + // ${d.groupId}:${d.artifactId}:${d.version} + download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}"); + ClassLoader $n = cachePut( + classLoader( ${n}File$parentString ), + ${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")} + );""" + } + } + val assignments = codeEach(zinc) ++ codeEach(scalaXml) + //{ case (name, dep) => s"$name =\n ${tree(dep, 4)};" }.mkString("\n\n ") + val code = s"""// This file was auto-generated using `cbt admin cbtEarlyDependencies` +package cbt; +import java.io.*; +import java.nio.file.*; +import java.net.*; +import java.security.*; +import static cbt.NailgunLauncher.*; + +class EarlyDependencies{ + + /** ClassLoader for stage1 */ + ClassLoader stage1; + /** ClassLoader for zinc */ + ClassLoader zinc; + +${(scalaDeps ++ transitive(scalaXml) ++ transitive(zinc)).map(d => s""" String ${valName(d)}File = MAVEN_CACHE + "${d.basePath}.jar";""").mkString("\n")} + + public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ +${scalaDeps.map(d => s""" download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} +${assignments.mkString("\n")} + + stage1 = scalaXml_${scalaXmlVersion.replace(".","_")}_; + + zinc = zinc_${zincVersion.replace(".","_")}_; + } +} +""" + val file = paths.nailgun ++ ("/" ++ "EarlyDependencies.java") + Files.write( file.toPath, code.getBytes ) + println( Console.GREEN ++ "Wrote " ++ file.string ++ Console.RESET ) + } } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 2f90197..9ed8c26 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -2,7 +2,6 @@ package cbt import cbt.paths._ import java.io._ -import java.lang.reflect.InvocationTargetException import java.net._ import java.nio.file.{Path =>_,_} import java.nio.file.Files.readAllBytes @@ -10,15 +9,13 @@ import java.security.MessageDigest import java.util.jar._ import scala.collection.immutable.Seq -import scala.reflect.runtime.{universe => ru} import scala.util._ -import ammonite.ops.{cwd => _,_} - class BasicBuild( context: Context ) extends Build( context ) class Build(val context: Context) extends Dependency with TriggerLoop{ // library available to builds implicit final val logger: Logger = context.logger + implicit final val classLoaderCache: ClassLoaderCache = context.classLoaderCache override final protected val lib: Lib = new Lib(logger) // ========== general stuff ========== @@ -26,7 +23,7 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ def enableConcurrency = false final def projectDirectory: File = lib.realpath(context.cwd) assert( projectDirectory.exists, "projectDirectory does not exist: " ++ projectDirectory.string ) - final def usage: Unit = new lib.ReflectBuild(this).usage + final def usage: String = lib.usage(this.getClass, context) // ========== meta data ========== @@ -51,6 +48,12 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ def apiTarget: File = scalaTarget ++ "/api" /** directory where the class files should be put (in package directories) */ def compileTarget: File = scalaTarget ++ "/classes" + /** + File which cbt uses to determine if it needs to trigger an incremental re-compile. + Last modified date is the time when the last successful compilation started. + Contents is the cbt version git hash. + */ + def compileStatusFile: File = compileTarget ++ ".last-success" /** Source directories and files. Defaults to .scala and .java files in src/ and top-level. */ def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(sourceFileFilter) @@ -113,7 +116,7 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ override def dependencyClasspath : ClassPath = ClassPath(localJars) ++ super.dependencyClasspath override def dependencyJars : Seq[File] = localJars ++ super.dependencyJars - def exportedClasspath : ClassPath = ClassPath(Seq(compile)) + def exportedClasspath : ClassPath = ClassPath(compile.toSeq:_*) def targetClasspath = ClassPath(Seq(compileTarget)) def exportedJars: Seq[File] = Seq() // ========== compile, run, test ========== @@ -121,38 +124,25 @@ class Build(val context: Context) extends Dependency with TriggerLoop{ /** scalac options used for zinc and scaladoc */ def scalacOptions: Seq[String] = Seq( "-feature", "-deprecation", "-unchecked" ) - val updated: Boolean = { - val existingClassFiles = lib.listFilesRecursive(compileTarget) - val sourcesChanged = existingClassFiles.nonEmpty && { - val oldestClassFile = existingClassFiles.sortBy(_.lastModified).head - val oldestClassFileAge = oldestClassFile.lastModified - val changedSourceFiles = sourceFiles.filter(_.lastModified > oldestClassFileAge) - if(changedSourceFiles.nonEmpty){ - /* - println(changedSourceFiles) - println(changedSourceFiles.map(_.lastModified)) - println(changedSourceFiles.map(_.lastModified > oldestClassFileAge)) - println(oldestClassFile) - println(oldestClassFileAge) - println("-"*80) - */ - } - changedSourceFiles.nonEmpty - } - sourcesChanged || transitiveDependencies.map(_.updated).fold(false)(_ || _) + private object needsUpdateCache extends Cache[Boolean] + def needsUpdate: Boolean = { + needsUpdateCache( + lib.needsUpdate( sourceFiles, compileStatusFile ) + || transitiveDependencies.exists(_.needsUpdate) + ) } - private object compileCache extends Cache[File] - def compile: File = compileCache{ + private object compileCache extends Cache[Option[File]] + def compile: Option[File] = compileCache{ lib.compile( - updated, - sourceFiles, compileTarget, dependencyClasspath, scalacOptions, - zincVersion = zincVersion, scalaVersion = scalaVersion + needsUpdate, + sourceFiles, compileTarget, compileStatusFile, dependencyClasspath, scalacOptions, + context.classLoaderCache, zincVersion = zincVersion, scalaVersion = scalaVersion ) } def runClass: String = "Main" - def run: ExitCode = lib.runMainIfFound( runClass, context.args, classLoader ) + def run: ExitCode = lib.runMainIfFound( runClass, context.args, classLoader(context.classLoaderCache) ) def test: ExitCode = lib.test(context) diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index 5e0f5d3..9746d8c 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -8,10 +8,14 @@ class BuildBuild(context: Context) extends Build(context){ val managedBuild = { val managedContext = context.copy( cwd = managedBuildDirectory ) val cl = new cbt.URLClassLoader( - classpath, + exportedClasspath, classOf[BuildBuild].getClassLoader // FIXME: this looks wrong. Should be ClassLoader.getSystemClassLoader but that crashes ) - lib.create( lib.buildClassName )( managedContext )( cl ).asInstanceOf[Build] + cl + .loadClass(lib.buildClassName) + .getConstructor(classOf[Context]) + .newInstance(managedContext) + .asInstanceOf[Build] } override def triggerLoopFiles = super.triggerLoopFiles ++ managedBuild.triggerLoopFiles override def finalBuild = managedBuild.finalBuild diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 84a0100..e3a01c7 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -25,7 +25,7 @@ case class BuildDependency(context: Context) extends TriggerLoop{ def exportedJars = Seq() def dependencies = Seq(build) def triggerLoopFiles = root.triggerLoopFiles - final val updated = build.updated + override final val needsUpdate = build.needsUpdate def targetClasspath = ClassPath(Seq()) } /* diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index c3e38b6..59de98a 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -7,7 +7,7 @@ import org.eclipse.jgit.lib.Ref case class GitDependency( url: String, ref: String // example: git://github.com/cvogt/cbt.git#<some-hash> -)(implicit val logger: Logger) extends Dependency{ +)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache ) extends Dependency{ override def lib = new Lib(logger) // TODO: add support for authentication via ssh and/or https @@ -37,7 +37,7 @@ case class GitDependency( } val managedBuild = lib.loadDynamic( - Context( cwd = checkoutDirectory, args = Seq(), logger ) + Context( cwd = checkoutDirectory, args = Seq(), logger, classLoaderCache ) ) Seq( managedBuild ) } @@ -45,5 +45,5 @@ case class GitDependency( def exportedClasspath = ClassPath(Seq()) def exportedJars = Seq() private[cbt] def targetClasspath = exportedClasspath - def updated: Boolean = false + def needsUpdate: Boolean = false } diff --git a/stage2/Lib.scala b/stage2/Lib.scala index 60e7dd4..dd4a12f 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -8,13 +8,11 @@ import java.nio.file.{Path =>_,_} import java.nio.file.Files.readAllBytes import java.security.MessageDigest import java.util.jar._ +import java.lang.reflect.Method import scala.collection.immutable.Seq -import scala.reflect.runtime.{universe => ru} import scala.util._ -import ammonite.ops.{cwd => _,_} - // pom model case class Developer(id: String, name: String, timezone: String, url: URL) case class License(name: String, url: URL) @@ -55,28 +53,18 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } } - def compile( - updated: Boolean, - sourceFiles: Seq[File], compileTarget: File, dependenyClasspath: ClassPath, - compileArgs: Seq[String], zincVersion: String, scalaVersion: String - ): File = { - if(sourceFiles.nonEmpty) - lib.zinc( - updated, sourceFiles, compileTarget, dependenyClasspath, compileArgs - )( zincVersion = zincVersion, scalaVersion = scalaVersion ) - compileTarget - } - - def srcJar(sources: Seq[File], artifactId: String, version: String, jarTarget: File): File = { - val file = jarTarget ++ ("/"++artifactId++"-"++version++"-sources.jar") - lib.jarFile(file, sources) - file + def srcJar(sourceFiles: Seq[File], artifactId: String, version: String, jarTarget: File): Option[File] = { + lib.jarFile( + jarTarget ++ ("/"++artifactId++"-"++version++"-sources.jar"), + sourceFiles + ) } - def jar(artifactId: String, version: String, compileTarget: File, jarTarget: File): File = { - val file = jarTarget ++ ("/"++artifactId++"-"++version++".jar") - lib.jarFile(file, Seq(compileTarget)) - file + def jar(artifactId: String, version: String, compileTarget: File, jarTarget: File): Option[File] = { + lib.jarFile( + jarTarget ++ ("/"++artifactId++"-"++version++".jar"), + Seq(compileTarget) + ) } def docJar( @@ -87,29 +75,31 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ jarTarget: File, artifactId: String, version: String, - compileArgs: Seq[String] - ): File = { - mkdir(Path(apiTarget)) - if(sourceFiles.nonEmpty){ + compileArgs: Seq[String], + classLoaderCache: ClassLoaderCache + ): Option[File] = { + if(sourceFiles.isEmpty){ + None + } else { + apiTarget.mkdirs val args = Seq( // FIXME: can we use compiler dependency here? "-cp", dependencyClasspath.string, // FIXME: does this break for builds that don't have scalac dependencies? "-d", apiTarget.toString ) ++ compileArgs ++ sourceFiles.map(_.toString) logger.lib("creating docs for source files "+args.mkString(", ")) - trapExitCode{ - redirectOutToErr{ - runMain( - "scala.tools.nsc.ScalaDoc", - args, - ScalaDependencies(scalaVersion)(logger).classLoader - ) - } + redirectOutToErr{ + runMain( + "scala.tools.nsc.ScalaDoc", + args, + ScalaDependencies(scalaVersion)(logger).classLoader(classLoaderCache) + ) } + lib.jarFile( + jarTarget ++ ("/"++artifactId++"-"++version++"-javadoc.jar"), + Vector(apiTarget) + ) } - val docJar = jarTarget ++ ("/"++artifactId++"-"++version++"-javadoc.jar") - lib.jarFile(docJar, Vector(apiTarget)) - docJar } def test( context: Context ): ExitCode = { @@ -128,60 +118,66 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } // task reflection helpers - import ru._ - private lazy val anyRefMembers: Set[String] = ru.typeOf[AnyRef].members.toSet.map(taskName) - def taskNames(tpe: Type): Seq[String] = tpe.members.toVector.flatMap(lib.toTask).map(taskName).sorted - private def taskName(method: Symbol): String = method.name.decodedName.toString - def toTask(symbol: Symbol): Option[MethodSymbol] = { - Option(symbol) - .filter(_.isPublic) - .filter(_.isMethod) - .map(_.asMethod) - .filter(_.paramLists.flatten.size == 0) - .filterNot(taskName(_) contains "$") - .filterNot(t => anyRefMembers contains taskName(t)) - } + def tasks(cls:Class[_]): Map[String, Method] = + Stream + .iterate(cls.asInstanceOf[Class[Any]])(_.getSuperclass) + .takeWhile(_ != null) + .toVector + .dropRight(1) // drop Object + .reverse + .flatMap( + c => + c + .getDeclaredMethods + .filterNot( _.getName contains "$" ) + .filter{ m => + java.lang.reflect.Modifier.isPublic(m.getModifiers) + } + .filter( _.getParameterCount == 0 ) + .map(m => NameTransformer.decode(m.getName) -> m) + ).toMap - class ReflectBuild(val build: Build) extends ReflectObject(build){ - def usage: String = { - val baseTasks = lib.taskNames(ru.typeOf[Build]) - val thisTasks = lib.taskNames(subclassType) diff baseTasks + def taskNames(cls: Class[_]): Seq[String] = tasks(cls).keys.toVector.sorted + + def usage(buildClass: Class[_], context: Context): String = { + val baseTasks = lib.taskNames(classOf[Build]) + val thisTasks = lib.taskNames(buildClass) diff baseTasks + ( ( - ( - if( thisTasks.nonEmpty ){ - s"""Methods provided by Build ${build.context.cwd} + if( thisTasks.nonEmpty ){ + s"""Methods provided by Build ${context} ${thisTasks.mkString(" ")} """ - } else "" - ) ++ s"""Methods provided by CBT (but possibly overwritten) + } else "" + ) ++ s"""Methods provided by CBT (but possibly overwritten) ${baseTasks.mkString(" ")}""" ) ++ "\n" - } } + class ReflectBuild[T:scala.reflect.ClassTag](build: Build) extends ReflectObject(build){ + def usage = lib.usage(build.getClass, build.context) + } abstract class ReflectObject[T:scala.reflect.ClassTag](obj: T){ - lazy val mirror = ru.runtimeMirror(obj.getClass.getClassLoader) - lazy val subclassType = mirror.classSymbol(obj.getClass).toType def usage: String def callNullary( taskName: Option[String] ): Unit = { - taskName - .map{ n => subclassType.member(ru.TermName(n).encodedName) } - .filter(_ != ru.NoSymbol) - .flatMap(toTask _) - .map{ methodSymbol => - val result = mirror.reflect(obj).reflectMethod(methodSymbol)() - + val ts = tasks(obj.getClass) + taskName.map( NameTransformer.encode ).flatMap(ts.get).map{ method => + val result: Option[Any] = Option(method.invoke(obj)) // null in case of Unit + result.flatMap{ + case v: Option[_] => v + case other => Some(other) + }.map{ + value => // Try to render console representation. Probably not the best way to do this. - scala.util.Try( result.getClass.getDeclaredMethod("toConsole") ) match { - case scala.util.Success(m) => - println(m.invoke(result)) + scala.util.Try( value.getClass.getDeclaredMethod("toConsole") ) match { + case scala.util.Success(toConsole) => + println(toConsole.invoke(value)) - case scala.util.Failure(e) if e.getMessage contains "toConsole" => - result match { - case () => "" + case scala.util.Failure(e) if Option(e.getMessage).getOrElse("") contains "toConsole" => + value match { case ExitCode(code) => System.exit(code) case other => println( other.toString ) // no method .toConsole, using to String } @@ -189,16 +185,17 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ case scala.util.Failure(e) => throw e } - }.getOrElse{ - taskName.foreach{ n => - System.err.println(s"Method not found: $n") - System.err.println("") - } - System.err.println(usage) - taskName.foreach{ _ => - ExitCode.Failure - } + }.getOrElse("") + }.getOrElse{ + taskName.foreach{ n => + System.err.println(s"Method not found: $n") + System.err.println("") } + System.err.println(usage) + taskName.foreach{ _ => + ExitCode.Failure + } + } } } @@ -207,35 +204,41 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ def dirname(path: File): File = new File(realpath(path).string.stripSuffix("/").split("/").dropRight(1).mkString("/")) def nameAndContents(file: File) = basename(file) -> readAllBytes(Paths.get(file.toString)) - def jarFile( jarFile: File, files: Seq[File] ): Unit = { - logger.lib("Start packaging "++jarFile.string) - val manifest = new Manifest - manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0") - val jar = new JarOutputStream(new FileOutputStream(jarFile.toString), manifest) - - val names = for { - base <- files.filter(_.exists).map(realpath) - file <- listFilesRecursive(base) if file.isFile - } yield { - val name = if(base.isDirectory){ - file.toString stripPrefix base.toString - } else file.toString - val entry = new JarEntry( name ) - entry.setTime(file.lastModified) - jar.putNextEntry(entry) - jar.write( readAllBytes( Paths.get(file.toString) ) ) - jar.closeEntry - name - } + def jarFile( jarFile: File, files: Seq[File] ): Option[File] = { + if( files.isEmpty ){ + None + } else { + logger.lib("Start packaging "++jarFile.string) + val manifest = new Manifest + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0") + val jar = new JarOutputStream(new FileOutputStream(jarFile.toString), manifest) + + val names = for { + base <- files.filter(_.exists).map(realpath) + file <- listFilesRecursive(base) if file.isFile + } yield { + val name = if(base.isDirectory){ + file.toString stripPrefix base.toString + } else file.toString + val entry = new JarEntry( name ) + entry.setTime(file.lastModified) + jar.putNextEntry(entry) + jar.write( readAllBytes( Paths.get(file.toString) ) ) + jar.closeEntry + name + } - val duplicateFiles = (names diff names.distinct).distinct - assert( - duplicateFiles.isEmpty, - s"Conflicting file names when trying to create $jarFile: "++duplicateFiles.mkString(", ") - ) + val duplicateFiles = (names diff names.distinct).distinct + assert( + duplicateFiles.isEmpty, + s"Conflicting file names when trying to create $jarFile: "++duplicateFiles.mkString(", ") + ) - jar.close - logger.lib("Done packaging " ++ jarFile.toString) + jar.close + logger.lib("Done packaging " ++ jarFile.toString) + + Some(jarFile) + } } lazy val passphrase = @@ -321,8 +324,9 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ </dependencies> </project> val path = jarTarget.toString ++ ( "/" ++ artifactId ++ "-" ++ version ++ ".pom" ) - write.over(Path(path), "<?xml version='1.0' encoding='UTF-8'?>\n" ++ xml.toString) - new File(path) + val file = new File(path) + Files.write(file.toPath, ("<?xml version='1.0' encoding='UTF-8'?>\n" ++ xml.toString).getBytes) + file } def concurrently[T,R]( concurrencyEnabled: Boolean )( items: Seq[T] )( projection: T => R ): Seq[R] = { @@ -364,7 +368,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ val httpCon = url.openConnection.asInstanceOf[HttpURLConnection] httpCon.setDoOutput(true) httpCon.setRequestMethod("PUT") - val userPassword = read(Path(sonatypeLogin)).trim + val userPassword = new String(readAllBytes(sonatypeLogin.toPath)).trim val encoding = new sun.misc.BASE64Encoder().encode(userPassword.getBytes) httpCon.setRequestProperty("Authorization", "Basic " ++ encoding) httpCon.setRequestProperty("Content-Type", "application/binary") diff --git a/stage2/NameTransformer.scala b/stage2/NameTransformer.scala new file mode 100644 index 0000000..33489ca --- /dev/null +++ b/stage2/NameTransformer.scala @@ -0,0 +1,161 @@ +// Adapted from https://github.com/scala/scala/blob/5cb3d4ec14488ce2fc5a1cc8ebdd12845859c57d/src/library/scala/reflect/NameTransformer.scala +/* __ *\ +** ________ ___ / / ___ Scala API ** +** / __/ __// _ | / / / _ | (c) 2003-2013, LAMP/EPFL ** +** __\ \/ /__/ __ |/ /__/ __ | http://scala-lang.org/ ** +** /____/\___/_/ |_/____/_/ | | ** +** |/ ** +\* */ + +package cbt + +/** Provides functions to encode and decode Scala symbolic names. + * Also provides some constants. + */ +object NameTransformer { + // XXX Short term: providing a way to alter these without having to recompile + // the compiler before recompiling the compiler. + val MODULE_SUFFIX_STRING = sys.props.getOrElse("SCALA_MODULE_SUFFIX_STRING", "$") + val NAME_JOIN_STRING = sys.props.getOrElse("SCALA_NAME_JOIN_STRING", "$") + val MODULE_INSTANCE_NAME = "MODULE$" + val LOCAL_SUFFIX_STRING = " " + val SETTER_SUFFIX_STRING = "_$eq" + val TRAIT_SETTER_SEPARATOR_STRING = "$_setter_$" + + private val nops = 128 + private val ncodes = 26 * 26 + + private class OpCodes(val op: Char, val code: String, val next: OpCodes) + + private val op2code = new Array[String](nops) + private val code2op = new Array[OpCodes](ncodes) + private def enterOp(op: Char, code: String) = { + op2code(op.toInt) = code + val c = (code.charAt(1) - 'a') * 26 + code.charAt(2) - 'a' + code2op(c.toInt) = new OpCodes(op, code, code2op(c)) + } + + /* Note: decoding assumes opcodes are only ever lowercase. */ + enterOp('~', "$tilde") + enterOp('=', "$eq") + enterOp('<', "$less") + enterOp('>', "$greater") + enterOp('!', "$bang") + enterOp('#', "$hash") + enterOp('%', "$percent") + enterOp('^', "$up") + enterOp('&', "$amp") + enterOp('|', "$bar") + enterOp('*', "$times") + enterOp('/', "$div") + enterOp('+', "$plus") + enterOp('-', "$minus") + enterOp(':', "$colon") + enterOp('\\', "$bslash") + enterOp('?', "$qmark") + enterOp('@', "$at") + + /** Replace operator symbols by corresponding `\$opname`. + * + * @param name the string to encode + * @return the string with all recognized opchars replaced with their encoding + */ + def encode(name: String): String = { + var buf: StringBuilder = null + val len = name.length() + var i = 0 + while (i < len) { + val c = name charAt i + if (c < nops && (op2code(c.toInt) ne null)) { + if (buf eq null) { + buf = new StringBuilder() + buf.append(name.substring(0, i)) + } + buf.append(op2code(c.toInt)) + /* Handle glyphs that are not valid Java/JVM identifiers */ + } + else if (!Character.isJavaIdentifierPart(c)) { + if (buf eq null) { + buf = new StringBuilder() + buf.append(name.substring(0, i)) + } + buf.append("$u%04X".format(c.toInt)) + } + else if (buf ne null) { + buf.append(c) + } + i += 1 + } + if (buf eq null) name else buf.toString() + } + + /** Replace `\$opname` by corresponding operator symbol. + * + * @param name0 the string to decode + * @return the string with all recognized operator symbol encodings replaced with their name + */ + def decode(name0: String): String = { + //System.out.println("decode: " + name);//DEBUG + val name = if (name0.endsWith("<init>")) name0.stripSuffix("<init>") + "this" + else name0 + var buf: StringBuilder = null + val len = name.length() + var i = 0 + while (i < len) { + var ops: OpCodes = null + var unicode = false + val c = name charAt i + if (c == '$' && i + 2 < len) { + val ch1 = name.charAt(i+1) + if ('a' <= ch1 && ch1 <= 'z') { + val ch2 = name.charAt(i+2) + if ('a' <= ch2 && ch2 <= 'z') { + ops = code2op((ch1 - 'a') * 26 + ch2 - 'a') + while ((ops ne null) && !name.startsWith(ops.code, i)) ops = ops.next + if (ops ne null) { + if (buf eq null) { + buf = new StringBuilder() + buf.append(name.substring(0, i)) + } + buf.append(ops.op) + i += ops.code.length() + } + /* Handle the decoding of Unicode glyphs that are + * not valid Java/JVM identifiers */ + } else if ((len - i) >= 6 && // Check that there are enough characters left + ch1 == 'u' && + ((Character.isDigit(ch2)) || + ('A' <= ch2 && ch2 <= 'F'))) { + /* Skip past "$u", next four should be hexadecimal */ + val hex = name.substring(i+2, i+6) + try { + val str = Integer.parseInt(hex, 16).toChar + if (buf eq null) { + buf = new StringBuilder() + buf.append(name.substring(0, i)) + } + buf.append(str) + /* 2 for "$u", 4 for hexadecimal number */ + i += 6 + unicode = true + } catch { + case _:NumberFormatException => + /* `hex` did not decode to a hexadecimal number, so + * do nothing. */ + } + } + } + } + /* If we didn't see an opcode or encoded Unicode glyph, and the + buffer is non-empty, write the current character and advance + one */ + if ((ops eq null) && !unicode) { + if (buf ne null) + buf.append(c) + i += 1 + } + } + //System.out.println("= " + (if (buf == null) name else buf.toString()));//DEBUG + if (buf eq null) name else buf.toString() + } +} diff --git a/stage2/PackageBuild.scala b/stage2/PackageBuild.scala index 2866b7c..79e54a7 100644 --- a/stage2/PackageBuild.scala +++ b/stage2/PackageBuild.scala @@ -4,23 +4,23 @@ import scala.collection.immutable.Seq abstract class PackageBuild(context: Context) extends BasicBuild(context) with ArtifactInfo{ def `package`: Seq[File] = lib.concurrently( enableConcurrency )( Seq(() => jar, () => docJar, () => srcJar) - )( _() ) + )( _() ).flatten - private object cacheJarBasicBuild extends Cache[File] - def jar: File = cacheJarBasicBuild{ - lib.jar( artifactId, version, compile, jarTarget ) + private object cacheJarBasicBuild extends Cache[Option[File]] + def jar: Option[File] = cacheJarBasicBuild{ + compile.flatMap( lib.jar( artifactId, version, _, jarTarget ) ) } - private object cacheSrcJarBasicBuild extends Cache[File] - def srcJar: File = cacheSrcJarBasicBuild{ + private object cacheSrcJarBasicBuild extends Cache[Option[File]] + def srcJar: Option[File] = cacheSrcJarBasicBuild{ lib.srcJar( sourceFiles, artifactId, version, scalaTarget ) } - private object cacheDocBasicBuild extends Cache[File] - def docJar: File = cacheDocBasicBuild{ - lib.docJar( scalaVersion, sourceFiles, dependencyClasspath, apiTarget, jarTarget, artifactId, version, scalacOptions ) + private object cacheDocBasicBuild extends Cache[Option[File]] + def docJar: Option[File] = cacheDocBasicBuild{ + lib.docJar( scalaVersion, sourceFiles, dependencyClasspath, apiTarget, jarTarget, artifactId, version, scalacOptions, context.classLoaderCache ) } - override def jars = jar +: dependencyJars - override def exportedJars: Seq[File] = Seq(jar) + override def jars = jar.toVector ++ dependencyJars + override def exportedJars: Seq[File] = jar.toVector } diff --git a/stage2/Scaffold.scala b/stage2/Scaffold.scala index e181ebf..3dcb9ae 100644 --- a/stage2/Scaffold.scala +++ b/stage2/Scaffold.scala @@ -1,13 +1,14 @@ package cbt import java.io._ +import java.nio.file._ import java.net._ -import ammonite.ops.{cwd => _,_} - trait Scaffold{ def logger: Logger private def createFile( projectDirectory: File, fileName: String, code: String ){ - write( Path( projectDirectory.string ++ "/" ++ fileName ), code ) + val outputFile = projectDirectory ++ ("/" ++ fileName) + outputFile.getParentFile.mkdirs + Files.write( ( outputFile ).toPath, code.getBytes, StandardOpenOption.CREATE_NEW ) import scala.Console._ println( GREEN ++ "Created " ++ fileName ++ RESET ) } diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index 4145e55..e893a06 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -8,26 +8,24 @@ import scala.collection.immutable.Seq import cbt.paths._ +object Stage2 extends Stage2Base{ + def run( args: Stage2Args ): Unit = { + import args.logger -object Stage2{ - def main(args: Array[String]): Unit = { - val init = new Init(args) - import init._ + val lib = new Lib(args.logger) - val lib = new Lib(init.logger) - - init.logger.stage2(s"[$now] Stage2 start") - val loop = argsV.lift(1) == Some("loop") - val direct = argsV.lift(1) == Some("direct") + logger.stage2(s"[$now] Stage2 start") + val loop = args.args.lift(0) == Some("loop") + val direct = args.args.lift(0) == Some("direct") val taskIndex = if (loop || direct) { - 2 - } else { 1 + } else { + 0 } - val task = argsV.lift( taskIndex ) + val task = args.args.lift( taskIndex ) - val context = Context( new File(argsV(0)), argsV.drop( taskIndex + 1 ), logger ) + val context = Context( args.cwd, args.args.drop( taskIndex ), logger, /*args.cbtHasChanged,*/ new ClassLoaderCache(logger) ) val first = lib.loadRoot( context ) val build = first.finalBuild @@ -47,14 +45,15 @@ object Stage2{ scala.util.control.Breaks.break case file if triggerFiles.exists(file.toString startsWith _.toString) => - val reflectBuild = new lib.ReflectBuild( lib.loadDynamic(context) ) - logger.loop(s"Re-running $task for " ++ reflectBuild.build.projectDirectory.toString) + val build = lib.loadDynamic(context) + val reflectBuild = new lib.ReflectBuild( build ) + logger.loop(s"Re-running $task for " ++ build.projectDirectory.toString) reflectBuild.callNullary(task) } } else { new lib.ReflectBuild(build).callNullary(task) } - init.logger.stage2(s"[$now] Stage2 end") + logger.stage2(s"[$now] Stage2 end") } } diff --git a/stage2/mixins.scala b/stage2/mixins.scala index 2b38cdf..c3a57da 100644 --- a/stage2/mixins.scala +++ b/stage2/mixins.scala @@ -20,16 +20,13 @@ trait ScalaTest extends Build with Test{ "org.scalatest" %% "scalatest" % scalaTestVersion ) ++ super.dependencies - // workaround probable ScalaTest bug throwing away the outer classloader. Not caching doesn't nest them. - override def cacheDependencyClassLoader = false - override def run: ExitCode = { val discoveryPath = compile.toString++"/" context.logger.lib("discoveryPath: " ++ discoveryPath) lib.runMain( "org.scalatest.tools.Runner", Seq("-R", discoveryPath, "-oF") ++ context.args.drop(1), - classLoader + classLoader(context.classLoaderCache) ) } } diff --git a/test/simple/Main.scala b/test/simple/Main.scala index 43542ff..1c423ca 100644 --- a/test/simple/Main.scala +++ b/test/simple/Main.scala @@ -1,4 +1,6 @@ import ai.x.diff +import org.eclipse.jgit.lib.Ref +import com.spotify.missinglink.ArtifactLoader object Main extends App{ println(diff.DiffShow.diff("a","b")) } diff --git a/test/simple/build/build.scala b/test/simple/build/build.scala index f3efc19..d3887b3 100644 --- a/test/simple/build/build.scala +++ b/test/simple/build/build.scala @@ -6,7 +6,9 @@ class Build(context: cbt.Context) extends BasicBuild(context){ ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), JavaDependency("joda-time", "joda-time", "2.9.2"), GitDependency("https://github.com/xdotai/diff.git", "2e275642041006ff39efde22da7742c2e9a0f63f"), - // the below tests pom inheritance with dependencyManagement and variable substitution - JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") + // the below tests pom inheritance with dependencyManagement and variable substitution for pom properties + JavaDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), + // the below tests pom inheritance with variable substitution for pom xml tag contents + JavaDependency("com.spotify", "missinglink-core", "0.1.1") ) ++ super.dependencies } diff --git a/test/test.scala b/test/test.scala index 47bd28b..7261287 100644 --- a/test/test.scala +++ b/test/test.scala @@ -4,9 +4,9 @@ import scala.collection.immutable.Seq // micro framework object Main{ - def main(args: Array[String]): Unit = { - val init = new Init(args) - implicit val logger: Logger = init.logger + def main(_args: Array[String]): Unit = { + val args = new Stage1ArgsParser(_args.toVector) + implicit val logger: Logger = new Logger(args.enabledLoggers) var successes = 0 var failures = 0 @@ -24,9 +24,9 @@ object Main{ }.get } - def runCbt(path: String, args: Seq[String])(implicit logger: Logger): Result = { + def runCbt(path: String, _args: Seq[String])(implicit logger: Logger): Result = { import java.io._ - val allArgs: Seq[String] = ((cbtHome.string ++ "/cbt") +: "direct" +: (args ++ init.propsRaw)) + val allArgs: Seq[String] = ((cbtHome.string ++ "/cbt") +: "direct" +: (_args ++ args.propsRaw)) logger.test(allArgs.toString) val pb = new ProcessBuilder( allArgs :_* ) pb.directory(cbtHome ++ ("/test/" ++ path)) @@ -41,7 +41,7 @@ object Main{ } case class Result(exit0: Boolean, out: String, err: String) def assertSuccess(res: Result, msg: => String)(implicit logger: Logger) = { - assert(res.exit0, msg + res.toString) + assert(res.exit0, msg ++ res.toString) } // tests @@ -49,19 +49,19 @@ object Main{ val usageString = "Methods provided by CBT" val res = runCbt(path, Seq()) logger.test(res.toString) - val debugToken = "usage " + path +" " + val debugToken = "usage " ++ path ++ " " assertSuccess(res,debugToken) - assert(res.out == "", debugToken+ res.toString) - assert(res.err contains usageString, debugToken+res.toString) + assert(res.out == "", debugToken ++ res.toString) + assert(res.err contains usageString, debugToken ++ res.toString) } def compile(path: String)(implicit logger: Logger) = { val res = runCbt(path, Seq("compile")) - val debugToken = "compile " + path +" " + val debugToken = "compile " ++ path ++ " " assertSuccess(res,debugToken) // assert(res.err == "", res.err) // FIXME: enable this } - logger.test( "Running tests " ++ args.toList.toString ) + logger.test( "Running tests " ++ _args.toList.toString ) usage("nothing") compile("nothing") @@ -71,7 +71,7 @@ object Main{ compile("simple") { - val noContext = Context(cbtHome ++ "/test/nothing", Seq(), logger) + val noContext = Context(cbtHome ++ "/test/nothing", Seq(), logger, new ClassLoaderCache(logger)) val b = new Build(noContext){ override def dependencies = Seq( JavaDependency("net.incongru.watchservice","barbary-watchservice","1.0"), |