diff options
50 files changed, 1522 insertions, 682 deletions
diff --git a/DEVELOPER_GUIDE.txt b/DEVELOPER_GUIDE.txt index 0e20339..a26d760 100644 --- a/DEVELOPER_GUIDE.txt +++ b/DEVELOPER_GUIDE.txt @@ -1,17 +1,28 @@ Welcome developer. -CBT has a very easy code base that you can fully master it in an afternoon. +CBT has a very easy code base that's easy to master. Don't shy away from submiting PRs :). And because CBT bootstraps from source you already have the code there. +The only tricky parts are class loading and cache invalidation. Most changes +will not need to interfere with this though. + The ./cbt bash script starts the process. You currently need javac, nailgun, gpg and realpath or gcc installed. +If you have any troubles with class not found, method not found, +abstract method error, NullPointerException, etc. +Try `killall -KILL java`. To restart nailgun. +Or try `cbt direct <taskname>` to circumvent nailgun. +It can also help to delete all target folders `find .|grep target\$|xargs rm -rf` +inside of CBT. Or (almost never) the `cache/` directory. + CBT's directory structure cbt Shell script launching cbt. Can be symlinked. -bootstrap_scala/ Self-contained downloader for the core Scala jars. Allows bootstrapping from Java into Scala. +compatibility/ Java interfaces that all CBT versions are source compatible to. For communication + between composed builds of different versions. nailgun_launcher/ Self-contained helper that allows using Nailgun with minimal permanent classpath. (Is this actually needed?) realpath/ Self-contained realpath source code to correctly figure our CBTs home directory. (Open for replacement ideas.) stage1/ CBT's code that only relies only on Scala/Java built-ins. Contains a Maven resolver to download libs for stage2. @@ -50,7 +50,7 @@ class Build(context: cbt.Context) extends PackageBuild(context){ override def artifactId = "play-json-extensions" override def dependencies = super.dependencies :+ - MavenRepository.central.resolve( + MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( // encouraged way to declare dependencies ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), MavenDependency("joda-time", "joda-time", "2.9.2") @@ -71,16 +71,16 @@ You can see how your build is configured via overrides. call `cbt` to see a full list of available commands for this build. -Look into the class `DefaultBuild` in CBT's source code to see their +Look into the class PackageBuild (and it's super class BasicBuild) in CBT's source code to see their details. The source code is really simple. Don't shy away from looking, even as a beginner. No crazy stuff, I promise ;). You -can find the relevant code in CBT's `stage2/DefaultBuild.scala` +can find the relevant code in CBT's stage2/BasicBuild.scala I order to keep executing the same command triggered by file changes use `cbt loop <command>`. You can find example builds in CBT's own `test/` folder. Not all of them have a build file, in which case CBT uses the default -`cbt.DefaultBuild`. +cbt.BasicBuild. A folder `build/` can have its own folder `build/` inside in order to add source or maven dependencies to your build. Eventually diff --git a/build/build.scala b/build/build.scala index 6dfd395..1622f01 100644 --- a/build/build.scala +++ b/build/build.scala @@ -5,12 +5,14 @@ import scala.collection.immutable.Seq class Build(context: Context) extends BasicBuild(context){ // FIXME: somehow consolidate this with cbt's own boot-strapping from source. - override def dependencies = super.dependencies :+ MavenRepository.central.resolve( - MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), - MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), - MavenDependency("com.typesafe.zinc","zinc","0.3.9"), - ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5") - ) + override def dependencies = { + super.dependencies :+ MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( + MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), + MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), + MavenDependency("com.typesafe.zinc","zinc","0.3.9"), + ScalaDependency("org.scala-lang.modules","scala-xml","1.0.5") + ) :+ BinaryDependency(new File(System.getenv("CBT_HOME")+"/compatibility"), Seq()) + } override def sources = Seq( "nailgun_launcher", "stage1", "stage2" ).map(d => projectDirectory ++ ("/" + d)) @@ -8,6 +8,19 @@ # - reduction of dependencies # - performance improvements +start_seconds=$(gdate +"%s") +start_nanos=1$(gdate +"%N") + +time_taken() { + i=$(( $(gdate +"%s") - start_seconds )) + n=$(( $(( 1$(gdate +"%N") - start_nanos )) / 1000000 )) + if [[ ( "$n" < 0 ) ]]; then + i=$(( i-1 )) + n=$(( n+1000 )) + fi + echo "$i.$n" +} + # utility function to log message to stderr with stating the time log () { msg=$1 @@ -23,8 +36,8 @@ log () { which gdate 2>&1 > /dev/null gdate_installed=$? if [ $gdate_installed -eq 0 ]; then - i=`gdate +"%S.%N"` - echo "[$i] $msg" 1>&2 + delta=$(time_taken) + echo "[$delta] $msg" 1>&2 fi fi } @@ -34,16 +47,22 @@ log "Checking for dependencies" $* which javac 2>&1 > /dev/null javac_installed=$? if [ ! $javac_installed -eq 0 ]; then - echo "You need to install javac! CBT needs it to bootstrap from Java sources into Scala." 1>&2 - exit 1 -fi -javac_version=$(javac -version 2>&1 | cut -d ' ' -f 2) -javac_version_minor=$(echo -n $javac_version | cut -d '.' -f 2) -if [ ! "$javac_version_minor" -ge "7" ]; then - echo "You need to install javac version 1.7 or greater!" 2>&1 - echo "Current javac version is $javac_version" 2>&1 + echo "You need to install javac 1.7 or later! CBT needs it to bootstrap from Java sources into Scala." 1>&2 exit 1 fi + +# log "cutting javac version" $* +# javac_version=$(javac -version 2>&1) # e.g. "javac 1.8.0_u60" +# javac_version_update=${javac_version/javac 1./} # e.g. "8.0_u60" +# javac_version_minor_pointed=${javac_version_update%_*} # e.g. "8.0" +# javac_version_minor=${javac_version_minor_pointed%.*} # e.g. "8" +# log "cutting javac version done" $* +# if [ ! "$javac_version_minor" -ge "7" ]; then +# echo "You need to install javac version 1.7 or greater!" 2>&1 +# echo "Current javac version is $javac_version" 2>&1 +# exit 1 +# fi + which ng 2>&1 > /dev/null ng_installed=$? which ng-server 2>&1 > /dev/null @@ -152,8 +171,10 @@ stage1 () { if [ $use_nailgun -eq 0 ]; then echo "Stopping nailgun" 1>&2 $NG ng-stop >> $nailgun_out 2>> $nailgun_err & + sleep 1 echo "Restarting nailgun" 1>&2 ng-server 127.0.0.1:$NAILGUN_PORT >> $nailgun_out 2>> $nailgun_err & + sleep 1 fi fi @@ -163,7 +184,7 @@ stage1 () { then log "Running JVM directly" $* # -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:5005 - java -cp $NAILGUN$TARGET cbt.NailgunLauncher "$CWD" $* + java -Xmx6072m -Xss10M -cp $NAILGUN$TARGET cbt.NailgunLauncher $(time_taken) "$CWD" $* else log "Running via nailgun." $* for i in 0 1 2 3 4 5 6 7 8 9; do @@ -186,7 +207,7 @@ stage1 () { sleep 0.3 done log "Running CBT via Nailgun." $* - $NG cbt.NailgunLauncher "$CWD" $* + $NG cbt.NailgunLauncher $(time_taken) "$CWD" $* fi exitCode=$? log "Done running CBT." $* diff --git a/compatibility/ArtifactInfo.java b/compatibility/ArtifactInfo.java new file mode 100644 index 0000000..a2e6006 --- /dev/null +++ b/compatibility/ArtifactInfo.java @@ -0,0 +1,7 @@ +package cbt; + +public interface ArtifactInfo extends Dependency{ + public abstract String artifactId(); + public abstract String groupId(); + public abstract String version(); +} diff --git a/compatibility/BuildInterface.java b/compatibility/BuildInterface.java new file mode 100644 index 0000000..fea43be --- /dev/null +++ b/compatibility/BuildInterface.java @@ -0,0 +1,11 @@ +package cbt; +import java.io.*; + +public interface BuildInterface extends Dependency{ + public abstract BuildInterface copy(Context context); // needed to configure builds + public abstract String show(); // needed for debugging + public abstract String scalaVersion(); // needed to propagate scalaVersion to dependent builds + public abstract String[] crossScalaVersionsArray(); // FIXME: this probably can't use Scala classes + public abstract BuildInterface finalBuild(); // needed to propagage through build builds. Maybe we can get rid of this. + public abstract File[] triggerLoopFilesArray(); // needed for watching files across composed builds +} diff --git a/compatibility/Context.java b/compatibility/Context.java new file mode 100644 index 0000000..387c24a --- /dev/null +++ b/compatibility/Context.java @@ -0,0 +1,21 @@ +package cbt; +import java.io.*; +import java.util.concurrent.ConcurrentHashMap; + +// TODO: try to reduce the number of members +public abstract class Context{ + public abstract File projectDirectory(); + public abstract File cwd(); + public abstract String[] argsArray(); + public abstract String[] enabledLoggersArray(); + public abstract Long startCompat(); + public abstract Boolean cbtHasChangedCompat(); + public abstract String versionOrNull(); + public abstract String scalaVersionOrNull(); // needed to propagate scalaVersion to dependendee builds + public abstract ConcurrentHashMap<String,Object> permanentKeys(); + public abstract ConcurrentHashMap<Object,ClassLoader> permanentClassLoaders(); + public abstract File cache(); + public abstract File cbtHome(); + public abstract File compatibilityTarget(); + public abstract BuildInterface parentBuildOrNull(); +} diff --git a/compatibility/Dependency.java b/compatibility/Dependency.java new file mode 100644 index 0000000..d491174 --- /dev/null +++ b/compatibility/Dependency.java @@ -0,0 +1,10 @@ +package cbt; +import java.io.*; + +public interface Dependency{ + public abstract String show(); + public abstract Boolean needsUpdateCompat(); + public abstract Dependency[] dependenciesArray(); + public abstract File[] dependencyClasspathArray(); + public abstract File[] exportedClasspathArray(); +} diff --git a/compatibility/Result.java b/compatibility/Result.java new file mode 100644 index 0000000..220aa3a --- /dev/null +++ b/compatibility/Result.java @@ -0,0 +1,11 @@ +/* +package cbt; +import java.io.*; +public interface Result<T>{ + public abstract Integer exitCode(); + public abstract OutputStream out(); + public abstract OutputStream err(); + public abstract InputStream in(); + public abstract T value(); +} +*/ diff --git a/coursier/Coursier.scala b/coursier/Coursier.scala index 48282b1..8a66aee 100644 --- a/coursier/Coursier.scala +++ b/coursier/Coursier.scala @@ -7,7 +7,7 @@ object Coursier{ import coursier._ val repositories = Seq( Cache.ivy2Local, - MavenRepository("https://repo1.maven.org/maven2") + MavenResolver("https://repo1.maven.org/maven2") ) val start = Resolution( diff --git a/nailgun_launcher/CBTUrlClassLoader.java b/nailgun_launcher/CBTUrlClassLoader.java index c05391b..b799bc0 100644 --- a/nailgun_launcher/CBTUrlClassLoader.java +++ b/nailgun_launcher/CBTUrlClassLoader.java @@ -3,6 +3,7 @@ import java.io.*; import java.net.*; import java.util.*; import static cbt.Stage0Lib.*; +import java.util.concurrent.ConcurrentHashMap; class CbtURLClassLoader extends java.net.URLClassLoader{ public String toString(){ return ( @@ -10,13 +11,35 @@ class CbtURLClassLoader extends java.net.URLClassLoader{ + "(\n " + Arrays.toString(getURLs()) + ",\n " - + join("\n ",getParent().toString().split("\n")) + + join("\n ",(getParent() == null?"":getParent().toString()).split("\n")) + "\n)" ); } + ClassLoaderCache2<Class> cache = new ClassLoaderCache2<Class>( + new ConcurrentHashMap<String, Object>(), + new ConcurrentHashMap<Object, Class>() + ); public Class loadClass(String name) throws ClassNotFoundException{ + Class _class = super.loadClass(name); + if(_class == null) throw new ClassNotFoundException(name); + else return _class; + } + public Class loadClass(String name, Boolean resolve) throws ClassNotFoundException{ //System.out.println("loadClass("+name+") on \n"+this); - return super.loadClass(name); + if(!cache.contains(name)) + try{ + cache.put(super.loadClass(name, resolve), name); + } catch (ClassNotFoundException e){ + cache.put(Object.class, name); + } + Class _class = cache.get(name); + if(_class == Object.class){ + if( name == "java.lang.Object" ) + return Object.class; + else return null; + } else { + return _class; + } } void assertExist(URL[] urls){ for(URL url: urls){ @@ -30,7 +53,7 @@ class CbtURLClassLoader extends java.net.URLClassLoader{ assertExist(urls); } public CbtURLClassLoader(URL[] urls){ - super(urls); + super(urls, null); assertExist(urls); } }
\ No newline at end of file diff --git a/nailgun_launcher/ClassLoaderCache2.java b/nailgun_launcher/ClassLoaderCache2.java new file mode 100644 index 0000000..bf9ca3b --- /dev/null +++ b/nailgun_launcher/ClassLoaderCache2.java @@ -0,0 +1,37 @@ +package cbt; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import static java.io.File.pathSeparator; +import static cbt.Stage0Lib.*; + +final class ClassLoaderCache2<T>{ + ConcurrentHashMap<String,Object> keys; + ConcurrentHashMap<Object,T> values; + + public ClassLoaderCache2( + ConcurrentHashMap<String,Object> keys, + ConcurrentHashMap<Object,T> values + ){ + this.keys = keys; + this.values = values; + } + + public T get( String key ){ + return values.get( + keys.get( key ) + ); + } + + public Boolean contains( String key ){ + return keys.containsKey( key ); + } + + public T put( T value, String key ){ + LockableKey2 keyObject = new LockableKey2(); + keys.put( key, keyObject ); + values.put( keyObject, value ); + return value; + } +} +class LockableKey2{}
\ No newline at end of file diff --git a/nailgun_launcher/EarlyDependencies.java b/nailgun_launcher/EarlyDependencies.java index f4d446c..7a8d033 100644 --- a/nailgun_launcher/EarlyDependencies.java +++ b/nailgun_launcher/EarlyDependencies.java @@ -10,90 +10,133 @@ import static cbt.NailgunLauncher.*; class EarlyDependencies{ /** ClassLoader for stage1 */ - ClassLoader stage1; + ClassLoader classLoader; + String[] classpathArray; /** ClassLoader for zinc */ ClassLoader zinc; - String scalaReflect_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; - String scalaCompiler_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; - String scalaXml_1_0_5_File = MAVEN_CACHE + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"; - String scalaLibrary_2_11_8_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"; - String zinc_0_3_9_File = MAVEN_CACHE + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"; - String incrementalCompiler_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"; - String compilerInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"; - String scalaCompiler_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"; - String sbtInterface_0_13_9_File = MAVEN_CACHE + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"; - String scalaReflect_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"; - String scalaLibrary_2_10_5_File = MAVEN_CACHE + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"; - - public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ - download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"), Paths.get(scalaReflect_2_11_8_File), "b74530deeba742ab4f3134de0c2da0edc49ca361"); - download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); + String scalaReflect_2_11_8_File; + String scalaCompiler_2_11_8_File; + String scalaXml_1_0_5_File; + String scalaLibrary_2_11_8_File; + String zinc_0_3_9_File; + String incrementalCompiler_0_13_9_File; + String compilerInterface_0_13_9_File; + String scalaCompiler_2_10_5_File; + String sbtInterface_0_13_9_File; + String scalaReflect_2_10_5_File; + String scalaLibrary_2_10_5_File; + + public EarlyDependencies( + String mavenCache, String mavenUrl, ClassLoaderCache2<ClassLoader> classLoaderCache, ClassLoader rootClassLoader + ) throws Exception { + scalaReflect_2_11_8_File = mavenCache + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"; + scalaCompiler_2_11_8_File = mavenCache + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"; + scalaXml_1_0_5_File = mavenCache + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"; + scalaLibrary_2_11_8_File = mavenCache + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"; + zinc_0_3_9_File = mavenCache + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"; + incrementalCompiler_0_13_9_File = mavenCache + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"; + compilerInterface_0_13_9_File = mavenCache + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"; + scalaCompiler_2_10_5_File = mavenCache + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"; + sbtInterface_0_13_9_File = mavenCache + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"; + scalaReflect_2_10_5_File = mavenCache + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"; + scalaLibrary_2_10_5_File = mavenCache + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"; + + download(new URL(mavenUrl + "/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar"), Paths.get(scalaReflect_2_11_8_File), "b74530deeba742ab4f3134de0c2da0edc49ca361"); + download(new URL(mavenUrl + "/org/scala-lang/scala-compiler/2.11.8/scala-compiler-2.11.8.jar"), Paths.get(scalaCompiler_2_11_8_File), "fe1285c9f7b58954c5ef6d80b59063569c065e9a"); // org.scala-lang:scala-library:2.10.5 - download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"), Paths.get(scalaLibrary_2_10_5_File), "57ac67a6cf6fd591e235c62f8893438e8d10431d"); - ClassLoader scalaLibrary_2_10_5_ = cachePut( - classLoader( scalaLibrary_2_10_5_File ), - scalaLibrary_2_10_5_File - ); + download(new URL(mavenUrl + "/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar"), Paths.get(scalaLibrary_2_10_5_File), "57ac67a6cf6fd591e235c62f8893438e8d10431d"); + + String[] scalaLibrary_2_10_5_ClasspathArray = new String[]{scalaLibrary_2_10_5_File}; + String scalaLibrary_2_10_5_Classpath = classpath( scalaLibrary_2_10_5_ClasspathArray ); + ClassLoader scalaLibrary_2_10_5_ = + classLoaderCache.contains( scalaLibrary_2_10_5_Classpath ) + ? classLoaderCache.get( scalaLibrary_2_10_5_Classpath ) + : classLoaderCache.put( classLoader( scalaLibrary_2_10_5_File, rootClassLoader ), scalaLibrary_2_10_5_Classpath ); // org.scala-lang:scala-reflect:2.10.5 - download(new URL(MAVEN_URL + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"), Paths.get(scalaReflect_2_10_5_File), "7392facb48876c67a89fcb086112b195f5f6bbc3"); - ClassLoader scalaReflect_2_10_5_ = cachePut( - classLoader( scalaReflect_2_10_5_File, scalaLibrary_2_10_5_ ), - scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar"), Paths.get(scalaReflect_2_10_5_File), "7392facb48876c67a89fcb086112b195f5f6bbc3"); + + String[] scalaReflect_2_10_5_ClasspathArray = new String[]{scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String scalaReflect_2_10_5_Classpath = classpath( scalaReflect_2_10_5_ClasspathArray ); + ClassLoader scalaReflect_2_10_5_ = + classLoaderCache.contains( scalaReflect_2_10_5_Classpath ) + ? classLoaderCache.get( scalaReflect_2_10_5_Classpath ) + : classLoaderCache.put( classLoader( scalaReflect_2_10_5_File, scalaLibrary_2_10_5_ ), scalaReflect_2_10_5_Classpath ); // com.typesafe.sbt:sbt-interface:0.13.9 - download(new URL(MAVEN_URL + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"), Paths.get(sbtInterface_0_13_9_File), "29848631415402c81b732e919be88f268df37250"); - ClassLoader sbtInterface_0_13_9_ = cachePut( - classLoader( sbtInterface_0_13_9_File, scalaReflect_2_10_5_ ), - sbtInterface_0_13_9_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/com/typesafe/sbt/sbt-interface/0.13.9/sbt-interface-0.13.9.jar"), Paths.get(sbtInterface_0_13_9_File), "29848631415402c81b732e919be88f268df37250"); + + String[] sbtInterface_0_13_9_ClasspathArray = new String[]{sbtInterface_0_13_9_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String sbtInterface_0_13_9_Classpath = classpath( sbtInterface_0_13_9_ClasspathArray ); + ClassLoader sbtInterface_0_13_9_ = + classLoaderCache.contains( sbtInterface_0_13_9_Classpath ) + ? classLoaderCache.get( sbtInterface_0_13_9_Classpath ) + : classLoaderCache.put( classLoader( sbtInterface_0_13_9_File, scalaReflect_2_10_5_ ), sbtInterface_0_13_9_Classpath ); // org.scala-lang:scala-compiler:2.10.5 - download(new URL(MAVEN_URL + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"), Paths.get(scalaCompiler_2_10_5_File), "f0f5bb444ca26a6e489af3dd35e24f7e2d2d118e"); - ClassLoader scalaCompiler_2_10_5_ = cachePut( - classLoader( scalaCompiler_2_10_5_File, sbtInterface_0_13_9_ ), - sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/org/scala-lang/scala-compiler/2.10.5/scala-compiler-2.10.5.jar"), Paths.get(scalaCompiler_2_10_5_File), "f0f5bb444ca26a6e489af3dd35e24f7e2d2d118e"); + + String[] scalaCompiler_2_10_5_ClasspathArray = new String[]{sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String scalaCompiler_2_10_5_Classpath = classpath( scalaCompiler_2_10_5_ClasspathArray ); + ClassLoader scalaCompiler_2_10_5_ = + classLoaderCache.contains( scalaCompiler_2_10_5_Classpath ) + ? classLoaderCache.get( scalaCompiler_2_10_5_Classpath ) + : classLoaderCache.put( classLoader( scalaCompiler_2_10_5_File, sbtInterface_0_13_9_ ), scalaCompiler_2_10_5_Classpath ); // com.typesafe.sbt:compiler-interface:0.13.9 - download(new URL(MAVEN_URL + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"), Paths.get(compilerInterface_0_13_9_File), "2311addbed1182916ad00f83c57c0eeca1af382b"); - ClassLoader compilerInterface_0_13_9_ = cachePut( - classLoader( compilerInterface_0_13_9_File, scalaCompiler_2_10_5_ ), - compilerInterface_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/com/typesafe/sbt/compiler-interface/0.13.9/compiler-interface-0.13.9-sources.jar"), Paths.get(compilerInterface_0_13_9_File), "2311addbed1182916ad00f83c57c0eeca1af382b"); + + String[] compilerInterface_0_13_9_ClasspathArray = new String[]{compilerInterface_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String compilerInterface_0_13_9_Classpath = classpath( compilerInterface_0_13_9_ClasspathArray ); + ClassLoader compilerInterface_0_13_9_ = + classLoaderCache.contains( compilerInterface_0_13_9_Classpath ) + ? classLoaderCache.get( compilerInterface_0_13_9_Classpath ) + : classLoaderCache.put( classLoader( compilerInterface_0_13_9_File, scalaCompiler_2_10_5_ ), compilerInterface_0_13_9_Classpath ); // com.typesafe.sbt:incremental-compiler:0.13.9 - download(new URL(MAVEN_URL + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"), Paths.get(incrementalCompiler_0_13_9_File), "fbbf1cadbed058aa226643e83543c35de43b13f0"); - ClassLoader incrementalCompiler_0_13_9_ = cachePut( - classLoader( incrementalCompiler_0_13_9_File, compilerInterface_0_13_9_ ), - compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/com/typesafe/sbt/incremental-compiler/0.13.9/incremental-compiler-0.13.9.jar"), Paths.get(incrementalCompiler_0_13_9_File), "fbbf1cadbed058aa226643e83543c35de43b13f0"); + + String[] incrementalCompiler_0_13_9_ClasspathArray = new String[]{compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String incrementalCompiler_0_13_9_Classpath = classpath( incrementalCompiler_0_13_9_ClasspathArray ); + ClassLoader incrementalCompiler_0_13_9_ = + classLoaderCache.contains( incrementalCompiler_0_13_9_Classpath ) + ? classLoaderCache.get( incrementalCompiler_0_13_9_Classpath ) + : classLoaderCache.put( classLoader( incrementalCompiler_0_13_9_File, compilerInterface_0_13_9_ ), incrementalCompiler_0_13_9_Classpath ); // com.typesafe.zinc:zinc:0.3.9 - download(new URL(MAVEN_URL + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"), Paths.get(zinc_0_3_9_File), "46a4556d1f36739879f4b2cc19a73d12b3036e9a"); - ClassLoader zinc_0_3_9_ = cachePut( - classLoader( zinc_0_3_9_File, incrementalCompiler_0_13_9_ ), - compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, zinc_0_3_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File - ); + download(new URL(mavenUrl + "/com/typesafe/zinc/zinc/0.3.9/zinc-0.3.9.jar"), Paths.get(zinc_0_3_9_File), "46a4556d1f36739879f4b2cc19a73d12b3036e9a"); + + String[] zinc_0_3_9_ClasspathArray = new String[]{compilerInterface_0_13_9_File, incrementalCompiler_0_13_9_File, sbtInterface_0_13_9_File, zinc_0_3_9_File, scalaCompiler_2_10_5_File, scalaLibrary_2_10_5_File, scalaReflect_2_10_5_File}; + String zinc_0_3_9_Classpath = classpath( zinc_0_3_9_ClasspathArray ); + ClassLoader zinc_0_3_9_ = + classLoaderCache.contains( zinc_0_3_9_Classpath ) + ? classLoaderCache.get( zinc_0_3_9_Classpath ) + : classLoaderCache.put( classLoader( zinc_0_3_9_File, incrementalCompiler_0_13_9_ ), zinc_0_3_9_Classpath ); // org.scala-lang:scala-library:2.11.8 - download(new URL(MAVEN_URL + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); - ClassLoader scalaLibrary_2_11_8_ = cachePut( - classLoader( scalaLibrary_2_11_8_File ), - scalaLibrary_2_11_8_File - ); + download(new URL(mavenUrl + "/org/scala-lang/scala-library/2.11.8/scala-library-2.11.8.jar"), Paths.get(scalaLibrary_2_11_8_File), "ddd5a8bced249bedd86fb4578a39b9fb71480573"); + + String[] scalaLibrary_2_11_8_ClasspathArray = new String[]{scalaLibrary_2_11_8_File}; + String scalaLibrary_2_11_8_Classpath = classpath( scalaLibrary_2_11_8_ClasspathArray ); + ClassLoader scalaLibrary_2_11_8_ = + classLoaderCache.contains( scalaLibrary_2_11_8_Classpath ) + ? classLoaderCache.get( scalaLibrary_2_11_8_Classpath ) + : classLoaderCache.put( classLoader( scalaLibrary_2_11_8_File, rootClassLoader ), scalaLibrary_2_11_8_Classpath ); // org.scala-lang.modules:scala-xml_2.11:1.0.5 - download(new URL(MAVEN_URL + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); - ClassLoader scalaXml_1_0_5_ = cachePut( - classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), - scalaXml_1_0_5_File, scalaLibrary_2_11_8_File - ); + download(new URL(mavenUrl + "/org/scala-lang/modules/scala-xml_2.11/1.0.5/scala-xml_2.11-1.0.5.jar"), Paths.get(scalaXml_1_0_5_File), "77ac9be4033768cf03cc04fbd1fc5e5711de2459"); + + String[] scalaXml_1_0_5_ClasspathArray = new String[]{scalaXml_1_0_5_File, scalaLibrary_2_11_8_File}; + String scalaXml_1_0_5_Classpath = classpath( scalaXml_1_0_5_ClasspathArray ); + ClassLoader scalaXml_1_0_5_ = + classLoaderCache.contains( scalaXml_1_0_5_Classpath ) + ? classLoaderCache.get( scalaXml_1_0_5_Classpath ) + : classLoaderCache.put( classLoader( scalaXml_1_0_5_File, scalaLibrary_2_11_8_ ), scalaXml_1_0_5_Classpath ); - stage1 = scalaXml_1_0_5_; + classLoader = scalaXml_1_0_5_; + classpathArray = scalaXml_1_0_5_ClasspathArray; zinc = zinc_0_3_9_; } diff --git a/nailgun_launcher/MultiClassLoader2.java b/nailgun_launcher/MultiClassLoader2.java new file mode 100644 index 0000000..fadd963 --- /dev/null +++ b/nailgun_launcher/MultiClassLoader2.java @@ -0,0 +1,29 @@ +package cbt; +import java.net.*; +import java.util.*; + +public class MultiClassLoader2 extends ClassLoader{ + public ClassLoader[] parents; + public ClassLoader[] parents(){ + return this.parents; + } + public MultiClassLoader2(ClassLoader... parents){ + super(null); + this.parents = parents; + } + public Class findClass(String name) throws ClassNotFoundException{ + for(ClassLoader parent: parents){ + try{ + return parent.loadClass(name); + } catch (ClassNotFoundException e) { + if(e.getMessage() != name) throw e; + } + } + // FIXME: have a logger in Java land + // System.err.println("NOT FOUND: "+name); + return null; + } + public String toString(){ + return super.toString() + "(" + Arrays.toString(parents) +")"; + } +} diff --git a/nailgun_launcher/NailgunLauncher.java b/nailgun_launcher/NailgunLauncher.java index a12d059..33ae4c2 100644 --- a/nailgun_launcher/NailgunLauncher.java +++ b/nailgun_launcher/NailgunLauncher.java @@ -6,6 +6,7 @@ import java.security.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import static cbt.Stage0Lib.*; +import static java.io.File.pathSeparator; /** * This launcher allows to start the JVM without loading anything else permanently into its @@ -13,98 +14,150 @@ import static cbt.Stage0Lib.*; * dependencies outside the JDK. */ public class NailgunLauncher{ - public static String SCALA_VERSION = "2.11.8"; - public static String SCALA_XML_VERSION = "1.0.5"; - public static String ZINC_VERSION = "0.3.9"; + /** Persistent cache for caching classloaders for the JVM life time. */ + private final static ClassLoaderCache2<ClassLoader> classLoaderCache = new ClassLoaderCache2<ClassLoader>( + new ConcurrentHashMap<String,Object>(), + new ConcurrentHashMap<Object,ClassLoader>() + ); - public static String CBT_HOME = System.getenv("CBT_HOME"); - public static String NAILGUN = System.getenv("NAILGUN"); - public static String TARGET = System.getenv("TARGET"); - public static String STAGE1 = CBT_HOME + "/stage1/"; - public static String MAVEN_CACHE = CBT_HOME + "/cache/maven"; - public static String MAVEN_URL = "https://repo1.maven.org/maven2"; - - /** - * Persistent cache for caching classloaders for the JVM life time. Can be used as needed by user - * code to improve startup time. - */ - public static ConcurrentHashMap<String, Object> classLoaderCacheKeys = new ConcurrentHashMap<String,Object>(); - public static ConcurrentHashMap<Object, ClassLoader> classLoaderCacheValues = new ConcurrentHashMap<Object,ClassLoader>(); - - public static SecurityManager defaultSecurityManager = System.getSecurityManager(); - - public static long lastSuccessfullCompile = 0; - static ClassLoader stage1classLoader = null; - public static ClassLoader stage2classLoader = null; + public final static SecurityManager defaultSecurityManager = System.getSecurityManager(); - public static void main(String[] args) throws ClassNotFoundException, - NoSuchMethodException, - IllegalAccessException, - InvocationTargetException, - MalformedURLException, - IOException, - NoSuchAlgorithmException { - long start = System.currentTimeMillis(); - //System.err.println("ClassLoader: "+stage1classLoader); - //System.err.println("lastSuccessfullCompile: "+lastSuccessfullCompile); - //System.err.println("now: "+now); - - _assert(CBT_HOME != null, CBT_HOME); - _assert(NAILGUN != null, NAILGUN); - _assert(TARGET != null, TARGET); - _assert(STAGE1 != null, STAGE1); + public static String TARGET = System.getenv("TARGET"); + private static String NAILGUN = "nailgun_launcher/"; + private static String STAGE1 = "stage1/"; + + @SuppressWarnings("unchecked") + public static Object getBuild( Object context ) throws Exception{ + BuildStage1Result res = buildStage1( + (Boolean) get(context, "cbtHasChanged"), + (Long) get(context, "start"), + ((File) get(context, "cache")).toString() + "/", + ((File) get(context, "cbtHome")).toString(), + ((File) get(context, "compatibilityTarget")).toString() + "/", + new ClassLoaderCache2<ClassLoader>( + (ConcurrentHashMap<String,Object>) get(context, "permanentKeys"), + (ConcurrentHashMap<Object,ClassLoader>) get(context, "permanentClassLoaders") + ) + ); + return + res + .classLoader + .loadClass("cbt.Stage1") + .getMethod( "getBuild", Object.class, Boolean.class ) + .invoke(null, context, res.changed); + } + public static void main( String[] args ) throws Exception { + Long _start = System.currentTimeMillis(); if(args[0].equals("check-alive")){ System.exit(33); return; } - List<File> stage1SourceFiles = new ArrayList<File>(); - for( File f: new File(STAGE1).listFiles() ){ - if( f.isFile() && f.toString().endsWith(".scala") ){ - stage1SourceFiles.add(f); + String[] diff = args[0].split("\\."); + long start = _start - (Long.parseLong(diff[0]) * 1000L) - Long.parseLong(diff[1]); + + _assert(System.getenv("CBT_HOME") != null, "environment variable CBT_HOME not defined"); + String CBT_HOME = System.getenv("CBT_HOME"); + String cache = CBT_HOME + "/cache/"; + BuildStage1Result res = buildStage1( + false, start, cache, CBT_HOME, CBT_HOME + "/compatibility/" + TARGET, classLoaderCache + ); + + System.exit( + (Integer) res + .classLoader + .loadClass("cbt.Stage1") + .getMethod( + "run", + String[].class, File.class, File.class, Boolean.class, + Long.class, ConcurrentHashMap.class, ConcurrentHashMap.class + ) + .invoke( + null, + (Object) args, new File(cache), new File(CBT_HOME), res.changed, + start, classLoaderCache.keys, classLoaderCache.values + ) + ); + } + + public static BuildStage1Result buildStage1( + Boolean changed, long start, String cache, String cbtHome, String compatibilityTarget, ClassLoaderCache2<ClassLoader> classLoaderCache + ) throws Exception { + _assert(TARGET != null, "environment variable TARGET not defined"); + String nailgunTarget = cbtHome + "/" + NAILGUN + TARGET; + String stage1Sources = cbtHome + "/" + STAGE1; + String stage1Target = stage1Sources + TARGET; + File compatibilitySources = new File(cbtHome + "/compatibility"); + String mavenCache = cache + "maven"; + String mavenUrl = "https://repo1.maven.org/maven2"; + + ClassLoader rootClassLoader = new CbtURLClassLoader( new URL[]{}, ClassLoader.getSystemClassLoader().getParent() ); // wrap for caching + EarlyDependencies earlyDeps = new EarlyDependencies(mavenCache, mavenUrl, classLoaderCache, rootClassLoader); + + List<File> compatibilitySourceFiles = new ArrayList<File>(); + for( File f: compatibilitySources.listFiles() ){ + if( f.isFile() && (f.toString().endsWith(".scala") || f.toString().endsWith(".java")) ){ + compatibilitySourceFiles.add(f); } } + changed = compile(changed, start, "", compatibilityTarget, earlyDeps, compatibilitySourceFiles, defaultSecurityManager); + + ClassLoader compatibilityClassLoader; + if( classLoaderCache.contains( compatibilityTarget ) ){ + compatibilityClassLoader = classLoaderCache.get( compatibilityTarget ); + } else { + compatibilityClassLoader = classLoaderCache.put( classLoader(compatibilityTarget, rootClassLoader), compatibilityTarget ); + } - Boolean changed = lastSuccessfullCompile == 0; - for( File file: stage1SourceFiles ){ - if( file.lastModified() > lastSuccessfullCompile ){ - changed = true; - //System.err.println("File change: "+file.lastModified()); - break; - } + String[] nailgunClasspathArray = append( earlyDeps.classpathArray, nailgunTarget ); + String nailgunClasspath = classpath( nailgunClasspathArray ); + ClassLoader nailgunClassLoader = new CbtURLClassLoader( new URL[]{}, NailgunLauncher.class.getClassLoader() ); // wrap for caching + if( !classLoaderCache.contains( nailgunClasspath ) ){ + nailgunClassLoader = classLoaderCache.put( nailgunClassLoader, nailgunClasspath ); } - if(changed){ - EarlyDependencies earlyDeps = new EarlyDependencies(); - int exitCode = zinc(earlyDeps, stage1SourceFiles); - if( exitCode == 0 ){ - lastSuccessfullCompile = start; - } else { - System.exit( exitCode ); - } + String[] stage1ClasspathArray = + append( append( nailgunClasspathArray, compatibilityTarget ), stage1Target ); + String stage1Classpath = classpath( stage1ClasspathArray ); - ClassLoader nailgunClassLoader; - if( classLoaderCacheKeys.containsKey( NAILGUN+TARGET ) ){ - nailgunClassLoader = cacheGet( NAILGUN+TARGET ); - } else { - nailgunClassLoader = cachePut( classLoader(NAILGUN+TARGET, earlyDeps.stage1), NAILGUN+TARGET ); // FIXME: key is wrong here, should be full CP + List<File> stage1SourceFiles = new ArrayList<File>(); + for( File f: new File(stage1Sources).listFiles() ){ + if( f.isFile() && f.toString().endsWith(".scala") ){ + stage1SourceFiles.add(f); } - - stage1classLoader = classLoader(STAGE1+TARGET, nailgunClassLoader); - stage2classLoader = null; } + changed = compile(changed, start, stage1Classpath, stage1Target, earlyDeps, stage1SourceFiles, defaultSecurityManager); - try{ - Integer exitCode = - (Integer) stage1classLoader - .loadClass("cbt.Stage1") - .getMethod("run", String[].class, ClassLoader.class, Boolean.class, Long.class) - .invoke( null, (Object) args, stage1classLoader, changed, start); - System.exit(exitCode); - }catch(Exception e){ - System.err.println(stage1classLoader); - throw e; + ClassLoader stage1classLoader; + if( !changed && classLoaderCache.contains( stage1Classpath ) ){ + stage1classLoader = classLoaderCache.get( stage1Classpath ); + } else { + stage1classLoader = + classLoaderCache.put( + classLoader( + stage1Target, + new MultiClassLoader2( + nailgunClassLoader, + compatibilityClassLoader, + earlyDeps.classLoader + ) + ), + stage1Classpath + ); } + + return new BuildStage1Result( + changed, + stage1classLoader + ); + } +} +class BuildStage1Result{ + Boolean changed; + ClassLoader classLoader; + BuildStage1Result( Boolean changed, ClassLoader classLoader ){ + this.changed = changed; + this.classLoader = classLoader; } } diff --git a/nailgun_launcher/Stage0Lib.java b/nailgun_launcher/Stage0Lib.java index d6f33e1..bd18748 100644 --- a/nailgun_launcher/Stage0Lib.java +++ b/nailgun_launcher/Stage0Lib.java @@ -8,8 +8,9 @@ import java.security.*; import java.util.*; import javax.xml.bind.annotation.adapters.HexBinaryAdapter; import static java.io.File.pathSeparator; -import static cbt.Stage0Lib.*; import static cbt.NailgunLauncher.*; +import java.nio.file.*; +import java.nio.file.attribute.FileTime; public class Stage0Lib{ public static void _assert(Boolean condition, Object msg){ @@ -18,7 +19,7 @@ public class Stage0Lib{ } } - public static int runMain(String cls, String[] args, ClassLoader cl) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ + public static int runMain(String cls, String[] args, ClassLoader cl, SecurityManager defaultSecurityManager) throws Exception{ try{ System.setSecurityManager( new TrapSecurityManager() ); cl.loadClass(cls) @@ -32,63 +33,78 @@ public class Stage0Lib{ } throw exception; } finally { - System.setSecurityManager(NailgunLauncher.defaultSecurityManager); + System.setSecurityManager(defaultSecurityManager); } } - public static int zinc( EarlyDependencies earlyDeps, List<File> sourceFiles ) throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException{ - String cp = NAILGUN+TARGET + pathSeparator + earlyDeps.scalaXml_1_0_5_File + pathSeparator + earlyDeps.scalaLibrary_2_11_8_File; - List<String> zincArgs = new ArrayList<String>( - Arrays.asList( - new String[]{ - "-scala-compiler", earlyDeps.scalaCompiler_2_11_8_File, - "-scala-library", earlyDeps.scalaLibrary_2_11_8_File, - "-scala-extra", earlyDeps.scalaReflect_2_11_8_File, - "-sbt-interface", earlyDeps.sbtInterface_0_13_9_File, - "-compiler-interface", earlyDeps.compilerInterface_0_13_9_File, - "-cp", cp, - "-d", STAGE1+TARGET - } - ) - ); + public static Object get(Object object, String method) throws Exception{ + return object.getClass().getMethod( method ).invoke(object); + } + + public static String classpath( String... files ){ + Arrays.sort(files); + return join( pathSeparator, files ); + } - for( File f: sourceFiles ){ - zincArgs.add(f.toString()); + public static Boolean compile( + Boolean changed, Long start, String classpath, String target, + EarlyDependencies earlyDeps, List<File> sourceFiles, SecurityManager defaultSecurityManager + ) throws Exception{ + File statusFile = new File( new File(target) + ".last-success" ); + Long lastSuccessfullCompile = statusFile.lastModified(); + for( File file: sourceFiles ){ + if( file.lastModified() > lastSuccessfullCompile ){ + changed = true; + break; + } } + if(changed){ + List<String> zincArgs = new ArrayList<String>( + Arrays.asList( + new String[]{ + "-scala-compiler", earlyDeps.scalaCompiler_2_11_8_File, + "-scala-library", earlyDeps.scalaLibrary_2_11_8_File, + "-scala-extra", earlyDeps.scalaReflect_2_11_8_File, + "-sbt-interface", earlyDeps.sbtInterface_0_13_9_File, + "-compiler-interface", earlyDeps.compilerInterface_0_13_9_File, + "-cp", classpath, + "-d", target + } + ) + ); - PrintStream oldOut = System.out; - try{ - System.setOut(System.err); - return runMain( "com.typesafe.zinc.Main", zincArgs.toArray(new String[zincArgs.size()]), earlyDeps.zinc ); - } finally { - System.setOut(oldOut); + for( File f: sourceFiles ){ + zincArgs.add(f.toString()); + } + + PrintStream oldOut = System.out; + try{ + System.setOut(System.err); + int exitCode = runMain( "com.typesafe.zinc.Main", zincArgs.toArray(new String[zincArgs.size()]), earlyDeps.zinc, defaultSecurityManager ); + if( exitCode == 0 ){ + Files.write( statusFile.toPath(), "".getBytes()); + Files.setLastModifiedTime( statusFile.toPath(), FileTime.fromMillis(start) ); + } else { + System.exit( exitCode ); + } + } finally { + System.setOut(oldOut); + } } + return changed; } - public static ClassLoader classLoader( String file ) throws MalformedURLException{ + public static ClassLoader classLoader( String file ) throws Exception{ return new CbtURLClassLoader( new URL[]{ new URL("file:"+file) } ); } - public static ClassLoader classLoader( String file, ClassLoader parent ) throws MalformedURLException{ + public static ClassLoader classLoader( String file, ClassLoader parent ) throws Exception{ return new CbtURLClassLoader( new URL[]{ new URL("file:"+file) }, parent ); } - public static ClassLoader cacheGet( String key ){ - return classLoaderCacheValues.get( - classLoaderCacheKeys.get( key ) - ); - } - public static ClassLoader cachePut( ClassLoader classLoader, String... jars ){ - String key = join( pathSeparator, jars ); - Object keyObject = new Object(); - classLoaderCacheKeys.put( key, keyObject ); - classLoaderCacheValues.put( keyObject, classLoader ); - return classLoader; - } - - public static void download(URL urlString, Path target, String sha1) throws IOException, NoSuchAlgorithmException { + public static void download(URL urlString, Path target, String sha1) throws Exception { final Path unverified = Paths.get(target+".unverified"); if(!Files.exists(target)) { new File(target.toString()).getParentFile().mkdirs(); @@ -107,7 +123,7 @@ public class Stage0Lib{ } } - public static String sha1(byte[] bytes) throws NoSuchAlgorithmException { + public static String sha1(byte[] bytes) throws Exception { final MessageDigest sha1 = MessageDigest.getInstance("SHA1"); sha1.update(bytes, 0, bytes.length); return (new HexBinaryAdapter()).marshal(sha1.digest()); @@ -120,4 +136,10 @@ public class Stage0Lib{ } return result; } + + public static String[] append( String[] array, String item ){ + String[] copy = Arrays.copyOf(array, array.length + 1); + copy[array.length] = item; + return copy; + } }
\ No newline at end of file diff --git a/plugins/scalatest/ScalaTest.scala b/plugins/scalatest/ScalaTest.scala new file mode 100644 index 0000000..0959087 --- /dev/null +++ b/plugins/scalatest/ScalaTest.scala @@ -0,0 +1,66 @@ +import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq +import org.scalatest._ +import org.scalatest + +/* FIXME: + - Separate out SbtLayout + - Allow depending on this via a git dependency. + Probably by adding support for subfolders to "GitDependency" +*/ + +trait SbtLayout extends BasicBuild{ + outer => + override def sources = Seq( projectDirectory ++ "/src/main/scala" ) + def testSources = projectDirectory ++ "/src/test/scala" + def testDependencies: Seq[Dependency] = Nil + lazy val testBuild = + new BasicBuild(context.copy(projectDirectory = testSources)) with ScalaTest{ + override def target = outer.target + override def compileTarget = outer.scalaTarget ++ "/test-classes" + override def dependencies = (outer +: testDependencies) ++ super.dependencies + } + override def test: Option[ExitCode] = + if(testSources.exists) Some( testBuild.run ) + else None +} + +trait ScalaTest extends BasicBuild{ + override def run: ExitCode = { + import ScalaTestLib._ + val _classLoader = classLoader(context.classLoaderCache) + val suiteNames = compile.map( d => discoverSuites(d, _classLoader) ).toVector.flatten + runSuites( suiteNames.map( loadSuite( _, _classLoader ) ) ) + ExitCode.Success + } +} + +object ScalaTestLib{ + def runSuites(suites: Seq[Suite]) = { + def color: Boolean = true + def durations: Boolean = true + def shortstacks: Boolean = true + def fullstacks: Boolean = true + def stats: Boolean = true + def testName: String = null + def configMap: ConfigMap = ConfigMap.empty + suites.foreach{ + _.execute(testName, configMap, color, durations, shortstacks, fullstacks, stats) + } + } + + def discoverSuites(discoveryPath: File, _classLoader: ClassLoader): Seq[String] = { + _classLoader + .loadClass("org.scalatest.tools.SuiteDiscoveryHelper") + .getMethod("discoverSuiteNames", classOf[List[_]], classOf[ClassLoader], classOf[Option[_]]) + .invoke(null, List(discoveryPath.string ++ "/"), _classLoader, None) + .asInstanceOf[Set[String]] + .to + } + def loadSuite(name: String, _classLoader: ClassLoader) = { + _classLoader.loadClass(name).getConstructor().newInstance().asInstanceOf[Suite] + } +} + diff --git a/plugins/scalatest/build/build.scala b/plugins/scalatest/build/build.scala new file mode 100644 index 0000000..d36d940 --- /dev/null +++ b/plugins/scalatest/build/build.scala @@ -0,0 +1,13 @@ +import cbt._ +import java.net.URL +import java.io.File +import scala.collection.immutable.Seq + +class Build(context: Context) extends BasicBuild(context){ + override def dependencies = super.dependencies ++ Seq( + MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( + ScalaDependency("org.scalatest","scalatest","2.2.4") + ), + context.cbtDependency + ) +} diff --git a/stage1/Cache.scala b/stage1/Cache.scala index 6e6b9eb..a8036e5 100644 --- a/stage1/Cache.scala +++ b/stage1/Cache.scala @@ -1,6 +1,6 @@ package cbt /** -Caches exactly one value +Caches exactly one value. Is there a less boiler-platy way to achieve this, that doesn't require creating an instance for each thing you want to cache? */ diff --git a/stage1/CachingClassLoader.scala b/stage1/CachingClassLoader.scala index e75f14c..4ddebda 100644 --- a/stage1/CachingClassLoader.scala +++ b/stage1/CachingClassLoader.scala @@ -5,8 +5,13 @@ import scala.util.Try trait CachingClassLoader extends ClassLoader{ def logger: Logger - val cache = new KeyLockedLazyCache[String,Try[Class[_]]]( new ConcurrentHashMap, new ConcurrentHashMap, Some(logger) ) + val cache = new KeyLockedLazyCache[String,Option[Class[_]]]( new ConcurrentHashMap, new ConcurrentHashMap, Some(logger) ) override def loadClass(name: String, resolve: Boolean) = { - cache.get( name, Try(super.loadClass(name, resolve)) ).get + cache.get( name, Try(super.loadClass(name, resolve)).toOption ).getOrElse(null) + } + override def loadClass(name: String) = { + val _class = super.loadClass(name) + if(_class == null) throw new ClassNotFoundException(name) + else _class } } diff --git a/stage1/ClassLoaderCache.scala b/stage1/ClassLoaderCache.scala index 44b8d7d..e430ee1 100644 --- a/stage1/ClassLoaderCache.scala +++ b/stage1/ClassLoaderCache.scala @@ -4,15 +4,14 @@ import java.net._ import java.util.concurrent.ConcurrentHashMap import collection.JavaConversions._ -class ClassLoaderCache(logger: Logger){ +case class ClassLoaderCache( + logger: Logger, + private[cbt] permanentKeys: ConcurrentHashMap[String,AnyRef], + private[cbt] permanentClassLoaders: ConcurrentHashMap[AnyRef,ClassLoader] +){ val persistent = new KeyLockedLazyCache( - NailgunLauncher.classLoaderCacheKeys.asInstanceOf[ConcurrentHashMap[String,AnyRef]], - NailgunLauncher.classLoaderCacheValues.asInstanceOf[ConcurrentHashMap[AnyRef,ClassLoader]], - Some(logger) - ) - val transient = new KeyLockedLazyCache( - new ConcurrentHashMap[String,AnyRef], - new ConcurrentHashMap[AnyRef,ClassLoader], + permanentKeys, + permanentClassLoaders, Some(logger) ) override def toString = ( @@ -20,10 +19,6 @@ class ClassLoaderCache(logger: Logger){ ++ persistent.keys.keySet.toVector.map(_.toString.split(":").mkString("\n")).sorted.mkString("\n\n","\n\n","\n\n") ++ - "---------" - ++ - transient.keys.keySet.toVector.map(_.toString.split(":").mkString("\n")).sorted.mkString("\n\n","\n\n^","\n\n") - ++ ")" ) } diff --git a/stage1/ContextImplementation.scala b/stage1/ContextImplementation.scala new file mode 100644 index 0000000..d8b1382 --- /dev/null +++ b/stage1/ContextImplementation.scala @@ -0,0 +1,22 @@ +package cbt +import java.io._ +import java.util.concurrent.ConcurrentHashMap +import scala.collection.immutable.Seq +import java.lang._ + +case class ContextImplementation( + projectDirectory: File, + cwd: File, + argsArray: Array[String], + enabledLoggersArray: Array[String], + startCompat: Long, + cbtHasChangedCompat: Boolean, + versionOrNull: String, + scalaVersionOrNull: String, + permanentKeys: ConcurrentHashMap[String,AnyRef], + permanentClassLoaders: ConcurrentHashMap[AnyRef,ClassLoader], + cache: File, + cbtHome: File, + compatibilityTarget: File, + parentBuildOrNull: BuildInterface +) extends Context
\ No newline at end of file diff --git a/stage1/KeyLockedLazyCache.scala b/stage1/KeyLockedLazyCache.scala index aca5f74..4eff5b2 100644 --- a/stage1/KeyLockedLazyCache.scala +++ b/stage1/KeyLockedLazyCache.scala @@ -47,8 +47,16 @@ final private[cbt] class KeyLockedLazyCache[Key <: AnyRef,Value <: AnyRef]( def remove( key: Key ) = keys.synchronized{ assert(keys containsKey key) val lockableKey = keys get key - keys.remove( key ) - assert(values containsKey lockableKey) - values.remove( lockableKey ) + lockableKey.synchronized{ + if(values containsKey lockableKey){ + // this is so values in the process of being replaced (which mean they have a key but no value) + // are not being removed + keys.remove( key ) + values.remove( lockableKey ) + } + } + } + def containsKey( key: Key ) = keys.synchronized{ + keys containsKey key } } diff --git a/stage1/MavenRepository.scala b/stage1/MavenRepository.scala index bfd52a7..aa31cb8 100644 --- a/stage1/MavenRepository.scala +++ b/stage1/MavenRepository.scala @@ -1,22 +1,19 @@ package cbt import scala.collection.immutable.Seq +import java.io._ import java.net._ -case class MavenRepository(url: URL){ - def resolve( dependencies: MavenDependency* )(implicit logger: Logger): BoundMavenDependencies - = new BoundMavenDependencies( Seq(url), dependencies.to ) - def resolveOne( dependency: MavenDependency )(implicit logger: Logger): BoundMavenDependency - = BoundMavenDependency( dependency, Seq(url) ) +case class MavenResolver( cbtHasChanged: Boolean, mavenCache: File, urls: URL* ){ + def resolve( dependencies: MavenDependency* )(implicit logger: Logger): BoundMavenDependencies + = new BoundMavenDependencies( cbtHasChanged, mavenCache, urls.to, dependencies.to ) + def resolveOne( dependency: MavenDependency )(implicit logger: Logger): BoundMavenDependency + = BoundMavenDependency( cbtHasChanged, mavenCache, dependency, urls.to ) } -object MavenRepository{ - case class combine(repositories: MavenRepository*){ - def resolve( dependencies: MavenDependency* )(implicit logger: Logger): BoundMavenDependencies - = new BoundMavenDependencies( repositories.map(_.url).to, dependencies.to ) - } - def central = MavenRepository(new URL(NailgunLauncher.MAVEN_URL)) - def jcenter = MavenRepository(new URL("https://jcenter.bintray.com/releases")) - def bintray(owner: String) = MavenRepository(new URL(s"https://dl.bintray.com/$owner/maven")) +object MavenResolver{ + def central = new URL("https://repo1.maven.org/maven2") + def jcenter = new URL("https://jcenter.bintray.com/releases") + def bintray(owner: String) = new URL(s"https://dl.bintray.com/$owner/maven") private val sonatypeBase = new URL("https://oss.sonatype.org/content/repositories/") - def sonatype = MavenRepository(sonatypeBase ++ "releases") - def sonatypeSnapshots = MavenRepository(sonatypeBase ++ "snapshots") + def sonatype = sonatypeBase ++ "releases" + def sonatypeSnapshots = sonatypeBase ++ "snapshots" } diff --git a/stage1/MultiClassLoader.scala b/stage1/MultiClassLoader.scala index 5a93a63..3e3ba26 100644 --- a/stage1/MultiClassLoader.scala +++ b/stage1/MultiClassLoader.scala @@ -1,21 +1,19 @@ package cbt import java.net._ -import scala.util.Try import scala.collection.immutable.Seq // do not make this a case class, required object identity equality -class MultiClassLoader(parents: Seq[ClassLoader])(implicit val logger: Logger) extends ClassLoader with CachingClassLoader{ +class MultiClassLoader(final val parents: Seq[ClassLoader])(implicit val logger: Logger) extends ClassLoader(null) with CachingClassLoader{ override def findClass(name: String) = { parents.find( parent => try{ - parent.loadClass(name) - true + null != parent.loadClass(name) // FIXME: is it correct to just ignore the resolve argument here? } catch { case _:ClassNotFoundException => false } ).map( _.loadClass(name) - ).getOrElse( throw new ClassNotFoundException(name) ) + ).getOrElse( null ) } override def toString = ( scala.Console.BLUE diff --git a/stage1/Stage1.scala b/stage1/Stage1.scala index 77b88a2..bf2c272 100644 --- a/stage1/Stage1.scala +++ b/stage1/Stage1.scala @@ -1,13 +1,13 @@ package cbt import java.io._ +import java.util.concurrent.ConcurrentHashMap import scala.collection.immutable.Seq import scala.collection.JavaConverters._ -import paths._ - -final case class Stage1ArgsParser(_args: Seq[String]) { +final case class Stage1ArgsParser(__args: Seq[String]) { + val _args = __args.drop(1) /** * Raw parameters including their `-D` flag. **/ @@ -41,60 +41,149 @@ case class Stage2Args( cwd: File, args: Seq[String], cbtHasChanged: Boolean, - logger: Logger, - classLoaderCache: ClassLoaderCache -) - + classLoaderCache: ClassLoaderCache, + cache: File, + cbtHome: File +){ + val ClassLoaderCache( + logger, + permanentKeys, + permanentClassLoaders + ) = classLoaderCache +} object Stage1{ protected def newerThan( a: File, b: File ) ={ a.lastModified > b.lastModified } - def run(_args: Array[String], classLoader: ClassLoader, _cbtChanged: java.lang.Boolean, start: java.lang.Long): Int = { - val args = Stage1ArgsParser(_args.toVector) - val logger = new Logger(args.enabledLoggers, start) - logger.stage1(s"Stage1 start") + def getBuild( _context: java.lang.Object, _cbtChanged: java.lang.Boolean ) = { + val context = _context.asInstanceOf[Context] + val logger = new Logger( context.enabledLoggers, context.start ) + val (changed, classLoader) = buildStage2( + context.compatibilityTarget, + ClassLoaderCache( + logger, + context.permanentKeys, + context.permanentClassLoaders + ), + _cbtChanged, + context.cbtHome, + context.cache + ) + + classLoader + .loadClass("cbt.Stage2") + .getMethod( "getBuild", classOf[java.lang.Object], classOf[java.lang.Boolean] ) + .invoke(null, context, (_cbtChanged || changed): java.lang.Boolean) + } + + def buildStage2( + _compatibilityTarget: File, classLoaderCache: ClassLoaderCache, _cbtChanged: Boolean, cbtHome: File, cache: File + ): (Boolean, ClassLoader) = { + import classLoaderCache.logger val lib = new Stage1Lib(logger) import lib._ - val classLoaderCache = new ClassLoaderCache(logger) + val paths = Paths(cbtHome, cache) + import paths._ + + val stage2sourceFiles = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) + val cbtHasChanged = _cbtChanged || lib.needsUpdate(stage2sourceFiles, stage2StatusFile) + + val cls = this.getClass.getClassLoader.loadClass("cbt.NailgunLauncher") + + val cbtDependency = CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, _compatibilityTarget) - val sourceFiles = stage2.listFiles.toVector.filter(_.isFile).filter(_.toString.endsWith(".scala")) - val cbtHasChanged = _cbtChanged || lib.needsUpdate(sourceFiles, stage2StatusFile) logger.stage1("Compiling stage2 if necessary") compile( cbtHasChanged, - sourceFiles, stage2Target, stage2StatusFile, - CbtDependency().dependencyClasspath, + cbtHasChanged, + stage2sourceFiles, stage2Target, stage2StatusFile, + cbtDependency.dependencyClasspath, + mavenCache, Seq("-deprecation"), classLoaderCache, zincVersion = "0.3.9", scalaVersion = constants.scalaVersion ) logger.stage1(s"calling CbtDependency.classLoader") - if(cbtHasChanged) { - NailgunLauncher.stage2classLoader = CbtDependency().classLoader(classLoaderCache) - }else{ - classLoaderCache.transient.get( CbtDependency().classpath.string, NailgunLauncher.stage2classLoader ) + if( cbtHasChanged && classLoaderCache.persistent.containsKey( cbtDependency.classpath.string ) ) { + classLoaderCache.persistent.remove( cbtDependency.classpath.string ) } + val stage2ClassLoader = classLoaderCache.persistent.get( + cbtDependency.classpath.string, + cbtDependency.classLoader(classLoaderCache) + ) + + { + // a few classloader sanity checks + val compatibilityClassLoader = + CompatibilityDependency(cbtHasChanged, compatibilityTarget) + .classLoader(classLoaderCache) + assert( + classOf[BuildInterface].getClassLoader == compatibilityClassLoader, + classOf[BuildInterface].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ compatibilityClassLoader.toString + ) + //------------- + val stage1Dependency = Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget) + val stage1ClassLoader = + stage1Dependency.classLoader(classLoaderCache) + assert( + classOf[Stage1Dependency].getClassLoader == stage1ClassLoader, + classOf[Stage1Dependency].getClassLoader.toString ++ "\n\nis not the same as\n\n" ++ stage1ClassLoader.toString + ) + //------------- + assert( + Stage0Lib.get(stage2ClassLoader.getParent,"parents").asInstanceOf[Seq[ClassLoader]].contains(stage1ClassLoader), + stage1ClassLoader.toString ++ "\n\nis not contained in parents of\n\n" ++ stage2ClassLoader.toString + ) + } + + ( cbtHasChanged, stage2ClassLoader ) + } + + def run( + _args: Array[String], + cache: File, + cbtHome: File, + _cbtChanged: java.lang.Boolean, + start: java.lang.Long, + classLoaderCacheKeys: ConcurrentHashMap[String,AnyRef], + classLoaderCacheValues: ConcurrentHashMap[AnyRef,ClassLoader] + ): Int = { + val args = Stage1ArgsParser(_args.toVector) + val logger = new Logger(args.enabledLoggers, start) + logger.stage1(s"Stage1 start") + + val classLoaderCache = ClassLoaderCache( + logger, + classLoaderCacheKeys, + classLoaderCacheValues + ) + + + val (cbtHasChanged, classLoader) = buildStage2( Paths(cbtHome, cache).compatibilityTarget, classLoaderCache, _cbtChanged, cbtHome, cache ) + + val stage2Args = Stage2Args( + new File( args.args(0) ), + args.args.drop(1).toVector, + // launcher changes cause entire nailgun restart, so no need for them here + cbtHasChanged = cbtHasChanged, + classLoaderCache = classLoaderCache, + cache, + cbtHome + ) + logger.stage1(s"Run Stage2") - val cl = NailgunLauncher.stage2classLoader val exitCode = ( - cl + classLoader .loadClass( if(args.admin) "cbt.AdminStage2" else "cbt.Stage2" ) .getMethod( "run", classOf[Stage2Args] ) .invoke( null, - Stage2Args( - new File( args.args(0) ), - args.args.drop(1).toVector, - // launcher changes cause entire nailgun restart, so no need for them here - cbtHasChanged = cbtHasChanged, - logger, - classLoaderCache - ) + stage2Args ) match { case code: ExitCode => code case _ => ExitCode.Success diff --git a/stage1/Stage1Lib.scala b/stage1/Stage1Lib.scala index 985200e..5fd19a8 100644 --- a/stage1/Stage1Lib.scala +++ b/stage1/Stage1Lib.scala @@ -1,15 +1,15 @@ package cbt -import cbt.paths._ - import java.io._ import java.lang.reflect.InvocationTargetException import java.net._ +import java.nio.charset.StandardCharsets import java.nio.file._ import java.nio.file.attribute.FileTime import javax.tools._ import java.security._ -import java.util._ +import java.util.{Set=>_,Map=>_,_} +import java.util.concurrent.ConcurrentHashMap import javax.xml.bind.annotation.adapters.HexBinaryAdapter import scala.collection.immutable.Seq @@ -31,19 +31,8 @@ object CatchTrappedExitCode{ } } -case class Context( - projectDirectory: File, - cwd: File, - args: Seq[String], - logger: Logger, - cbtHasChanged: Boolean, - classLoaderCache: ClassLoaderCache, - version: Option[String] = None, - scalaVersion: Option[String] = None -) - class BaseLib{ - def realpath(name: File) = new File(Paths.get(name.getAbsolutePath).normalize.toString) + def realpath(name: File) = new File(java.nio.file.Paths.get(name.getAbsolutePath).normalize.toString) } class Stage1Lib( val logger: Logger ) extends BaseLib{ @@ -67,9 +56,10 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ def download(url: URL, target: File, sha1: Option[String]): Boolean = { if( target.exists ){ + logger.resolver(green("found ") ++ url.string) true } else { - val incomplete = Paths.get( target.string ++ ".incomplete" ); + val incomplete = ( target ++ ".incomplete" ).toPath; val connection = url.openConnection.asInstanceOf[HttpURLConnection] if(connection.getResponseCode != HttpURLConnection.HTTP_OK){ logger.resolver(blue("not found: ") ++ url.string) @@ -91,7 +81,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ assert( expected == actual, s"$expected == $actual" ) logger.resolver( green("verified") ++ " checksum for " ++ target.string) } - Files.move(incomplete, Paths.get(target.string), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + Files.move(incomplete, target.toPath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); true } } @@ -139,11 +129,13 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ } def compile( + cbtHasChanged: Boolean, needsRecompile: Boolean, files: Seq[File], compileTarget: File, statusFile: File, classpath: ClassPath, + mavenCache: File, scalacOptions: Seq[String] = Seq(), classLoaderCache: ClassLoaderCache, zincVersion: String, @@ -158,7 +150,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ None }else{ if( needsRecompile ){ - import MavenRepository.central + val central = MavenResolver(cbtHasChanged, mavenCache,MavenResolver.central) val zinc = central.resolveOne(MavenDependency("com.typesafe.zinc","zinc", zincVersion)) val zincDeps = zinc.transitiveDependencies @@ -166,8 +158,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ zincDeps .collect{ case d @ BoundMavenDependency( - MavenDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none), - _ + _, _, MavenDependency( "com.typesafe.sbt", "sbt-interface", _, Classifier.none), _ ) => d } .headOption @@ -178,8 +169,7 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ zincDeps .collect{ case d @ BoundMavenDependency( - MavenDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources), - _ + _, _, MavenDependency( "com.typesafe.sbt", "compiler-interface", _, Classifier.sources), _ ) => d } .headOption @@ -192,21 +182,51 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ val start = System.currentTimeMillis - val code = redirectOutToErr{ - lib.runMain( - "com.typesafe.zinc.Main", - Seq( - "-scala-compiler", scalaCompiler.toString, - "-scala-library", scalaLibrary.toString, - "-sbt-interface", sbtInterface.toString, - "-compiler-interface", compilerInterface.toString, - "-scala-extra", scalaReflect.toString, - "-cp", cp, - "-d", compileTarget.toString - ) ++ scalacOptions.map("-S"++_) ++ files.map(_.toString), - zinc.classLoader(classLoaderCache) + val _class = "com.typesafe.zinc.Main" + val dualArgs = + Seq( + "-scala-compiler", scalaCompiler.toString, + "-scala-library", scalaLibrary.toString, + "-sbt-interface", sbtInterface.toString, + "-compiler-interface", compilerInterface.toString, + "-scala-extra", scalaReflect.toString, + "-d", compileTarget.toString ) - } + val singleArgs = scalacOptions.map( "-S" ++ _ ) + + val code = + try{ + System.err.println("Compiling to " ++ compileTarget.toString) + redirectOutToErr{ + lib.runMain( + _class, + dualArgs ++ singleArgs ++ Seq( + "-cp", cp // let's put cp last. It so long + ) ++ files.map(_.toString), + zinc.classLoader(classLoaderCache) + ) + } + } catch { + case e: Exception => + System.err.println(red("The Scala compiler crashed. Try running it by hand:")) + System.out.println(s""" +java -cp \\ +${zinc.classpath.strings.mkString(":\\\n")} \\ +\\ +${_class} \\ +\\ +${dualArgs.grouped(2).map(_.mkString(" ")).mkString(" \\\n")} \\ +\\ +${singleArgs.mkString(" \\\n")} \\ +\\ +-cp \\ +${classpath.strings.mkString(":\\\n")} \\ +\\ +${files.sorted.mkString(" \\\n")} +""" + ) + ExitCode.Failure + } if(code == ExitCode.Success){ // write version and when last compilation started so we can trigger @@ -249,4 +269,93 @@ class Stage1Lib( val logger: Logger ) extends BaseLib{ MavenDependency( groupId, artifactId ++ "_" ++ scalaMajorVersion, version, classifier ) + + def cacheOnDisk[T] + ( cbtHasChanged: Boolean, cacheFile: File ) + ( deserialize: String => T ) + ( serialize: T => String ) + ( compute: => Seq[T] ) = { + if(!cbtHasChanged && cacheFile.exists){ + import collection.JavaConversions._ + Files + .readAllLines( cacheFile.toPath, StandardCharsets.UTF_8 ) + .toStream + .map(deserialize) + } else { + val result = compute + val string = result.map(serialize).mkString("\n") + Files.write(cacheFile.toPath, string.getBytes) + result + } + } + + def dependencyTreeRecursion(root: Dependency, indent: Int = 0): String = ( + ( " " * indent ) + ++ (if(root.needsUpdate) red(root.show) else root.show) + ++ root.dependencies.map( d => + "\n" ++ dependencyTreeRecursion(d,indent + 1) + ).mkString + ) + + def transitiveDependencies(dependency: Dependency): Seq[Dependency] = { + def linearize(deps: Seq[Dependency]): Seq[Dependency] = { + // Order is important here in order to generate the correct lineraized dependency order for EarlyDependencies + // (and maybe this as well in case we want to get rid of MultiClassLoader) + try{ + if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) ) + } catch{ + case e: Exception => throw new Exception(dependency.show, e) + } + } + + // FIXME: this is probably wrong too eager. + // We should consider replacing versions during traversals already + // not just replace after traversals, because that could mean we + // pulled down dependencies current versions don't even rely + // on anymore. + + val deps: Seq[Dependency] = linearize(dependency.dependencies).reverse.distinct.reverse + val hasInfo: Seq[Dependency with ArtifactInfo] = deps.collect{ case d:Dependency with ArtifactInfo => d } + val noInfo: Seq[Dependency] = deps.filter{ + case _:Dependency with ArtifactInfo => false + case _ => true + } + noInfo ++ BoundMavenDependency.updateOutdated( hasInfo ).reverse.distinct + } + + + def actual(current: Dependency, latest: Map[(String,String),Dependency]) = current match { + case d: ArtifactInfo => latest((d.groupId,d.artifactId)) + case d => d + } + + def classLoaderRecursion( dependency: Dependency, latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + val d = dependency + val dependencies = dependency.dependencies + def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { + if( dependency.dependencies.isEmpty ){ + // wrap for caching + new cbt.URLClassLoader( ClassPath(Seq()), ClassLoader.getSystemClassLoader().getParent() ) + } else if( dependencies.size == 1 ){ + classLoaderRecursion( dependencies.head, latest, cache ) + } else{ + val cp = d.dependencyClasspath.string + if( dependencies.exists(_.needsUpdate) && cache.persistent.containsKey(cp) ){ + cache.persistent.remove(cp) + } + cache.persistent.get( + cp, + new MultiClassLoader( + dependencies.map( classLoaderRecursion(_, latest, cache) ) + ) + ) + } + } + + val a = actual( dependency, latest ) + cache.persistent.get( + a.classpath.string, + new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) + ) + } } diff --git a/stage1/URLClassLoader.scala b/stage1/URLClassLoader.scala index 9e96992..5014cee 100644 --- a/stage1/URLClassLoader.scala +++ b/stage1/URLClassLoader.scala @@ -17,8 +17,8 @@ class URLClassLoader( classPath: ClassPath, parent: ClassLoader )( implicit val ++ ( getURLs.map(_.toString).sorted.mkString(",\n") ++ ( - if(getParent() != ClassLoader.getSystemClassLoader()) - ",\n" ++ getParent().toString + if(getParent() != ClassLoader.getSystemClassLoader().getParent()) + ",\n" ++ Option(getParent()).map(_.toString).getOrElse("null") else "" ) ).split("\n").map(" "++_).mkString("\n") diff --git a/stage1/cbt.scala b/stage1/cbt.scala index 01af0d5..4594135 100644 --- a/stage1/cbt.scala +++ b/stage1/cbt.scala @@ -2,6 +2,9 @@ package cbt import java.io._ import java.nio.file._ import java.net._ +import scala.collection.immutable.Seq +import java.util.concurrent.ConcurrentHashMap + object `package`{ private val lib = new BaseLib implicit class FileExtensionMethods( file: File ){ @@ -18,4 +21,71 @@ object `package`{ def ++( s: String ): URL = new URL( url.toString ++ s ) def string = url.toString } + implicit class BuildInterfaceExtensions(build: BuildInterface){ + import build._ + def triggerLoopFiles: Seq[File] = triggerLoopFilesArray.to + def crossScalaVersions: Seq[String] = crossScalaVersionsArray.to + } + implicit class ArtifactInfoExtensions(subject: ArtifactInfo){ + import subject._ + def str = s"$groupId:$artifactId:$version" + def show = this.getClass.getSimpleName ++ s"($str)" + } + implicit class DependencyExtensions(subject: Dependency){ + import subject._ + def dependencyClasspath: ClassPath = ClassPath(dependencyClasspathArray.to) + def exportedClasspath: ClassPath = ClassPath(exportedClasspathArray.to) + def classpath = exportedClasspath ++ dependencyClasspath + def dependencies: Seq[Dependency] = dependenciesArray.to + def needsUpdate: Boolean = needsUpdateCompat + } + implicit class ContextExtensions(subject: Context){ + import subject._ + val paths = Paths(cbtHome, cache) + implicit def logger: Logger = new Logger(enabledLoggers, start) + def classLoaderCache: ClassLoaderCache = new ClassLoaderCache( + logger, + permanentKeys, + permanentClassLoaders + ) + def cbtDependency = { + import paths._ + CbtDependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, stage2Target, compatibilityTarget) + } + def args: Seq[String] = argsArray.to + def enabledLoggers: Set[String] = enabledLoggersArray.to + def scalaVersion = Option(scalaVersionOrNull) + def version = Option(versionOrNull) + def parentBuild = Option(parentBuildOrNull) + def start: scala.Long = startCompat + def cbtHasChanged: scala.Boolean = cbtHasChangedCompat + + def copy( + projectDirectory: File = projectDirectory, + args: Seq[String] = args, + enabledLoggers: Set[String] = enabledLoggers, + cbtHasChanged: Boolean = cbtHasChanged, + version: Option[String] = version, + scalaVersion: Option[String] = scalaVersion, + cache: File = cache, + cbtHome: File = cbtHome, + parentBuild: Option[BuildInterface] = None + ): Context = ContextImplementation( + projectDirectory, + cwd, + args.to, + enabledLoggers.to, + startCompat, + cbtHasChangedCompat, + version.getOrElse(null), + scalaVersion.getOrElse(null), + permanentKeys, + permanentClassLoaders, + cache, + cbtHome, + compatibilityTarget, + parentBuild.getOrElse(null) + ) + } } + diff --git a/stage1/constants.scala b/stage1/constants.scala index 4c39237..437cf19 100644 --- a/stage1/constants.scala +++ b/stage1/constants.scala @@ -1,7 +1,7 @@ package cbt object constants{ - val scalaXmlVersion = NailgunLauncher.SCALA_XML_VERSION - val scalaVersion = NailgunLauncher.SCALA_VERSION - val zincVersion = NailgunLauncher.ZINC_VERSION + val scalaXmlVersion = "1.0.5" + val scalaVersion = "2.11.8" + val zincVersion = "0.3.9" val scalaMajorVersion = scalaVersion.split("\\.").take(2).mkString(".") } diff --git a/stage1/logger.scala b/stage1/logger.scala index c21dc86..1e0a693 100644 --- a/stage1/logger.scala +++ b/stage1/logger.scala @@ -12,7 +12,7 @@ case class Logger(enabledLoggers: Set[String], start: Long) { def log(name: String, msg: => String) = { val timeTaken = ((System.currentTimeMillis.toDouble - start) / 1000).toString - System.err.println( s"[${" "*(6-timeTaken.size)}$timeTaken][$name] $msg" ) + System.err.println( s"[$timeTaken][$name] $msg" ) } def showInvocation(method: String, args: Any) = method ++ "( " ++ args.toString ++ " )" diff --git a/stage1/paths.scala b/stage1/paths.scala index f27e538..c16c614 100644 --- a/stage1/paths.scala +++ b/stage1/paths.scala @@ -1,17 +1,17 @@ package cbt import java.io._ -object paths{ - val cbtHome: File = new File(Option(System.getenv("CBT_HOME")).get) - val mavenCache: File = cbtHome ++ "/cache/maven" +case class Paths(private val cbtHome: File, private val cache: File){ val userHome: File = new File(Option(System.getProperty("user.home")).get) - val bootstrapScala: File = cbtHome ++ "/bootstrap_scala" - val nailgun: File = new File(Option(System.getenv("NAILGUN")).get) - val stage1: File = new File(NailgunLauncher.STAGE1) + val nailgun: File = cbtHome ++ "/nailgun_launcher" + val stage1: File = cbtHome ++ "/stage1" val stage2: File = cbtHome ++ "/stage2" - private val target = Option(System.getenv("TARGET")).get.stripSuffix("/") + val mavenCache: File = cache ++ "/maven" + private val target = NailgunLauncher.TARGET.stripSuffix("/") val stage1Target: File = stage1 ++ ("/" ++ target) val stage2Target: File = stage2 ++ ("/" ++ target) val stage2StatusFile: File = stage2Target ++ ".last-success" + val compatibility: File = cbtHome ++ "/compatibility" + val compatibilityTarget: File = compatibility ++ ("/" ++ target) + val compatibilityStatusFile: File = compatibilityTarget ++ ".last-success" val nailgunTarget: File = nailgun ++ ("/" ++ target) - val sonatypeLogin: File = cbtHome ++ "/sonatype.login" } diff --git a/stage1/resolver.scala b/stage1/resolver.scala index 694bd83..f979247 100644 --- a/stage1/resolver.scala +++ b/stage1/resolver.scala @@ -5,39 +5,24 @@ import java.net._ import java.io._ import scala.collection.immutable.Seq import scala.xml._ -import paths._ import scala.concurrent._ import scala.concurrent.duration._ -private final class Tree( val root: Dependency, computeChildren: => Seq[Tree] ){ - lazy val children = computeChildren - def linearize: Seq[Dependency] = root +: children.flatMap(_.linearize) - def show(indent: Int = 0): Stream[Char] = { - (" " * indent ++ root.show ++ "\n").toStream #::: children.map(_.show(indent+1)).foldLeft(Stream.empty[Char])(_ #::: _) - } -} - -trait ArtifactInfo extends Dependency{ - def artifactId: String - def groupId: String - def version: String - - protected def str = s"$groupId:$artifactId:$version" - override def show = super.show ++ s"($str)" -} -abstract class Dependency{ - implicit def logger: Logger +abstract class DependencyImplementation extends Dependency{ + implicit protected def logger: Logger protected def lib = new Stage1Lib(logger) def needsUpdate: Boolean //def cacheClassLoader: Boolean = false private[cbt] def targetClasspath: ClassPath + def dependencyClasspathArray: Array[File] = dependencyClasspath.files.toArray + def exportedClasspathArray: Array[File] = exportedClasspath.files.toArray def exportedClasspath: ClassPath - def exportedJars: Seq[File] - def jars: Seq[File] = exportedJars ++ dependencyJars + def dependenciesArray: Array[Dependency] = dependencies.to - def canBeCached: Boolean + def needsUpdateCompat: java.lang.Boolean = needsUpdate + /* //private type BuildCache = KeyLockedLazyCache[Dependency, Future[ClassPath]] def exportClasspathConcurrently: ClassPath = { // FIXME: this should separate a blocking and a non-blocking EC @@ -49,7 +34,7 @@ abstract class Dependency{ .groupBy( d => (d.groupId,d.artifactId) ) .mapValues( _.head ) //, new BuildCache - ), + ), // FIXME Duration.Inf ) } @@ -60,9 +45,9 @@ abstract class Dependency{ The implementation of this method is untested and likely buggy at this stage. */ - private def exportClasspathConcurrently( - latest: Map[(String, String),ArtifactInfo]//, cache: BuildCache - )( implicit ec: ExecutionContext ): Future[ClassPath] = { + def exportClasspathConcurrently( + latest: Map[(String, String),Dependency with ArtifactInfo]//, cache: BuildCache + )( implicit ec: ExecutionContext ): Future[AnyRef] = { Future.sequence( // trigger compilation / download of all dependencies first this.dependencies.map{ d => @@ -73,7 +58,7 @@ abstract class Dependency{ } // // trigger compilation if not already triggered // cache.get( l, l.exportClasspathConcurrently( latest, cache ) ) - l.exportClasspathConcurrently( latest ) + l.exportClasspathConcurrently( latest ) // FIXME } ).map( // merge dependency classpaths into one @@ -84,165 +69,102 @@ abstract class Dependency{ exportedClasspath ) } + */ - protected def actual(current: Dependency, latest: Map[(String,String),Dependency]) = current match { - case d: ArtifactInfo => latest((d.groupId,d.artifactId)) - case d => d - } - private def dependencyClassLoader( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { - if( dependencies.isEmpty ){ - ClassLoader.getSystemClassLoader - } else if( dependencies.size == 1 ){ - dependencies.head.classLoaderRecursion( latest, cache ) - } else if( dependencies.forall(_.canBeCached) ){ - assert(transitiveDependencies.forall(_.canBeCached)) - cache.persistent.get( - dependencyClasspath.string, - new MultiClassLoader( - dependencies.map( _.classLoaderRecursion(latest, cache) ) - ) - ) - } else { - cache.transient.get( - dependencyClasspath.string, - new MultiClassLoader( - dependencies.map( _.classLoaderRecursion(latest, cache) ) - ) - ) - } - } - protected def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ): ClassLoader = { - val a = actual( this, latest ) - ( - if( canBeCached ){ - cache.persistent - } else { - cache.transient - } - ).get( - a.classpath.string, - new cbt.URLClassLoader( a.exportedClasspath, dependencyClassLoader(latest, cache) ) - ) - } - def classLoader( cache: ClassLoaderCache ): ClassLoader = { + def classLoader( cache: ClassLoaderCache ): ClassLoader = { + /* if( concurrencyEnabled ){ // trigger concurrent building / downloading dependencies exportClasspathConcurrently } - classLoaderRecursion( + */ + lib.classLoaderRecursion( + this, (this +: transitiveDependencies).collect{ case d: ArtifactInfo => d }.groupBy( d => (d.groupId,d.artifactId) ).mapValues(_.head), - cache + cache // FIXME ) } // FIXME: these probably need to update outdated as well def classpath : ClassPath = exportedClasspath ++ dependencyClasspath - def dependencyJars : Seq[File] = transitiveDependencies.flatMap(_.jars) - def dependencyClasspath : ClassPath = ClassPath.flatten( transitiveDependencies.map(_.exportedClasspath) ) + def dependencyClasspath : ClassPath = ClassPath( + transitiveDependencies + .flatMap(_.exportedClasspath.files) + .distinct // <- currently needed here to handle diamond dependencies on builds (duplicate in classpath) + ) def dependencies: Seq[Dependency] - private def resolveRecursive(parents: List[Dependency] = List()): Tree = { - // diff removes circular dependencies - new Tree(this, (dependencies diff parents).map(_.resolveRecursive(this :: parents))) - } - - def linearize(deps: Seq[Dependency]): Seq[Dependency] = - if(deps.isEmpty) deps else ( deps ++ linearize(deps.flatMap(_.dependencies)) ) - private object transitiveDependenciesCache extends Cache[Seq[Dependency]] /** return dependencies in order of linearized dependence. this is a bit tricky. */ def transitiveDependencies: Seq[Dependency] = transitiveDependenciesCache{ - val deps = linearize(dependencies) - val hasInfo = deps.collect{ case d:ArtifactInfo => d } - val noInfo = deps.filter{ - case _:ArtifactInfo => false - case _ => true - } - noInfo ++ BoundMavenDependency.updateOutdated( hasInfo ).reverse.distinct + lib.transitiveDependencies(this) } - def show: String = this.getClass.getSimpleName + override def show: String = this.getClass.getSimpleName // ========== debug ========== - def dependencyTree: String = dependencyTreeRecursion() - private def dependencyTreeRecursion(indent: Int = 0): String = ( - ( " " * indent ) - ++ (if(needsUpdate) lib.red(show) else show) - ++ dependencies.map( - "\n" ++ _.dependencyTreeRecursion(indent + 1) - ).mkString - ) + def dependencyTree: String = lib.dependencyTreeRecursion(this) } // TODO: all this hard codes the scala version, needs more flexibility -class ScalaCompilerDependency(version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(MavenRepository.central.url)) -class ScalaLibraryDependency (version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(MavenRepository.central.url)) -class ScalaReflectDependency (version: String)(implicit logger: Logger) extends BoundMavenDependency(MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(MavenRepository.central.url)) +class ScalaCompilerDependency(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-compiler",version, Classifier.none), Seq(MavenResolver.central)) +class ScalaLibraryDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-library",version, Classifier.none), Seq(MavenResolver.central)) +class ScalaReflectDependency (cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit logger: Logger) extends BoundMavenDependency(cbtHasChanged, mavenCache, MavenDependency("org.scala-lang","scala-reflect",version, Classifier.none), Seq(MavenResolver.central)) -case class ScalaDependencies(version: String)(implicit val logger: Logger) extends Dependency{ sd => +case class ScalaDependencies(cbtHasChanged: Boolean, mavenCache: File, version: String)(implicit val logger: Logger) extends DependencyImplementation{ sd => override final val needsUpdate = false - override def canBeCached = true def targetClasspath = ClassPath(Seq()) def exportedClasspath = ClassPath(Seq()) - def exportedJars = Seq[File]() def dependencies = Seq( - new ScalaCompilerDependency(version), - new ScalaLibraryDependency(version), - new ScalaReflectDependency(version) + new ScalaCompilerDependency(cbtHasChanged, mavenCache, version), + new ScalaLibraryDependency(cbtHasChanged, mavenCache, version), + new ScalaReflectDependency(cbtHasChanged, mavenCache, version) ) } -case class BinaryDependency( path: File, dependencies: Seq[Dependency], canBeCached: Boolean )(implicit val logger: Logger) extends Dependency{ +case class BinaryDependency( path: File, dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ def exportedClasspath = ClassPath(Seq(path)) - def exportedJars = Seq[File](path) override def needsUpdate = false def targetClasspath = exportedClasspath } /** Allows to easily assemble a bunch of dependencies */ -case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends Dependency{ +case class Dependencies( dependencies: Seq[Dependency] )(implicit val logger: Logger) extends DependencyImplementation{ override def needsUpdate = dependencies.exists(_.needsUpdate) - override def canBeCached = dependencies.forall(_.canBeCached) override def exportedClasspath = ClassPath(Seq()) - override def exportedJars = Seq() override def targetClasspath = ClassPath(Seq()) } object Dependencies{ def apply( dependencies: Dependency* )(implicit logger: Logger): Dependencies = Dependencies( dependencies.to ) } -case class Stage1Dependency()(implicit val logger: Logger) extends Dependency{ - override def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations - override def canBeCached = false +case class Stage1Dependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ + override def needsUpdate = cbtHasChanged override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq(nailgunTarget, stage1Target) ) - override def exportedJars = ???//Seq[File]() override def dependencies = Seq( - MavenRepository.central.resolve( + CompatibilityDependency(cbtHasChanged, compatibilityTarget), + MavenResolver(cbtHasChanged,mavenCache,MavenResolver.central).resolve( MavenDependency("org.scala-lang","scala-library",constants.scalaVersion), - MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,"1.0.5") + MavenDependency("org.scala-lang.modules","scala-xml_"+constants.scalaMajorVersion,constants.scalaXmlVersion) ) ) - // FIXME: implement sanity check to prevent using incompatible scala-library and xml version on cp - override def classLoaderRecursion( latest: Map[(String,String),Dependency], cache: ClassLoaderCache ) = { - val a = actual( this, latest ) - cache.transient.get( - a.classpath.string, - getClass.getClassLoader - ) - } } -case class CbtDependency()(implicit val logger: Logger) extends Dependency{ - override def needsUpdate = false // FIXME: think this through, might allow simplifications and/or optimizations - override def canBeCached = false +case class CompatibilityDependency(cbtHasChanged: Boolean, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ + override def needsUpdate = cbtHasChanged + override def targetClasspath = exportedClasspath + override def exportedClasspath = ClassPath( Seq(compatibilityTarget) ) + override def dependencies = Seq() +} +case class CbtDependency(cbtHasChanged: Boolean, mavenCache: File, nailgunTarget: File, stage1Target: File, stage2Target: File, compatibilityTarget: File)(implicit val logger: Logger) extends DependencyImplementation{ + override def needsUpdate = cbtHasChanged override def targetClasspath = exportedClasspath override def exportedClasspath = ClassPath( Seq( stage2Target ) ) - override def exportedJars = ??? override def dependencies = Seq( - Stage1Dependency(), - MavenRepository.central.resolve( + Stage1Dependency(cbtHasChanged, mavenCache, nailgunTarget, stage1Target, compatibilityTarget), + MavenResolver(cbtHasChanged, mavenCache,MavenResolver.central).resolve( MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r") ) @@ -259,16 +181,24 @@ abstract class DependenciesProxy{ } class BoundMavenDependencies( - urls: Seq[URL], mavenDependencies: Seq[MavenDependency] + cbtHasChanged: Boolean, mavenCache: File, urls: Seq[URL], mavenDependencies: Seq[MavenDependency] )(implicit logger: Logger) extends Dependencies( - mavenDependencies.map( BoundMavenDependency(_,urls) ) + mavenDependencies.map( BoundMavenDependency(cbtHasChanged,mavenCache,_,urls) ) ) case class MavenDependency( groupId: String, artifactId: String, version: String, classifier: Classifier = Classifier.none -) +){ + private[cbt] def serialize = groupId ++ ":" ++ artifactId ++ ":"++ version ++ ":" ++ classifier.name.getOrElse("") +} +object MavenDependency{ + private[cbt] def deserialize = (_:String).split(":") match { + case col => MavenDependency( col(0), col(1), col(2), Classifier(col.lift(3)) ) + } +} +// FIXME: take MavenResolver instead of mavenCache and repositories separately case class BoundMavenDependency( - mavenDependency: MavenDependency, repositories: Seq[URL] -)(implicit val logger: Logger) extends ArtifactInfo{ + cbtHasChanged: Boolean, mavenCache: File, mavenDependency: MavenDependency, repositories: Seq[URL] +)(implicit val logger: Logger) extends DependencyImplementation with ArtifactInfo{ val MavenDependency( groupId, artifactId, version, classifier ) = mavenDependency assert( Option(groupId).collect{ @@ -288,19 +218,19 @@ case class BoundMavenDependency( ) override def needsUpdate = false - override def canBeCached = dependencies.forall(_.canBeCached) private val groupPath = groupId.split("\\.").mkString("/") protected[cbt] def basePath = s"/$groupPath/$artifactId/$version/$artifactId-$version" ++ classifier.name.map("-"++_).getOrElse("") //private def coursierJarFile = userHome++"/.coursier/cache/v1/https/repo1.maven.org/maven2"++basePath++".jar" - override def exportedJars = Seq( jar ) + def exportedJars = Seq( jar ) override def exportedClasspath = ClassPath( exportedJars ) override def targetClasspath = exportedClasspath import scala.collection.JavaConversions._ private def resolve(suffix: String, hash: Option[String]): File = { + logger.resolver("Resolving "+this) val file = mavenCache ++ basePath ++ "." ++ suffix val urls = repositories.map(_ ++ basePath ++ "." ++ suffix) urls.find( @@ -330,13 +260,15 @@ case class BoundMavenDependency( private object pomCache extends Cache[File] def pom: File = pomCache{ resolve("pom", Some(pomSha1)) } - def pomXml = XML.loadFile(pom.string) + private def pomXml = XML.loadFile(pom.string) // ========== pom traversal ========== - lazy val transitivePom: Seq[BoundMavenDependency] = { + private lazy val transitivePom: Seq[BoundMavenDependency] = { (pomXml \ "parent").collect{ case parent => BoundMavenDependency( + cbtHasChanged: Boolean, + mavenCache, MavenDependency( (parent \ "groupId").text, (parent \ "artifactId").text, @@ -347,7 +279,7 @@ case class BoundMavenDependency( }.flatMap(_.transitivePom) :+ this } - lazy val properties: Map[String, String] = ( + private lazy val properties: Map[String, String] = ( transitivePom.flatMap{ d => val props = (d.pomXml \ "properties").flatMap(_.child).map{ tag => tag.label -> tag.text @@ -357,7 +289,7 @@ case class BoundMavenDependency( } ).toMap - lazy val dependencyVersions: Map[String, (String,String)] = + private lazy val dependencyVersions: Map[String, (String,String)] = transitivePom.flatMap( p => (p.pomXml \ "dependencyManagement" \ "dependencies" \ "dependency").map{ @@ -371,33 +303,36 @@ case class BoundMavenDependency( def dependencies: Seq[BoundMavenDependency] = { if(classifier == Classifier.sources) Seq() - else (pomXml \ "dependencies" \ "dependency").collect{ - case xml if (xml \ "scope").text == "" && (xml \ "optional").text != "true" => - val artifactId = lookup(xml,_ \ "artifactId").get - val groupId = - lookup(xml,_ \ "groupId").getOrElse( - dependencyVersions - .get(artifactId).map(_._1) - .getOrElse( - throw new Exception(s"$artifactId not found in \n$dependencyVersions") + else { + lib.cacheOnDisk( + cbtHasChanged, mavenCache ++ basePath ++ ".pom.dependencies" + )( MavenDependency.deserialize )( _.serialize ){ + (pomXml \ "dependencies" \ "dependency").collect{ + case xml if ( (xml \ "scope").text == "" || (xml \ "scope").text == "compile" ) && (xml \ "optional").text != "true" => + val artifactId = lookup(xml,_ \ "artifactId").get + val groupId = + lookup(xml,_ \ "groupId").getOrElse( + dependencyVersions + .get(artifactId).map(_._1) + .getOrElse( + throw new Exception(s"$artifactId not found in \n$dependencyVersions") + ) ) - ) - val version = - lookup(xml,_ \ "version").getOrElse( - dependencyVersions - .get(artifactId).map(_._2) - .getOrElse( - throw new Exception(s"$artifactId not found in \n$dependencyVersions") + val version = + lookup(xml,_ \ "version").getOrElse( + dependencyVersions + .get(artifactId).map(_._2) + .getOrElse( + throw new Exception(s"$artifactId not found in \n$dependencyVersions") + ) ) - ) - BoundMavenDependency( - MavenDependency( - groupId, artifactId, version, - Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) - ), - repositories - ) - }.toVector + val classifier = Classifier( Some( (xml \ "classifier").text ).filterNot(_ == "").filterNot(_ == null) ) + MavenDependency( groupId, artifactId, version, classifier ) + }.toVector + }.map( + BoundMavenDependency( cbtHasChanged, mavenCache, _, repositories ) + ).to + } } def lookup( xml: Node, accessor: Node => NodeSeq ): Option[String] = { //println("lookup in "++pomUrl) @@ -427,9 +362,9 @@ case class BoundMavenDependency( } object BoundMavenDependency{ def ValidIdentifier = "^([A-Za-z0-9_\\-.]+)$".r // according to maven's DefaultModelValidator.java - def semanticVersionLessThan(left: String, right: String) = { + def semanticVersionLessThan(left: Array[Either[Int,String]], right: Array[Either[Int,String]]) = { // FIXME: this ignores ends when different size - val zipped = left.split("\\.|\\-").map(toInt) zip right.split("\\.|\\-").map(toInt) + val zipped = left zip right val res = zipped.map { case (Left(i),Left(j)) => i compare j case (Right(i),Right(j)) => i compare j @@ -445,13 +380,16 @@ object BoundMavenDependency{ } /* this obviously should be overridable somehow */ def updateOutdated( - deps: Seq[ArtifactInfo], - versionLessThan: (String, String) => Boolean = semanticVersionLessThan - )(implicit logger: Logger): Seq[ArtifactInfo] = { + deps: Seq[Dependency with ArtifactInfo], + versionLessThan: (Array[Either[Int,String]], Array[Either[Int,String]]) => Boolean = semanticVersionLessThan + )(implicit logger: Logger): Seq[Dependency with ArtifactInfo] = { val latest = deps .groupBy( d => (d.groupId, d.artifactId) ) .mapValues( - _.sortBy( _.version )( Ordering.fromLessThan(versionLessThan) ) + _.groupBy(_.version) // remove duplicates + .map( _._2.head ) + .toVector + .sortBy( _.version.split("\\.|\\-").map(toInt) )( Ordering.fromLessThan(versionLessThan) ) .last ) deps.map{ diff --git a/stage2/AdminStage2.scala b/stage2/AdminStage2.scala index 9d7dcb2..f4e61d0 100644 --- a/stage2/AdminStage2.scala +++ b/stage2/AdminStage2.scala @@ -4,7 +4,7 @@ object AdminStage2 extends Stage2Base{ def run( _args: Stage2Args ): Unit = { val args = _args.args.dropWhile(Seq("admin","direct") contains _) val lib = new Lib(_args.logger) - val adminTasks = new AdminTasks(lib, args, _args.cwd, _args.classLoaderCache) + val adminTasks = new AdminTasks(lib, args, _args.cwd, _args.classLoaderCache, _args.cache, _args.cbtHome, _args.cbtHasChanged) new lib.ReflectObject(adminTasks){ def usage: String = "Available methods: " ++ lib.taskNames(adminTasks.getClass).mkString(" ") }.callNullary(args.lift(0)) diff --git a/stage2/AdminTasks.scala b/stage2/AdminTasks.scala index f189805..9086470 100644 --- a/stage2/AdminTasks.scala +++ b/stage2/AdminTasks.scala @@ -2,14 +2,25 @@ package cbt import scala.collection.immutable.Seq import java.io.{Console=>_,_} import java.nio.file._ -class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: ClassLoaderCache){ +class AdminTasks( + lib: Lib, + args: Seq[String], + cwd: File, + classLoaderCache: ClassLoaderCache, + cache: File, + cbtHome: File, + cbtHasChanged: Boolean +){ + private val paths = Paths(cbtHome, cache) + import paths._ + private val mavenCentral = MavenResolver(cbtHasChanged,mavenCache,MavenResolver.central) implicit val logger: Logger = lib.logger def resolve = { ClassPath.flatten( args(1).split(",").toVector.map{ d => val v = d.split(":") - MavenRepository.central.resolveOne(MavenDependency(v(0),v(1),v(2))).classpath + mavenCentral.resolveOne(MavenDependency(v(0),v(1),v(2))).classpath } ) } @@ -17,14 +28,14 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class args(1).split(",").toVector.map{ d => val v = d.split(":") - MavenRepository.central.resolveOne(MavenDependency(v(0),v(1),v(2))).dependencyTree + mavenCentral.resolveOne(MavenDependency(v(0),v(1),v(2))).dependencyTree }.mkString("\n\n") } def amm = ammonite def ammonite = { val version = args.lift(1).getOrElse(constants.scalaVersion) - val scalac = new ScalaCompilerDependency( version ) - val d = MavenRepository.central.resolveOne( + val scalac = new ScalaCompilerDependency( cbtHasChanged,mavenCache, version ) + val d = mavenCentral.resolveOne( MavenDependency( "com.lihaoyi","ammonite-repl_2.11.7",args.lift(1).getOrElse("0.5.7") ) @@ -36,7 +47,7 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class } def scala = { val version = args.lift(1).getOrElse(constants.scalaVersion) - val scalac = new ScalaCompilerDependency( version ) + val scalac = new ScalaCompilerDependency( cbtHasChanged, mavenCache, version ) lib.runMain( "scala.tools.nsc.MainGenericRunner", Seq("-cp", scalac.classpath.string), scalac.classLoader(classLoaderCache) ) @@ -49,16 +60,16 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class val scalaXmlVersion = args.lift(2).getOrElse(constants.scalaXmlVersion) val zincVersion = args.lift(3).getOrElse(constants.zincVersion) val scalaDeps = Seq( - MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)), - MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion)) + mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-reflect",scalaVersion)), + mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-compiler",scalaVersion)) ) val scalaXml = Dependencies( - MavenRepository.central.resolveOne(MavenDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion)), - MavenRepository.central.resolveOne(MavenDependency("org.scala-lang","scala-library",scalaVersion)) + mavenCentral.resolveOne(MavenDependency("org.scala-lang.modules","scala-xml_"+scalaMajorVersion,scalaXmlVersion)), + mavenCentral.resolveOne(MavenDependency("org.scala-lang","scala-library",scalaVersion)) ) - val zinc = MavenRepository.central.resolveOne(MavenDependency("com.typesafe.zinc","zinc",zincVersion)) + val zinc = mavenCentral.resolveOne(MavenDependency("com.typesafe.zinc","zinc",zincVersion)) def valName(dep: BoundMavenDependency) = { val words = dep.artifactId.split("_").head.split("-") @@ -66,24 +77,28 @@ class AdminTasks(lib: Lib, args: Seq[String], cwd: File, classLoaderCache: Class } def jarVal(dep: BoundMavenDependency) = "_" + valName(dep) +"Jar" - def transitive(dep: Dependency) = (dep +: dep.transitiveDependencies.reverse).collect{case d: BoundMavenDependency => d} + def transitive(dep: Dependency) = (dep +: lib.transitiveDependencies(dep).reverse).collect{case d: BoundMavenDependency => d} def codeEach(dep: Dependency) = { transitive(dep).tails.map(_.reverse).toVector.reverse.drop(1).map{ deps => val d = deps.last val parents = deps.dropRight(1) - val parentString = if(parents.isEmpty) "" else ( ", " ++ valName(parents.last) ) + val parentString = if(parents.isEmpty) "rootClassLoader" else ( valName(parents.last) ) val n = valName(d) s""" // ${d.groupId}:${d.artifactId}:${d.version} - download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}"); - ClassLoader $n = cachePut( - classLoader( ${n}File$parentString ), - ${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")} - );""" + download(new URL(mavenUrl + "${d.basePath}.jar"), Paths.get(${n}File), "${d.jarSha1}"); + + String[] ${n}ClasspathArray = new String[]{${deps.sortBy(_.jar).map(valName(_)+"File").mkString(", ")}}; + String ${n}Classpath = classpath( ${n}ClasspathArray ); + ClassLoader $n = + classLoaderCache.contains( ${n}Classpath ) + ? classLoaderCache.get( ${n}Classpath ) + : classLoaderCache.put( classLoader( ${n}File, $parentString ), ${n}Classpath );""" } } val assignments = codeEach(zinc) ++ codeEach(scalaXml) + val files = scalaDeps ++ transitive(scalaXml) ++ transitive(zinc) //{ case (name, dep) => s"$name =\n ${tree(dep, 4)};" }.mkString("\n\n ") val code = s"""// This file was auto-generated using `cbt admin cbtEarlyDependencies` package cbt; @@ -97,23 +112,29 @@ import static cbt.NailgunLauncher.*; class EarlyDependencies{ /** ClassLoader for stage1 */ - ClassLoader stage1; + ClassLoader classLoader; + String[] classpathArray; /** ClassLoader for zinc */ ClassLoader zinc; -${(scalaDeps ++ transitive(scalaXml) ++ transitive(zinc)).map(d => s""" String ${valName(d)}File = MAVEN_CACHE + "${d.basePath}.jar";""").mkString("\n")} +${files.map(d => s""" String ${valName(d)}File;""").mkString("\n")} + + public EarlyDependencies( + String mavenCache, String mavenUrl, ClassLoaderCache2<ClassLoader> classLoaderCache, ClassLoader rootClassLoader + ) throws Exception { +${files.map(d => s""" ${valName(d)}File = mavenCache + "${d.basePath}.jar";""").mkString("\n")} - public EarlyDependencies() throws MalformedURLException, IOException, NoSuchAlgorithmException{ -${scalaDeps.map(d => s""" download(new URL(MAVEN_URL + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} +${scalaDeps.map(d => s""" download(new URL(mavenUrl + "${d.basePath}.jar"), Paths.get(${valName(d)}File), "${d.jarSha1}");""").mkString("\n")} ${assignments.mkString("\n")} - stage1 = scalaXml_${scalaXmlVersion.replace(".","_")}_; + classLoader = scalaXml_${scalaXmlVersion.replace(".","_")}_; + classpathArray = scalaXml_${scalaXmlVersion.replace(".","_")}_ClasspathArray; zinc = zinc_${zincVersion.replace(".","_")}_; } } """ - val file = paths.nailgun ++ ("/" ++ "EarlyDependencies.java") + val file = nailgun ++ ("/" ++ "EarlyDependencies.java") Files.write( file.toPath, code.getBytes ) println( Console.GREEN ++ "Wrote " ++ file.string ++ Console.RESET ) } diff --git a/stage2/BasicBuild.scala b/stage2/BasicBuild.scala index 9f9dbdc..fb5e652 100644 --- a/stage2/BasicBuild.scala +++ b/stage2/BasicBuild.scala @@ -1,5 +1,4 @@ package cbt -import cbt.paths._ import java.io._ import java.net._ @@ -11,21 +10,30 @@ import java.util.jar._ import scala.collection.immutable.Seq import scala.util._ -class BasicBuild( context: Context ) extends Build( context ) -class Build(val context: Context) extends Dependency with TriggerLoop with SbtDependencyDsl{ +trait Recommended extends BasicBuild{ + override def scalacOptions = super.scalacOptions ++ Seq( + "-feature", + "-deprecation", + "-unchecked", + "-language:postfixOps", + "-language:implicitConversions", + "-language:higherKinds", + "-language:existentials" + ) +} +class BasicBuild(val context: Context) extends DependencyImplementation with BuildInterface with TriggerLoop with SbtDependencyDsl{ // library available to builds - implicit final val logger: Logger = context.logger - implicit final val classLoaderCache: ClassLoaderCache = context.classLoaderCache - implicit final val _context = context - override final protected val lib: Lib = new Lib(logger) + implicit protected final val logger: Logger = context.logger + implicit protected final val classLoaderCache: ClassLoaderCache = context.classLoaderCache + implicit protected final val _context = context + override protected final val lib: Lib = new Lib(logger) // ========== general stuff ========== - override def canBeCached = false def enableConcurrency = false final def projectDirectory: File = lib.realpath(context.projectDirectory) assert( projectDirectory.exists, "projectDirectory does not exist: " ++ projectDirectory.string ) - final def usage: String = lib.usage(this.getClass, context) + final def usage: String = lib.usage(this.getClass, show) // ========== meta data ========== @@ -33,11 +41,15 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe final def scalaVersion = context.scalaVersion getOrElse defaultScalaVersion final def scalaMajorVersion: String = lib.scalaMajorVersion(scalaVersion) def crossScalaVersions: Seq[String] = Seq(scalaVersion, "2.10.6") - def copy(context: Context) = lib.copy(this.getClass, context).asInstanceOf[Build] + final def crossScalaVersionsArray: Array[String] = crossScalaVersions.to + + // TODO: this should probably provide a nice error message if class has constructor signature + def copy(context: Context): BuildInterface = lib.copy(this.getClass, context).asInstanceOf[BuildInterface] def zincVersion = "0.3.9" def dependencies: Seq[Dependency] = Seq( - MavenRepository.central.resolve( + // FIXME: this should probably be removed + MavenResolver(context.cbtHasChanged, context.paths.mavenCache, MavenResolver.central).resolve( "org.scala-lang" % "scala-library" % scalaVersion ) ) @@ -63,16 +75,10 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe def compileStatusFile: File = compileTarget ++ ".last-success" /** Source directories and files. Defaults to .scala and .java files in src/ and top-level. */ - def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(sourceFileFilter) - - /** Which file endings to consider being source files. */ - def sourceFileFilter(file: File): Boolean = file.toString.endsWith(".scala") || file.toString.endsWith(".java") + def sources: Seq[File] = Seq(defaultSourceDirectory) ++ projectDirectory.listFiles.toVector.filter(lib.sourceFileFilter) /** Absolute path names for all individual files found in sources directly or contained in directories. */ - final def sourceFiles: Seq[File] = for { - base <- sources.filter(_.exists).map(lib.realpath) - file <- lib.listFilesRecursive(base) if file.isFile && sourceFileFilter(file) - } yield file + final def sourceFiles: Seq[File] = lib.sourceFiles(sources) protected def assertSourceDirectories(): Unit = { val nonExisting = @@ -103,38 +109,87 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe .flatMap(_.listFiles) .filter(_.toString.endsWith(".jar")) - //def cacheJar = false override def dependencyClasspath : ClassPath = ClassPath(localJars) ++ super.dependencyClasspath - override def dependencyJars : Seq[File] = localJars ++ super.dependencyJars def exportedClasspath : ClassPath = ClassPath(compile.toSeq:_*) def targetClasspath = ClassPath(Seq(compileTarget)) - def exportedJars: Seq[File] = Seq() // ========== compile, run, test ========== /** scalac options used for zinc and scaladoc */ - def scalacOptions: Seq[String] = Seq( "-feature", "-deprecation", "-unchecked" ) + def scalacOptions: Seq[String] = Seq() private object needsUpdateCache extends Cache[Boolean] def needsUpdate: Boolean = needsUpdateCache( context.cbtHasChanged || lib.needsUpdate( sourceFiles, compileStatusFile ) - || transitiveDependencies.exists(_.needsUpdate) + || transitiveDependencies.filterNot(_ == context.parentBuild).exists(_.needsUpdate) ) private object compileCache extends Cache[Option[File]] def compile: Option[File] = compileCache{ lib.compile( - needsUpdate, - sourceFiles, compileTarget, compileStatusFile, dependencyClasspath, scalacOptions, - context.classLoaderCache, zincVersion = zincVersion, scalaVersion = scalaVersion + context.cbtHasChanged, + needsUpdate || context.parentBuild.map(_.needsUpdate).getOrElse(false), + sourceFiles, compileTarget, compileStatusFile, dependencyClasspath, + context.paths.mavenCache, scalacOptions, context.classLoaderCache, + zincVersion = zincVersion, scalaVersion = scalaVersion ) } def runClass: String = "Main" def run: ExitCode = lib.runMainIfFound( runClass, context.args, classLoader(context.classLoaderCache) ) - def test: ExitCode = lib.test(context) + def test: Option[ExitCode] = { + lib.test(context) + } + + def recursiveSafe(_run: BuildInterface => Any): ExitCode = { + val builds = (this +: transitiveDependencies).collect{ + case b: BuildInterface => b + } + val results = builds.map(_run) + if( + results.forall{ + case Some(_:ExitCode) => true + case None => true + case _:ExitCode => true + case other => false + } + ){ + if( + results.collect{ + case Some(c:ExitCode) => c + case c:ExitCode => c + }.filter(_ != 0) + .nonEmpty + ) ExitCode.Failure + else ExitCode.Success + } else ExitCode.Success + } + + def recursive: ExitCode = { + recursiveUnsafe(context.args.lift(1)) + } + + def recursiveUnsafe(taskName: Option[String]): ExitCode = { + recursiveSafe{ + b => + System.err.println(b.show) + lib.trapExitCode{ // FIXME: trapExitCode does not seem to work here + try{ + new lib.ReflectBuild(b).callNullary(taskName) + ExitCode.Success + } catch { + case e: Throwable => println(e.getClass); throw e + } + } + ExitCode.Success + } + } + + def c = compile + def t = test + def rt = recursiveUnsafe(Some("test")) /* context.logger.composition(">"*80) @@ -148,6 +203,6 @@ class Build(val context: Context) extends Dependency with TriggerLoop with SbtDe */ // ========== cbt internals ========== - private[cbt] def finalBuild = this + def finalBuild: BuildInterface = this override def show = this.getClass.getSimpleName ++ "(" ++ projectDirectory.string ++ ")" } diff --git a/stage2/BuildBuild.scala b/stage2/BuildBuild.scala index c52c3c6..bab8d88 100644 --- a/stage2/BuildBuild.scala +++ b/stage2/BuildBuild.scala @@ -1,25 +1,63 @@ package cbt import java.io.File +import java.nio.file._ import scala.collection.immutable.Seq -class BuildBuild(context: Context) extends Build(context){ - override def dependencies = Seq( CbtDependency()(context.logger) ) ++ super.dependencies +class BuildBuild(context: Context) extends BasicBuild(context){ + override def dependencies = + super.dependencies :+ context.cbtDependency def managedBuildDirectory: File = lib.realpath( projectDirectory.parent ) - val managedBuild = try{ - val managedContext = context.copy( projectDirectory = managedBuildDirectory ) - val cl = classLoader(context.classLoaderCache) - logger.composition("Loading build at "+managedContext.projectDirectory) - cl - .loadClass(lib.buildClassName) - .getConstructor(classOf[Context]) - .newInstance(managedContext) - .asInstanceOf[Build] - } catch { - case e: ClassNotFoundException if e.getMessage == lib.buildClassName => - throw new Exception("You need to remove the directory or define a class Build in: "+context.projectDirectory) - case e: Exception => - throw new Exception("during build: "+context.projectDirectory, e) + private object managedBuildCache extends Cache[BuildInterface] + def managedBuild = managedBuildCache{ + try{ + val managedContext = context.copy( + projectDirectory = managedBuildDirectory, + parentBuild=Some(this) + ) + val managedBuildFile = projectDirectory++"/build.scala" + logger.composition("Loading build at "++managedContext.projectDirectory.toString) + ( + if(managedBuildFile.exists){ + val contents = new String(Files.readAllBytes(managedBuildFile.toPath)) + val cbtUrl = ("cbt:"++GitDependency.GitUrl.regex++"#[a-z0-9A-Z]+").r + cbtUrl + .findFirstIn(contents) + .flatMap{ + url => + val Array(base,hash) = url.drop(4).split("#") + if(context.cbtHome.string.contains(hash)) + None + else Some{ + val checkoutDirectory = new GitDependency(base, hash).checkout + val build = new BasicBuild( context.copy( projectDirectory = checkoutDirectory ++ "/nailgun_launcher" ) ) + val cl = build + .classLoader(classLoaderCache) + // Note: cbt can't use an old version of itself for building, + // otherwise we'd have to recursively build all versions since + // the beginning. Instead CBT always needs to build the pure Java + // Launcher in the checkout with itself and then run it via reflection. + cl + .loadClass( "cbt.NailgunLauncher" ) + .getMethod( "getBuild", classOf[AnyRef] ) + .invoke( null, managedContext.copy(cbtHome=checkoutDirectory) ) + } + }.getOrElse{ + classLoader(context.classLoaderCache) + .loadClass(lib.buildClassName) + .getConstructors.head + .newInstance(managedContext) + } + } else { + new BasicBuild(managedContext) + } + ).asInstanceOf[BuildInterface] + } catch { + case e: ClassNotFoundException if e.getMessage == lib.buildClassName => + throw new Exception("You need to remove the directory or define a class Build in: "+context.projectDirectory) + case e: Exception => + throw new Exception("during build: "+context.projectDirectory, e) + } } override def triggerLoopFiles = super.triggerLoopFiles ++ managedBuild.triggerLoopFiles - override def finalBuild = if( context.projectDirectory == context.cwd ) this else managedBuild.finalBuild + override def finalBuild: BuildInterface = if( context.projectDirectory == context.cwd ) this else managedBuild.finalBuild } diff --git a/stage2/BuildDependency.scala b/stage2/BuildDependency.scala index 8965cee..f6b6911 100644 --- a/stage2/BuildDependency.scala +++ b/stage2/BuildDependency.scala @@ -11,7 +11,8 @@ sealed abstract class ProjectProxy extends Ha{ def dependencies = Seq(delegate) } */ -trait TriggerLoop extends Dependency{ +trait TriggerLoop extends DependencyImplementation{ + final def triggerLoopFilesArray = triggerLoopFiles.toArray def triggerLoopFiles: Seq[File] } /** You likely want to use the factory method in the BasicBuild class instead of this. */ @@ -21,9 +22,7 @@ case class BuildDependency(context: Context) extends TriggerLoop{ final override lazy val lib: Lib = new Lib(logger) private val root = lib.loadRoot( context.copy(args=Seq()) ) lazy val build = root.finalBuild - override def canBeCached = build.canBeCached def exportedClasspath = ClassPath(Seq()) - def exportedJars = Seq() def dependencies = Seq(build) def triggerLoopFiles = root.triggerLoopFiles override final val needsUpdate = build.needsUpdate diff --git a/stage2/GitDependency.scala b/stage2/GitDependency.scala index 16423df..174f9ff 100644 --- a/stage2/GitDependency.scala +++ b/stage2/GitDependency.scala @@ -5,25 +5,25 @@ import scala.collection.immutable.Seq import org.eclipse.jgit.api._ import org.eclipse.jgit.lib.Ref +object GitDependency{ + val GitUrl = "(git:|https:|file:/)//([^/]+)/(.+)".r +} case class GitDependency( url: String, ref: String // example: git://github.com/cvogt/cbt.git#<some-hash> -)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends Dependency{ +)(implicit val logger: Logger, classLoaderCache: ClassLoaderCache, context: Context ) extends DependencyImplementation{ + import GitDependency._ override def lib = new Lib(logger) - override def canBeCached = true // TODO: add support for authentication via ssh and/or https // See http://www.codeaffine.com/2014/12/09/jgit-authentication/ + private val GitUrl( _, domain, path ) = url + - private val GitUrl = "(git|https)://([^/]+)/(.+)".r - private val GitUrl( _, domain, path ) = url - - private object dependenciesCache extends Cache[Seq[Dependency]] - def dependencies = dependenciesCache{ - val checkoutDirectory = paths.cbtHome ++ s"/cache/git/$domain/$path/$ref" + def checkout: File = { + val checkoutDirectory = context.cache ++ s"/git/$domain/$path/$ref" if(checkoutDirectory.exists){ logger.git(s"Found existing checkout of $url#$ref in $checkoutDirectory") } else { - logger.git(s"Cloning $url into $checkoutDirectory") val git = Git.cloneRepository() @@ -35,16 +35,17 @@ case class GitDependency( git.checkout() .setName(ref) .call() - } - val managedBuild = lib.loadDynamic( - context.copy( projectDirectory = checkoutDirectory, args = Seq() ) - ) - Seq( managedBuild ) + checkoutDirectory + } + private object dependencyCache extends Cache[Dependency] + def dependency = dependencyCache{ + BuildDependency( context.copy( projectDirectory = checkout ) ) } + def dependencies = Seq(dependency) + def exportedClasspath = ClassPath(Seq()) - def exportedJars = Seq() private[cbt] def targetClasspath = exportedClasspath def needsUpdate: Boolean = false } diff --git a/stage2/Lib.scala b/stage2/Lib.scala index cebdb92..1b19a5e 100644 --- a/stage2/Lib.scala +++ b/stage2/Lib.scala @@ -1,5 +1,4 @@ package cbt -import cbt.paths._ import java.io._ import java.net._ @@ -30,7 +29,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ .newInstance(context) /** Loads Build for given Context */ - def loadDynamic(context: Context, default: Context => Build = new Build(_)): Build = { + def loadDynamic(context: Context, default: Context => BuildInterface = new BasicBuild(_)): BuildInterface = { context.logger.composition( context.logger.showInvocation("Build.loadDynamic",context) ) loadRoot(context, default).finalBuild } @@ -38,7 +37,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ Loads whatever Build needs to be executed first in order to eventually build the build for the given context. This can either the Build itself, of if exists a BuildBuild or a BuildBuild for a BuildBuild and so on. */ - def loadRoot(context: Context, default: Context => Build = new Build(_)): Build = { + def loadRoot(context: Context, default: Context => BuildInterface = new BasicBuild(_)): BuildInterface = { context.logger.composition( context.logger.showInvocation("Build.loadRoot",context.projectDirectory) ) def findStartDir(projectDirectory: File): File = { val buildDir = realpath( projectDirectory ++ "/build" ) @@ -73,6 +72,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } def docJar( + cbtHasChanged: Boolean, scalaVersion: String, sourceFiles: Seq[File], dependencyClasspath: ClassPath, @@ -82,7 +82,8 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ scalaMajorVersion: String, version: String, compileArgs: Seq[String], - classLoaderCache: ClassLoaderCache + classLoaderCache: ClassLoaderCache, + mavenCache: File ): Option[File] = { if(sourceFiles.isEmpty){ None @@ -98,7 +99,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ runMain( "scala.tools.nsc.ScalaDoc", args, - ScalaDependencies(scalaVersion)(logger).classLoader(classLoaderCache) + ScalaDependencies(cbtHasChanged,mavenCache,scalaVersion)(logger).classLoader(classLoaderCache) ) } lib.jarFile( @@ -108,19 +109,24 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ } } - def test( context: Context ): ExitCode = { - val loggers = logger.enabledLoggers.mkString(",") - // FIXME: this is a hack to pass logger args on to the tests. - // should probably have a more structured way - val loggerArg = if(loggers != "") Some("-Dlog="++loggers) else None - - logger.lib(s"invoke testDefault( $context )") - val exitCode: ExitCode = loadDynamic( - context.copy( projectDirectory = context.projectDirectory ++ "/test", args = loggerArg.toVector ++ context.args ), - new Build(_) with mixins.Test - ).run - logger.lib(s"return testDefault( $context )") - exitCode + def test( context: Context ): Option[ExitCode] = { + if((context.projectDirectory ++ "/test").exists){ + val loggers = logger.enabledLoggers.mkString(",") + // FIXME: this is a hack to pass logger args on to the tests. + // should probably have a more structured way + val loggerArg = if(loggers != "") Some("-Dlog="++loggers) else None + + logger.lib(s"invoke testDefault( $context )") + val exitCode: ExitCode = + new ReflectBuild( + loadDynamic( + context.copy( projectDirectory = context.projectDirectory ++ "/test", args = loggerArg.toVector ++ context.args ), + new BasicBuild(_) with mixins.Test + ) + ).callNullary( Some("run") ) + logger.lib(s"return testDefault( $context )") + Some(exitCode) + } else None } // task reflection helpers @@ -145,13 +151,18 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ def taskNames(cls: Class[_]): Seq[String] = tasks(cls).keys.toVector.sorted - def usage(buildClass: Class[_], context: Context): String = { - val baseTasks = lib.taskNames(classOf[Build]) + def usage(buildClass: Class[_], show: String): String = { + val baseTasks = Seq( + classOf[BasicBuild], + classOf[PackageBuild], + classOf[PublishBuild], + classOf[Recommended] + ).flatMap(lib.taskNames).distinct.sorted val thisTasks = lib.taskNames(buildClass) diff baseTasks ( ( if( thisTasks.nonEmpty ){ - s"""Methods provided by Build ${context.projectDirectory} + s"""Methods provided by Build ${show} ${thisTasks.mkString(" ")} @@ -163,12 +174,13 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ ) ++ "\n" } - class ReflectBuild[T:scala.reflect.ClassTag](build: Build) extends ReflectObject(build){ - def usage = lib.usage(build.getClass, build.context) + class ReflectBuild[T:scala.reflect.ClassTag](build: BuildInterface) extends ReflectObject(build){ + def usage = lib.usage(build.getClass, build.show) } - abstract class ReflectObject[T:scala.reflect.ClassTag](obj: T){ + abstract class ReflectObject[T](obj: T){ def usage: String - def callNullary( taskName: Option[String] ): Unit = { + def callNullary( taskName: Option[String] ): ExitCode = { + logger.lib("Calling task " ++ taskName.toString) val ts = tasks(obj.getClass) taskName.map( NameTransformer.encode ).flatMap(ts.get).map{ method => val result: Option[Any] = Option(method.invoke(obj)) // null in case of Unit @@ -181,26 +193,30 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ scala.util.Try( value.getClass.getDeclaredMethod("toConsole") ) match { case scala.util.Success(toConsole) => println(toConsole.invoke(value)) + ExitCode.Success case scala.util.Failure(e) if Option(e.getMessage).getOrElse("") contains "toConsole" => value match { - case ExitCode(code) => System.exit(code) - case other => println( other.toString ) // no method .toConsole, using to String + case code:ExitCode => + code + case other => + println( other.toString ) // no method .toConsole, using to String + ExitCode.Success } case scala.util.Failure(e) => throw e } - }.getOrElse("") + }.getOrElse(ExitCode.Success) }.getOrElse{ taskName.foreach{ n => System.err.println(s"Method not found: $n") System.err.println("") } System.err.println(usage) - taskName.foreach{ _ => + taskName.map{ _ => ExitCode.Failure - } + }.getOrElse( ExitCode.Success ) } } } @@ -208,7 +224,17 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ // file system helpers def basename(path: File): String = path.toString.stripSuffix("/").split("/").last def dirname(path: File): File = new File(realpath(path).string.stripSuffix("/").split("/").dropRight(1).mkString("/")) - def nameAndContents(file: File) = basename(file) -> readAllBytes(Paths.get(file.toString)) + def nameAndContents(file: File) = basename(file) -> readAllBytes(file.toPath) + + /** Which file endings to consider being source files. */ + def sourceFileFilter(file: File): Boolean = file.toString.endsWith(".scala") || file.toString.endsWith(".java") + + def sourceFiles( sources: Seq[File], sourceFileFilter: File => Boolean = sourceFileFilter ): Seq[File] = { + for { + base <- sources.filter(_.exists).map(lib.realpath) + file <- lib.listFilesRecursive(base) if file.isFile && sourceFileFilter(file) + } yield file + } def jarFile( jarFile: File, files: Seq[File] ): Option[File] = { if( files.isEmpty ){ @@ -229,7 +255,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ val entry = new JarEntry( name ) entry.setTime(file.lastModified) jar.putNextEntry(entry) - jar.write( readAllBytes( Paths.get(file.toString) ) ) + jar.write( readAllBytes( file.toPath ) ) jar.closeEntry name } @@ -340,14 +366,14 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ else items.map(projection) } - def publishSnapshot( sourceFiles: Seq[File], artifacts: Seq[File], url: URL ): Unit = { + def publishSnapshot( sourceFiles: Seq[File], artifacts: Seq[File], url: URL, credentials: String ): Unit = { if(sourceFiles.nonEmpty){ val files = artifacts.map(nameAndContents) - uploadAll(url, files) + uploadAll(url, files, credentials) } } - def publishSigned( sourceFiles: Seq[File], artifacts: Seq[File], url: URL ): Unit = { + def publishSigned( sourceFiles: Seq[File], artifacts: Seq[File], url: URL, credentials: String ): Unit = { // TODO: make concurrency configurable here if(sourceFiles.nonEmpty){ val files = (artifacts ++ artifacts.map(sign)).map(nameAndContents) @@ -358,15 +384,15 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ ) } val all = (files ++ checksums) - uploadAll(url, all) + uploadAll(url, all, credentials) } } - def uploadAll(url: URL, nameAndContents: Seq[(String, Array[Byte])]): Unit = - nameAndContents.map{ case(name, content) => upload(name, content, url) } + def uploadAll(url: URL, nameAndContents: Seq[(String, Array[Byte])], credentials: String ): Unit = + nameAndContents.map{ case(name, content) => upload(name, content, url, credentials: String ) } - def upload(fileName: String, fileContents: Array[Byte], baseUrl: URL): Unit = { + def upload(fileName: String, fileContents: Array[Byte], baseUrl: URL, credentials: String): Unit = { import java.net._ import java.io._ logger.task("uploading "++fileName) @@ -374,8 +400,7 @@ final class Lib(logger: Logger) extends Stage1Lib(logger) with Scaffold{ val httpCon = url.openConnection.asInstanceOf[HttpURLConnection] httpCon.setDoOutput(true) httpCon.setRequestMethod("PUT") - val userPassword = new String(readAllBytes(sonatypeLogin.toPath)).trim - val encoding = new sun.misc.BASE64Encoder().encode(userPassword.getBytes) + val encoding = new sun.misc.BASE64Encoder().encode(credentials.getBytes) httpCon.setRequestProperty("Authorization", "Basic " ++ encoding) httpCon.setRequestProperty("Content-Type", "application/binary") httpCon.getOutputStream.write( diff --git a/stage2/PackageBuild.scala b/stage2/PackageBuild.scala index d24bf38..869af0e 100644 --- a/stage2/PackageBuild.scala +++ b/stage2/PackageBuild.scala @@ -20,9 +20,11 @@ abstract class PackageBuild(context: Context) extends BasicBuild(context) with A private object cacheDocBasicBuild extends Cache[Option[File]] def docJar: Option[File] = cacheDocBasicBuild{ - lib.docJar( scalaVersion, sourceFiles, dependencyClasspath, apiTarget, jarTarget, artifactId, scalaMajorVersion, version, scalacOptions, context.classLoaderCache ) + lib.docJar( + context.cbtHasChanged, + scalaVersion, sourceFiles, dependencyClasspath, apiTarget, + jarTarget, artifactId, scalaMajorVersion, version, + scalacOptions, context.classLoaderCache, context.paths.mavenCache + ) } - - override def jars = jar.toVector ++ dependencyJars - override def exportedJars: Seq[File] = jar.toVector } diff --git a/stage2/PublishBuild.scala b/stage2/PublishBuild.scala index 6b85b22..cc4f5e5 100644 --- a/stage2/PublishBuild.scala +++ b/stage2/PublishBuild.scala @@ -1,6 +1,7 @@ package cbt import java.io.File import java.net.URL +import java.nio.file.Files.readAllBytes import scala.collection.immutable.Seq abstract class PublishBuild(context: Context) extends PackageBuild(context){ @@ -37,10 +38,22 @@ abstract class PublishBuild(context: Context) extends PackageBuild(context){ def snapshotUrl = new URL("https://oss.sonatype.org/content/repositories/snapshots") def releaseUrl = new URL("https://oss.sonatype.org/service/local/staging/deploy/maven2") override def copy(context: Context) = super.copy(context).asInstanceOf[PublishBuild] + + protected def sonatypeCredentials = { + // FIXME: this should probably not use cbtHome, but some reference to the system's host cbt + new String(readAllBytes((context.cbtHome ++ "/sonatype.login").toPath)).trim + } + def publishSnapshot: Unit = { val snapshotBuild = copy( context.copy(version = Some(version+"-SNAPSHOT")) ) val files = snapshotBuild.pom +: snapshotBuild.`package` - lib.publishSnapshot(sourceFiles, files, snapshotUrl ++ releaseFolder ) + lib.publishSnapshot( + sourceFiles, files, snapshotUrl ++ releaseFolder, sonatypeCredentials + ) + } + def publishSigned: Unit = { + lib.publishSigned( + sourceFiles, pom +: `package`, releaseUrl ++ releaseFolder, sonatypeCredentials + ) } - def publishSigned: Unit = lib.publishSigned(sourceFiles, pom +: `package`, releaseUrl ++ releaseFolder ) } diff --git a/stage2/SbtDependencyDsl.scala b/stage2/SbtDependencyDsl.scala index 4fd4250..d8c0786 100644 --- a/stage2/SbtDependencyDsl.scala +++ b/stage2/SbtDependencyDsl.scala @@ -1,5 +1,5 @@ package cbt -trait SbtDependencyDsl{ self: Build => +trait SbtDependencyDsl{ self: BasicBuild => /** SBT-like dependency builder DSL for syntax compatibility */ class DependencyBuilder2( groupId: String, artifactId: String, scalaVersion: Option[String] ){ def %(version: String) = scalaVersion.map( @@ -13,7 +13,7 @@ trait SbtDependencyDsl{ self: Build => def %(artifactId: String) = new DependencyBuilder2( groupId, artifactId, None ) } implicit class DependencyBuilder3(d: MavenDependency){ - def %(classifier: String) = d.copy(classifier = Classifier(Some(classifier))) + def %(classifier: String): MavenDependency = d//.copy(classifier = Classifier(Some(classifier))) } /* diff --git a/stage2/Stage2.scala b/stage2/Stage2.scala index fa41d79..5316e78 100644 --- a/stage2/Stage2.scala +++ b/stage2/Stage2.scala @@ -4,12 +4,23 @@ import java.io._ import scala.collection.immutable.Seq -import cbt.paths._ object Stage2 extends Stage2Base{ + def getBuild(__context: java.lang.Object, _cbtChanged: java.lang.Boolean) = { + val cl1 = __context.getClass.getClassLoader + val cl2 = classOf[Context].getClassLoader + val _context = __context.asInstanceOf[Context] + val context = _context.copy( + cbtHasChanged = _context.cbtHasChanged || _cbtChanged + ) + val first = new Lib(context.logger).loadRoot( context ) + first.finalBuild + } + def run( args: Stage2Args ): Unit = { import args.logger - + val paths = Paths(args.cbtHome,args.cache) + import paths._ val lib = new Lib(args.logger) logger.stage2(s"Stage2 start") @@ -24,11 +35,26 @@ object Stage2 extends Stage2Base{ } val task = args.args.lift( taskIndex ) - val context = Context( args.cwd, args.cwd, args.args.drop( taskIndex ), logger, args.cbtHasChanged, args.classLoaderCache ) + val context: Context = ContextImplementation( + args.cwd, + args.cwd, + args.args.drop( taskIndex ).toArray, + logger.enabledLoggers.toArray, + logger.start, + args.cbtHasChanged, + null, + null, + args.permanentKeys, + args.permanentClassLoaders, + args.cache, + args.cbtHome, + compatibilityTarget, + null + ) val first = lib.loadRoot( context ) val build = first.finalBuild - def call(build: Build) = { + def call(build: BuildInterface) = { if(cross){ build.crossScalaVersions.foreach{ v => new lib.ReflectBuild( @@ -57,7 +83,7 @@ object Stage2 extends Stage2Base{ case file if triggerFiles.exists(file.toString startsWith _.toString) => val build = lib.loadDynamic(context) - logger.loop(s"Re-running $task for " ++ build.projectDirectory.toString) + logger.loop(s"Re-running $task for " ++ build.show) call(build) } } else { diff --git a/stage2/mixins.scala b/stage2/mixins.scala index fcffd97..1383324 100644 --- a/stage2/mixins.scala +++ b/stage2/mixins.scala @@ -2,31 +2,11 @@ package cbt package mixins import scala.collection.immutable.Seq import java.io._ -trait Test extends Build{ +trait Test extends BasicBuild{ lazy val testedBuild = BuildDependency( projectDirectory.parent ) override def dependencies = Seq( testedBuild ) ++ super.dependencies override def defaultScalaVersion = testedBuild.build.scalaVersion } -trait Sbt extends Build{ - override def sources = Seq( projectDirectory ++ "/src/main/scala" ) -} trait SbtTest extends Test{ override def sources = Vector( projectDirectory.parent ++ "/src/test/scala" ) } -trait ScalaTest extends Build with Test{ - def scalaTestVersion: String - - override def dependencies = super.dependencies :+ MavenRepository.central.resolve( - "org.scalatest" %% "scalatest" % scalaTestVersion - ) - - override def run: ExitCode = { - val discoveryPath = compile.toString++"/" - context.logger.lib("discoveryPath: " ++ discoveryPath) - lib.runMain( - "org.scalatest.tools.Runner", - Seq("-R", discoveryPath, "-oF") ++ context.args.drop(1), - classLoader(context.classLoaderCache) - ) - } -} diff --git a/test/build/build.scala b/test/build/build.scala index 29665a6..8989431 100644 --- a/test/build/build.scala +++ b/test/build/build.scala @@ -2,5 +2,5 @@ import cbt._ import java.io.File import scala.collection.immutable.Seq class Build(context: cbt.Context) extends BasicBuild(context){ - override def dependencies = Seq( CbtDependency() ) ++ super.dependencies + override def dependencies = Seq( context.cbtDependency ) ++ super.dependencies } diff --git a/test/simple-fixed/Main.scala b/test/simple-fixed/Main.scala new file mode 100644 index 0000000..1c423ca --- /dev/null +++ b/test/simple-fixed/Main.scala @@ -0,0 +1,6 @@ +import ai.x.diff +import org.eclipse.jgit.lib.Ref +import com.spotify.missinglink.ArtifactLoader +object Main extends App{ + println(diff.DiffShow.diff("a","b")) +} diff --git a/test/simple-fixed/build/build.scala b/test/simple-fixed/build/build.scala new file mode 100644 index 0000000..d088a40 --- /dev/null +++ b/test/simple-fixed/build/build.scala @@ -0,0 +1,34 @@ +import cbt._ +import cbt.extensions._ // FIXME: do not require this import +import scala.collection.immutable.Seq +import java.io.File + +// cbt:file:///Users/chris/code/cbt/#b65159f95421d9484f29327c11c0fa179eb7483f +class Build(context: cbt.Context) extends BasicBuild(context){ + override def dependencies = ( + super.dependencies + ++ + Seq( + GitDependency("https://github.com/xdotai/diff.git", "80a08bf45f7c4c3fd20c4bc6dbc9cae0072e3c0f"), + MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( + ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), + MavenDependency("joda-time", "joda-time", "2.9.2"), + // the below tests pom inheritance with dependencyManagement and variable substitution for pom properties + MavenDependency("org.eclipse.jgit", "org.eclipse.jgit", "4.2.0.201601211800-r"), + // the below tests pom inheritance with variable substitution for pom xml tag contents + MavenDependency("com.spotify", "missinglink-core", "0.1.1") + ), + MavenResolver( + context.cbtHasChanged, + context.paths.mavenCache, + MavenResolver.central, + MavenResolver.bintray("tpolecat"), + MavenResolver.sonatypeSnapshots + ).resolve( + "org.cvogt" %% "play-json-extensions" % "0.8.0", + "org.tpolecat" %% "tut-core" % "0.4.2", + "ai.x" %% "lens" % "1.0.0-SNAPSHOT" + ) + ) + ) +} diff --git a/test/simple/build/build.scala b/test/simple/build/build.scala index 173c1da..f54c3dc 100644 --- a/test/simple/build/build.scala +++ b/test/simple/build/build.scala @@ -7,8 +7,9 @@ class Build(context: cbt.Context) extends BasicBuild(context){ super.dependencies ++ Seq( - GitDependency("https://github.com/xdotai/diff.git", "666bbbf4dbff6fadc81c011ade7b83e91d3f9256"), - MavenRepository.central.resolve( + GitDependency("https://github.com/xdotai/diff.git", "698717469b8dd86e8570b86354892be9c0654caf"), + // FIXME: make the below less verbose + MavenResolver(context.cbtHasChanged,context.paths.mavenCache,MavenResolver.central).resolve( ScalaDependency("com.typesafe.play", "play-json", "2.4.4"), MavenDependency("joda-time", "joda-time", "2.9.2"), // the below tests pom inheritance with dependencyManagement and variable substitution for pom properties @@ -17,11 +18,16 @@ class Build(context: cbt.Context) extends BasicBuild(context){ MavenDependency("com.spotify", "missinglink-core", "0.1.1"), // the below tests pom inheritance with variable substitution being parts of strings MavenDependency("cc.factorie","factorie_2.11","1.2") + // the dependency below uses a maven version range. Currently not supported. + // TODO: put in a proper error message for version range not supported + //MavenDependency("com.github.nikita-volkov", "sext", "0.2.4") ), - MavenRepository.combine( - MavenRepository.central, - MavenRepository.bintray("tpolecat"), - MavenRepository.sonatypeSnapshots + MavenResolver( + context.cbtHasChanged, + context.paths.mavenCache, + MavenResolver.central, + MavenResolver.bintray("tpolecat"), + MavenResolver.sonatypeSnapshots ).resolve( "org.cvogt" %% "play-json-extensions" % "0.8.0", "org.tpolecat" %% "tut-core" % "0.4.2", diff --git a/test/test.scala b/test/test.scala index 29e6afa..4cee1f1 100644 --- a/test/test.scala +++ b/test/test.scala @@ -1,14 +1,16 @@ import cbt._ -import cbt.paths._ import scala.collection.immutable.Seq +import java.util.concurrent.ConcurrentHashMap import java.io.File // micro framework object Main{ def main(_args: Array[String]): Unit = { + val start = System.currentTimeMillis val args = new Stage1ArgsParser(_args.toVector) implicit val logger: Logger = new Logger(args.enabledLoggers, System.currentTimeMillis) val lib = new Lib(logger) + val cbtHome = new File(System.getenv("CBT_HOME")) var successes = 0 var failures = 0 @@ -71,11 +73,32 @@ object Main{ logger.test( "Running tests " ++ _args.toList.toString ) + val cache = cbtHome ++ "/cache" + val mavenCache = cache ++ "/maven" + val cbtHasChanged = true + val mavenCentral = MavenResolver(cbtHasChanged, mavenCache, MavenResolver.central) + { - val noContext = Context(cbtHome ++ "/test/nothing", cbtHome, Seq(), logger, false, new ClassLoaderCache(logger)) - val b = new Build(noContext){ + val noContext = ContextImplementation( + cbtHome ++ "/test/nothing", + cbtHome, + Array(), + Array(), + start, + cbtHasChanged, + null, + null, + new ConcurrentHashMap[String,AnyRef], + new ConcurrentHashMap[AnyRef,ClassLoader], + cache, + cbtHome, + cbtHome ++ "/compatibilityTarget", + null + ) + + val b = new BasicBuild(noContext){ override def dependencies = Seq( - MavenRepository.central.resolve( + mavenCentral.resolve( MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0"), MavenDependency("net.incongru.watchservice","barbary-watchservice","1.0") ) @@ -87,34 +110,34 @@ object Main{ // test that messed up artifacts crash with an assertion (which should tell the user what's up) assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("com.jcraft", "jsch", " 0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("com.jcraft", "jsch", " 0.1.53") ).classpath } assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("com.jcraft", null, "0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("com.jcraft", null, "0.1.53") ).classpath } assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("com.jcraft", "", " 0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("com.jcraft", "", " 0.1.53") ).classpath } assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("com.jcraft%", "jsch", " 0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("com.jcraft%", "jsch", " 0.1.53") ).classpath } assertException[AssertionError](){ - MavenRepository.central.resolveOne( MavenDependency("", "jsch", " 0.1.53") ).classpath + mavenCentral.resolveOne( MavenDependency("", "jsch", " 0.1.53") ).classpath } ( - MavenRepository.combine( - MavenRepository.central, MavenRepository.bintray("tpolecat") + MavenResolver( + cbtHasChanged, mavenCache, MavenResolver.central, MavenResolver.bintray("tpolecat") ).resolve( lib.ScalaDependency("org.tpolecat","tut-core","0.4.2", scalaMajorVersion="2.11") ).classpath.strings ++ - MavenRepository.sonatype.resolve( + MavenResolver(cbtHasChanged, mavenCache,MavenResolver.sonatype).resolve( MavenDependency("org.cvogt","play-json-extensions_2.11","0.8.0") ).classpath.strings ++ - MavenRepository.combine( - MavenRepository.central, MavenRepository.sonatypeSnapshots + MavenResolver( + cbtHasChanged, mavenCache, MavenResolver.central, MavenResolver.sonatypeSnapshots ).resolve( MavenDependency("ai.x","lens_2.11","1.0.0-SNAPSHOT") ).classpath.strings |